diff --git a/pkg/sqlite/database.go b/pkg/sqlite/database.go index 5432faccd..54732b0c5 100644 --- a/pkg/sqlite/database.go +++ b/pkg/sqlite/database.go @@ -21,7 +21,7 @@ import ( "github.com/stashapp/stash/pkg/logger" ) -var appSchemaVersion uint = 33 +var appSchemaVersion uint = 34 //go:embed migrations/*.sql var migrationsBox embed.FS diff --git a/pkg/sqlite/file.go b/pkg/sqlite/file.go index 7bfcd7804..448e9d3ff 100644 --- a/pkg/sqlite/file.go +++ b/pkg/sqlite/file.go @@ -7,7 +7,6 @@ import ( "fmt" "path/filepath" "strings" - "time" "github.com/doug-martin/goqu/v9" "github.com/doug-martin/goqu/v9/exp" @@ -30,14 +29,14 @@ const ( ) type basicFileRow struct { - ID file.ID `db:"id" goqu:"skipinsert"` - Basename string `db:"basename"` - ZipFileID null.Int `db:"zip_file_id"` - ParentFolderID file.FolderID `db:"parent_folder_id"` - Size int64 `db:"size"` - ModTime time.Time `db:"mod_time"` - CreatedAt time.Time `db:"created_at"` - UpdatedAt time.Time `db:"updated_at"` + ID file.ID `db:"id" goqu:"skipinsert"` + Basename string `db:"basename"` + ZipFileID null.Int `db:"zip_file_id"` + ParentFolderID file.FolderID `db:"parent_folder_id"` + Size int64 `db:"size"` + ModTime models.SQLiteTimestamp `db:"mod_time"` + CreatedAt models.SQLiteTimestamp `db:"created_at"` + UpdatedAt models.SQLiteTimestamp `db:"updated_at"` } func (r *basicFileRow) fromBasicFile(o file.BaseFile) { @@ -46,9 +45,9 @@ func (r *basicFileRow) fromBasicFile(o file.BaseFile) { r.ZipFileID = nullIntFromFileIDPtr(o.ZipFileID) r.ParentFolderID = o.ParentFolderID r.Size = o.Size - r.ModTime = o.ModTime - r.CreatedAt = o.CreatedAt - r.UpdatedAt = o.UpdatedAt + r.ModTime = models.SQLiteTimestamp{Timestamp: o.ModTime} + r.CreatedAt = models.SQLiteTimestamp{Timestamp: o.CreatedAt} + r.UpdatedAt = models.SQLiteTimestamp{Timestamp: o.UpdatedAt} } type videoFileRow struct { @@ -167,14 +166,14 @@ func (f *imageFileQueryRow) resolve() *file.ImageFile { } type fileQueryRow struct { - FileID null.Int `db:"file_id"` - Basename null.String `db:"basename"` - ZipFileID null.Int `db:"zip_file_id"` - ParentFolderID null.Int `db:"parent_folder_id"` - Size null.Int `db:"size"` - ModTime null.Time `db:"mod_time"` - CreatedAt null.Time `db:"file_created_at"` - UpdatedAt null.Time `db:"file_updated_at"` + FileID null.Int `db:"file_id"` + Basename null.String `db:"basename"` + ZipFileID null.Int `db:"zip_file_id"` + ParentFolderID null.Int `db:"parent_folder_id"` + Size null.Int `db:"size"` + ModTime models.NullSQLiteTimestamp `db:"mod_time"` + CreatedAt models.NullSQLiteTimestamp `db:"file_created_at"` + UpdatedAt models.NullSQLiteTimestamp `db:"file_updated_at"` ZipBasename null.String `db:"zip_basename"` ZipFolderPath null.String `db:"zip_folder_path"` @@ -190,14 +189,14 @@ func (r *fileQueryRow) resolve() file.File { ID: file.ID(r.FileID.Int64), DirEntry: file.DirEntry{ ZipFileID: nullIntFileIDPtr(r.ZipFileID), - ModTime: r.ModTime.Time, + ModTime: r.ModTime.Timestamp, }, Path: filepath.Join(r.FolderPath.String, r.Basename.String), ParentFolderID: file.FolderID(r.ParentFolderID.Int64), Basename: r.Basename.String, Size: r.Size.Int64, - CreatedAt: r.CreatedAt.Time, - UpdatedAt: r.UpdatedAt.Time, + CreatedAt: r.CreatedAt.Timestamp, + UpdatedAt: r.UpdatedAt.Timestamp, } if basic.ZipFileID != nil && r.ZipFolderPath.Valid && r.ZipBasename.Valid { diff --git a/pkg/sqlite/folder.go b/pkg/sqlite/folder.go index f9333c782..ea9153b2c 100644 --- a/pkg/sqlite/folder.go +++ b/pkg/sqlite/folder.go @@ -6,25 +6,25 @@ import ( "errors" "fmt" "path/filepath" - "time" "github.com/doug-martin/goqu/v9" "github.com/doug-martin/goqu/v9/exp" "github.com/jmoiron/sqlx" "github.com/stashapp/stash/pkg/file" + "github.com/stashapp/stash/pkg/models" "gopkg.in/guregu/null.v4" ) const folderTable = "folders" type folderRow struct { - ID file.FolderID `db:"id" goqu:"skipinsert"` - Path string `db:"path"` - ZipFileID null.Int `db:"zip_file_id"` - ParentFolderID null.Int `db:"parent_folder_id"` - ModTime time.Time `db:"mod_time"` - CreatedAt time.Time `db:"created_at"` - UpdatedAt time.Time `db:"updated_at"` + ID file.FolderID `db:"id" goqu:"skipinsert"` + Path string `db:"path"` + ZipFileID null.Int `db:"zip_file_id"` + ParentFolderID null.Int `db:"parent_folder_id"` + ModTime models.SQLiteTimestamp `db:"mod_time"` + CreatedAt models.SQLiteTimestamp `db:"created_at"` + UpdatedAt models.SQLiteTimestamp `db:"updated_at"` } func (r *folderRow) fromFolder(o file.Folder) { @@ -32,9 +32,9 @@ func (r *folderRow) fromFolder(o file.Folder) { r.Path = o.Path r.ZipFileID = nullIntFromFileIDPtr(o.ZipFileID) r.ParentFolderID = nullIntFromFolderIDPtr(o.ParentFolderID) - r.ModTime = o.ModTime - r.CreatedAt = o.CreatedAt - r.UpdatedAt = o.UpdatedAt + r.ModTime = models.SQLiteTimestamp{Timestamp: o.ModTime} + r.CreatedAt = models.SQLiteTimestamp{Timestamp: o.CreatedAt} + r.UpdatedAt = models.SQLiteTimestamp{Timestamp: o.UpdatedAt} } type folderQueryRow struct { @@ -49,12 +49,12 @@ func (r *folderQueryRow) resolve() *file.Folder { ID: r.ID, DirEntry: file.DirEntry{ ZipFileID: nullIntFileIDPtr(r.ZipFileID), - ModTime: r.ModTime, + ModTime: r.ModTime.Timestamp, }, Path: string(r.Path), ParentFolderID: nullIntFolderIDPtr(r.ParentFolderID), - CreatedAt: r.CreatedAt, - UpdatedAt: r.UpdatedAt, + CreatedAt: r.CreatedAt.Timestamp, + UpdatedAt: r.UpdatedAt.Timestamp, } if ret.ZipFileID != nil && r.ZipFolderPath.Valid && r.ZipBasename.Valid { diff --git a/pkg/sqlite/folder_test.go b/pkg/sqlite/folder_test.go index 5596205c8..71e45305a 100644 --- a/pkg/sqlite/folder_test.go +++ b/pkg/sqlite/folder_test.go @@ -102,9 +102,9 @@ func Test_FolderStore_Create(t *testing.T) { func Test_FolderStore_Update(t *testing.T) { var ( path = "path" - fileModTime = time.Date(2000, 1, 2, 3, 4, 5, 6, time.UTC) - createdAt = time.Date(2001, 1, 2, 3, 4, 5, 6, time.UTC) - updatedAt = time.Date(2002, 1, 2, 3, 4, 5, 6, time.UTC) + fileModTime = time.Date(2000, 1, 2, 3, 4, 5, 0, time.UTC) + createdAt = time.Date(2001, 1, 2, 3, 4, 5, 0, time.UTC) + updatedAt = time.Date(2002, 1, 2, 3, 4, 5, 0, time.UTC) ) tests := []struct { diff --git a/pkg/sqlite/gallery.go b/pkg/sqlite/gallery.go index 2a3edb95c..66c720dad 100644 --- a/pkg/sqlite/gallery.go +++ b/pkg/sqlite/gallery.go @@ -105,8 +105,8 @@ func (r *galleryRowRecord) fromPartial(o models.GalleryPartial) { r.setNullInt("rating", o.Rating) r.setBool("organized", o.Organized) r.setNullInt("studio_id", o.StudioID) - r.setTime("created_at", o.CreatedAt) - r.setTime("updated_at", o.UpdatedAt) + r.setSQLiteTimestamp("created_at", o.CreatedAt) + r.setSQLiteTimestamp("updated_at", o.UpdatedAt) } type GalleryStore struct { diff --git a/pkg/sqlite/image.go b/pkg/sqlite/image.go index ec1b90e58..ea68b2d96 100644 --- a/pkg/sqlite/image.go +++ b/pkg/sqlite/image.go @@ -89,8 +89,8 @@ func (r *imageRowRecord) fromPartial(i models.ImagePartial) { r.setBool("organized", i.Organized) r.setInt("o_counter", i.OCounter) r.setNullInt("studio_id", i.StudioID) - r.setTime("created_at", i.CreatedAt) - r.setTime("updated_at", i.UpdatedAt) + r.setSQLiteTimestamp("created_at", i.CreatedAt) + r.setSQLiteTimestamp("updated_at", i.UpdatedAt) } type ImageStore struct { diff --git a/pkg/sqlite/migrations/33_time_fix.up.sql b/pkg/sqlite/migrations/33_noop.up.sql similarity index 100% rename from pkg/sqlite/migrations/33_time_fix.up.sql rename to pkg/sqlite/migrations/33_noop.up.sql diff --git a/pkg/sqlite/migrations/33_postmigrate.go b/pkg/sqlite/migrations/33_postmigrate.go deleted file mode 100644 index 5858d280e..000000000 --- a/pkg/sqlite/migrations/33_postmigrate.go +++ /dev/null @@ -1,116 +0,0 @@ -package migrations - -import ( - "context" - "fmt" - "time" - - "github.com/jmoiron/sqlx" - "github.com/stashapp/stash/pkg/logger" - "github.com/stashapp/stash/pkg/sqlite" -) - -type schema33Migrator struct { - migrator -} - -func post33(ctx context.Context, db *sqlx.DB) error { - logger.Info("Running post-migration for schema version 33") - - m := schema33Migrator{ - migrator: migrator{ - db: db, - }, - } - - if err := m.migrateObjects(ctx, "scenes"); err != nil { - return fmt.Errorf("migrating scenes: %w", err) - } - if err := m.migrateObjects(ctx, "images"); err != nil { - return fmt.Errorf("migrating images: %w", err) - } - if err := m.migrateObjects(ctx, "galleries"); err != nil { - return fmt.Errorf("migrating galleries: %w", err) - } - - return nil -} - -func (m *schema33Migrator) migrateObjects(ctx context.Context, table string) error { - logger.Infof("Migrating %s table", table) - - const ( - limit = 1000 - logEvery = 10000 - ) - - lastID := 0 - count := 0 - - for { - gotSome := false - - if err := m.withTxn(ctx, func(tx *sqlx.Tx) error { - query := fmt.Sprintf("SELECT `id`, `created_at`, `updated_at` FROM `%s` WHERE `created_at` like '%% %%' OR `updated_at` like '%% %%'", table) - - if lastID != 0 { - query += fmt.Sprintf("AND `id` > %d ", lastID) - } - - query += fmt.Sprintf("ORDER BY `id` LIMIT %d", limit) - - rows, err := m.db.Query(query) - if err != nil { - return err - } - defer rows.Close() - - for rows.Next() { - var ( - id int - createdAt time.Time - updatedAt time.Time - ) - - err := rows.Scan(&id, &createdAt, &updatedAt) - if err != nil { - return err - } - - lastID = id - gotSome = true - count++ - - // convert incorrect timestamp string to correct one - // based on models.SQLTimestamp - fixedCreated := createdAt.Format(time.RFC3339) - fixedUpdated := updatedAt.Format(time.RFC3339) - - updateSQL := fmt.Sprintf("UPDATE `%s` SET `created_at` = ?, `updated_at` = ? WHERE `id` = ?", table) - - _, err = m.db.Exec(updateSQL, fixedCreated, fixedUpdated, id) - if err != nil { - return err - } - } - - return rows.Err() - }); err != nil { - return err - } - - if !gotSome { - break - } - - if count%logEvery == 0 { - logger.Infof("Migrated %d rows", count, table) - } - } - - return nil -} - -func init() { - sqlite.RegisterPostMigration(33, post33) -} diff --git a/pkg/sqlite/migrations/34_indexes.up.sql b/pkg/sqlite/migrations/34_indexes.up.sql new file mode 100644 index 000000000..bbd75a84a --- /dev/null +++ b/pkg/sqlite/migrations/34_indexes.up.sql @@ -0,0 +1,3 @@ +CREATE INDEX `index_performer_stash_ids_on_performer_id` ON `performer_stash_ids` (`performer_id`); +CREATE INDEX `index_scene_stash_ids_on_scene_id` ON `scene_stash_ids` (`scene_id`); +CREATE INDEX `index_studio_stash_ids_on_studio_id` ON `studio_stash_ids` (`studio_id`); diff --git a/pkg/sqlite/migrations/34_postmigrate.go b/pkg/sqlite/migrations/34_postmigrate.go new file mode 100644 index 000000000..e167c9a97 --- /dev/null +++ b/pkg/sqlite/migrations/34_postmigrate.go @@ -0,0 +1,154 @@ +package migrations + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/jmoiron/sqlx" + "github.com/stashapp/stash/pkg/logger" + "github.com/stashapp/stash/pkg/sqlite" +) + +type schema34Migrator struct { + migrator +} + +func post34(ctx context.Context, db *sqlx.DB) error { + logger.Info("Running post-migration for schema version 34") + + m := schema34Migrator{ + migrator: migrator{ + db: db, + }, + } + + objectCols := []string{ + "created_at", + "updated_at", + } + + filesystemCols := objectCols + filesystemCols = append(filesystemCols, "mod_time") + + if err := m.migrateObjects(ctx, "scenes", objectCols); err != nil { + return fmt.Errorf("migrating scenes: %w", err) + } + if err := m.migrateObjects(ctx, "images", objectCols); err != nil { + return fmt.Errorf("migrating images: %w", err) + } + if err := m.migrateObjects(ctx, "galleries", objectCols); err != nil { + return fmt.Errorf("migrating galleries: %w", err) + } + if err := m.migrateObjects(ctx, "files", filesystemCols); err != nil { + return fmt.Errorf("migrating files: %w", err) + } + if err := m.migrateObjects(ctx, "folders", filesystemCols); err != nil { + return fmt.Errorf("migrating folders: %w", err) + } + + return nil +} + +func (m *schema34Migrator) migrateObjects(ctx context.Context, table string, cols []string) error { + logger.Infof("Migrating %s table", table) + + quotedCols := make([]string, len(cols)+1) + quotedCols[0] = "`id`" + whereClauses := make([]string, len(cols)) + updateClauses := make([]string, len(cols)) + for i, v := range cols { + quotedCols[i+1] = "`" + v + "`" + whereClauses[i] = "`" + v + "` like '% %'" + updateClauses[i] = "`" + v + "` = ?" + } + + colList := strings.Join(quotedCols, ", ") + clauseList := strings.Join(whereClauses, " OR ") + updateList := strings.Join(updateClauses, ", ") + + const ( + limit = 1000 + logEvery = 10000 + ) + + lastID := 0 + count := 0 + + for { + gotSome := false + + if err := m.withTxn(ctx, func(tx *sqlx.Tx) error { + query := fmt.Sprintf("SELECT %s FROM `%s` WHERE (%s)", colList, table, clauseList) + + if lastID != 0 { + query += fmt.Sprintf(" AND `id` > %d ", lastID) + } + + query += fmt.Sprintf(" ORDER BY `id` LIMIT %d", limit) + + rows, err := m.db.Query(query) + if err != nil { + return err + } + defer rows.Close() + + for rows.Next() { + var ( + id int + ) + + timeValues := make([]interface{}, len(cols)+1) + timeValues[0] = &id + for i := range cols { + v := time.Time{} + timeValues[i+1] = &v + } + + err := rows.Scan(timeValues...) + if err != nil { + return err + } + + lastID = id + gotSome = true + count++ + + // convert incorrect timestamp string to correct one + // based on models.SQLTimestamp + args := make([]interface{}, len(cols)+1) + for i := range cols { + tv := timeValues[i+1].(*time.Time) + args[i] = tv.Format(time.RFC3339) + } + args[len(cols)] = id + + updateSQL := fmt.Sprintf("UPDATE `%s` SET %s WHERE `id` = ?", table, updateList) + + _, err = m.db.Exec(updateSQL, args...) + if err != nil { + return err + } + } + + return rows.Err() + }); err != nil { + return err + } + + if !gotSome { + break + } + + if count%logEvery == 0 { + logger.Infof("Migrated %d rows", count) + } + } + + return nil +} + +func init() { + sqlite.RegisterPostMigration(34, post34) +} diff --git a/pkg/sqlite/record.go b/pkg/sqlite/record.go index 2214766c4..a0fb789ef 100644 --- a/pkg/sqlite/record.go +++ b/pkg/sqlite/record.go @@ -83,12 +83,12 @@ func (r *updateRecord) setNullInt(destField string, v models.OptionalInt) { // } // } -func (r *updateRecord) setTime(destField string, v models.OptionalTime) { +func (r *updateRecord) setSQLiteTimestamp(destField string, v models.OptionalTime) { if v.Set { if v.Null { panic("null value not allowed in optional time") } - r.set(destField, v.Value) + r.set(destField, models.SQLiteTimestamp{Timestamp: v.Value}) } } diff --git a/pkg/sqlite/scene.go b/pkg/sqlite/scene.go index dc665c4a2..dd22907e2 100644 --- a/pkg/sqlite/scene.go +++ b/pkg/sqlite/scene.go @@ -130,8 +130,8 @@ func (r *sceneRowRecord) fromPartial(o models.ScenePartial) { r.setBool("organized", o.Organized) r.setInt("o_counter", o.OCounter) r.setNullInt("studio_id", o.StudioID) - r.setTime("created_at", o.CreatedAt) - r.setTime("updated_at", o.UpdatedAt) + r.setSQLiteTimestamp("created_at", o.CreatedAt) + r.setSQLiteTimestamp("updated_at", o.UpdatedAt) } type SceneStore struct {