diff --git a/pkg/gallery/scan.go b/pkg/gallery/scan.go index 7c31c2ccf..c594cc953 100644 --- a/pkg/gallery/scan.go +++ b/pkg/gallery/scan.go @@ -16,6 +16,7 @@ import ( type FinderCreatorUpdater interface { Finder Create(ctx context.Context, newGallery *models.Gallery, fileIDs []file.ID) error + UpdatePartial(ctx context.Context, id int, updatedGallery models.GalleryPartial) (*models.Gallery, error) AddFileID(ctx context.Context, id int, fileID file.ID) error models.FileLoader } @@ -100,6 +101,10 @@ func (h *ScanHandler) associateExisting(ctx context.Context, existing []*models. if err := h.CreatorUpdater.AddFileID(ctx, i.ID, f.Base().ID); err != nil { return fmt.Errorf("adding file to gallery: %w", err) } + // update updated_at time + if _, err := h.CreatorUpdater.UpdatePartial(ctx, i.ID, models.NewGalleryPartial()); err != nil { + return fmt.Errorf("updating gallery: %w", err) + } } } diff --git a/pkg/image/scan.go b/pkg/image/scan.go index 61ef9e6e3..4caf567fc 100644 --- a/pkg/image/scan.go +++ b/pkg/image/scan.go @@ -22,6 +22,7 @@ type FinderCreatorUpdater interface { FindByFileID(ctx context.Context, fileID file.ID) ([]*models.Image, error) FindByFingerprints(ctx context.Context, fp []file.Fingerprint) ([]*models.Image, error) Create(ctx context.Context, newImage *models.ImageCreateInput) error + UpdatePartial(ctx context.Context, id int, updatedImage models.ImagePartial) (*models.Image, error) AddFileID(ctx context.Context, id int, fileID file.ID) error models.GalleryIDLoader models.ImageFileLoader @@ -169,6 +170,10 @@ func (h *ScanHandler) associateExisting(ctx context.Context, existing []*models. if err := h.CreatorUpdater.AddFileID(ctx, i.ID, f.ID); err != nil { return fmt.Errorf("adding file to image: %w", err) } + // update updated_at time + if _, err := h.CreatorUpdater.UpdatePartial(ctx, i.ID, models.NewImagePartial()); err != nil { + return fmt.Errorf("updating image: %w", err) + } } } diff --git a/pkg/scene/scan.go b/pkg/scene/scan.go index 41490b952..f6fe8adcb 100644 --- a/pkg/scene/scan.go +++ b/pkg/scene/scan.go @@ -133,6 +133,11 @@ func (h *ScanHandler) associateExisting(ctx context.Context, existing []*models. if err := h.CreatorUpdater.AddFileID(ctx, s.ID, f.ID); err != nil { return fmt.Errorf("adding file to scene: %w", err) } + + // update updated_at time + if _, err := h.CreatorUpdater.UpdatePartial(ctx, s.ID, models.NewScenePartial()); err != nil { + return fmt.Errorf("updating scene: %w", err) + } } } diff --git a/pkg/sqlite/database.go b/pkg/sqlite/database.go index d73abb977..5432faccd 100644 --- a/pkg/sqlite/database.go +++ b/pkg/sqlite/database.go @@ -21,7 +21,7 @@ import ( "github.com/stashapp/stash/pkg/logger" ) -var appSchemaVersion uint = 32 +var appSchemaVersion uint = 33 //go:embed migrations/*.sql var migrationsBox embed.FS diff --git a/pkg/sqlite/gallery.go b/pkg/sqlite/gallery.go index 995f37cb8..2a3edb95c 100644 --- a/pkg/sqlite/gallery.go +++ b/pkg/sqlite/gallery.go @@ -7,7 +7,6 @@ import ( "fmt" "path/filepath" "regexp" - "time" "github.com/doug-martin/goqu/v9" "github.com/doug-martin/goqu/v9/exp" @@ -31,17 +30,17 @@ const ( ) type galleryRow struct { - ID int `db:"id" goqu:"skipinsert"` - Title zero.String `db:"title"` - URL zero.String `db:"url"` - Date models.SQLiteDate `db:"date"` - Details zero.String `db:"details"` - Rating null.Int `db:"rating"` - Organized bool `db:"organized"` - StudioID null.Int `db:"studio_id,omitempty"` - FolderID null.Int `db:"folder_id,omitempty"` - CreatedAt time.Time `db:"created_at"` - UpdatedAt time.Time `db:"updated_at"` + ID int `db:"id" goqu:"skipinsert"` + Title zero.String `db:"title"` + URL zero.String `db:"url"` + Date models.SQLiteDate `db:"date"` + Details zero.String `db:"details"` + Rating null.Int `db:"rating"` + Organized bool `db:"organized"` + StudioID null.Int `db:"studio_id,omitempty"` + FolderID null.Int `db:"folder_id,omitempty"` + CreatedAt models.SQLiteTimestamp `db:"created_at"` + UpdatedAt models.SQLiteTimestamp `db:"updated_at"` } func (r *galleryRow) fromGallery(o models.Gallery) { @@ -56,8 +55,8 @@ func (r *galleryRow) fromGallery(o models.Gallery) { r.Organized = o.Organized r.StudioID = intFromPtr(o.StudioID) r.FolderID = nullIntFromFolderIDPtr(o.FolderID) - r.CreatedAt = o.CreatedAt - r.UpdatedAt = o.UpdatedAt + r.CreatedAt = models.SQLiteTimestamp{Timestamp: o.CreatedAt} + r.UpdatedAt = models.SQLiteTimestamp{Timestamp: o.UpdatedAt} } type galleryQueryRow struct { @@ -81,8 +80,8 @@ func (r *galleryQueryRow) resolve() *models.Gallery { StudioID: nullIntPtr(r.StudioID), FolderID: nullIntFolderIDPtr(r.FolderID), PrimaryFileID: nullIntFileIDPtr(r.PrimaryFileID), - CreatedAt: r.CreatedAt, - UpdatedAt: r.UpdatedAt, + CreatedAt: r.CreatedAt.Timestamp, + UpdatedAt: r.UpdatedAt.Timestamp, } if r.PrimaryFileFolderPath.Valid && r.PrimaryFileBasename.Valid { diff --git a/pkg/sqlite/image.go b/pkg/sqlite/image.go index 5d2eb22fd..ec1b90e58 100644 --- a/pkg/sqlite/image.go +++ b/pkg/sqlite/image.go @@ -5,7 +5,6 @@ import ( "database/sql" "fmt" "path/filepath" - "time" "github.com/jmoiron/sqlx" "github.com/stashapp/stash/pkg/file" @@ -28,14 +27,14 @@ const ( ) type imageRow struct { - ID int `db:"id" goqu:"skipinsert"` - Title zero.String `db:"title"` - Rating null.Int `db:"rating"` - Organized bool `db:"organized"` - OCounter int `db:"o_counter"` - StudioID null.Int `db:"studio_id,omitempty"` - CreatedAt time.Time `db:"created_at"` - UpdatedAt time.Time `db:"updated_at"` + ID int `db:"id" goqu:"skipinsert"` + Title zero.String `db:"title"` + Rating null.Int `db:"rating"` + Organized bool `db:"organized"` + OCounter int `db:"o_counter"` + StudioID null.Int `db:"studio_id,omitempty"` + CreatedAt models.SQLiteTimestamp `db:"created_at"` + UpdatedAt models.SQLiteTimestamp `db:"updated_at"` } func (r *imageRow) fromImage(i models.Image) { @@ -45,8 +44,8 @@ func (r *imageRow) fromImage(i models.Image) { r.Organized = i.Organized r.OCounter = i.OCounter r.StudioID = intFromPtr(i.StudioID) - r.CreatedAt = i.CreatedAt - r.UpdatedAt = i.UpdatedAt + r.CreatedAt = models.SQLiteTimestamp{Timestamp: i.CreatedAt} + r.UpdatedAt = models.SQLiteTimestamp{Timestamp: i.UpdatedAt} } type imageQueryRow struct { @@ -69,8 +68,8 @@ func (r *imageQueryRow) resolve() *models.Image { PrimaryFileID: nullIntFileIDPtr(r.PrimaryFileID), Checksum: r.PrimaryFileChecksum.String, - CreatedAt: r.CreatedAt, - UpdatedAt: r.UpdatedAt, + CreatedAt: r.CreatedAt.Timestamp, + UpdatedAt: r.UpdatedAt.Timestamp, } if r.PrimaryFileFolderPath.Valid && r.PrimaryFileBasename.Valid { diff --git a/pkg/sqlite/migrations/33_postmigrate.go b/pkg/sqlite/migrations/33_postmigrate.go new file mode 100644 index 000000000..5858d280e --- /dev/null +++ b/pkg/sqlite/migrations/33_postmigrate.go @@ -0,0 +1,116 @@ +package migrations + +import ( + "context" + "fmt" + "time" + + "github.com/jmoiron/sqlx" + "github.com/stashapp/stash/pkg/logger" + "github.com/stashapp/stash/pkg/sqlite" +) + +type schema33Migrator struct { + migrator +} + +func post33(ctx context.Context, db *sqlx.DB) error { + logger.Info("Running post-migration for schema version 33") + + m := schema33Migrator{ + migrator: migrator{ + db: db, + }, + } + + if err := m.migrateObjects(ctx, "scenes"); err != nil { + return fmt.Errorf("migrating scenes: %w", err) + } + if err := m.migrateObjects(ctx, "images"); err != nil { + return fmt.Errorf("migrating images: %w", err) + } + if err := m.migrateObjects(ctx, "galleries"); err != nil { + return fmt.Errorf("migrating galleries: %w", err) + } + + return nil +} + +func (m *schema33Migrator) migrateObjects(ctx context.Context, table string) error { + logger.Infof("Migrating %s table", table) + + const ( + limit = 1000 + logEvery = 10000 + ) + + lastID := 0 + count := 0 + + for { + gotSome := false + + if err := m.withTxn(ctx, func(tx *sqlx.Tx) error { + query := fmt.Sprintf("SELECT `id`, `created_at`, `updated_at` FROM `%s` WHERE `created_at` like '%% %%' OR `updated_at` like '%% %%'", table) + + if lastID != 0 { + query += fmt.Sprintf("AND `id` > %d ", lastID) + } + + query += fmt.Sprintf("ORDER BY `id` LIMIT %d", limit) + + rows, err := m.db.Query(query) + if err != nil { + return err + } + defer rows.Close() + + for rows.Next() { + var ( + id int + createdAt time.Time + updatedAt time.Time + ) + + err := rows.Scan(&id, &createdAt, &updatedAt) + if err != nil { + return err + } + + lastID = id + gotSome = true + count++ + + // convert incorrect timestamp string to correct one + // based on models.SQLTimestamp + fixedCreated := createdAt.Format(time.RFC3339) + fixedUpdated := updatedAt.Format(time.RFC3339) + + updateSQL := fmt.Sprintf("UPDATE `%s` SET `created_at` = ?, `updated_at` = ? WHERE `id` = ?", table) + + _, err = m.db.Exec(updateSQL, fixedCreated, fixedUpdated, id) + if err != nil { + return err + } + } + + return rows.Err() + }); err != nil { + return err + } + + if !gotSome { + break + } + + if count%logEvery == 0 { + logger.Infof("Migrated %d rows", count, table) + } + } + + return nil +} + +func init() { + sqlite.RegisterPostMigration(33, post33) +} diff --git a/pkg/sqlite/migrations/33_time_fix.up.sql b/pkg/sqlite/migrations/33_time_fix.up.sql new file mode 100644 index 000000000..fdf3e9cde --- /dev/null +++ b/pkg/sqlite/migrations/33_time_fix.up.sql @@ -0,0 +1 @@ +-- no schema changes \ No newline at end of file diff --git a/pkg/sqlite/scene.go b/pkg/sqlite/scene.go index f096db130..dc665c4a2 100644 --- a/pkg/sqlite/scene.go +++ b/pkg/sqlite/scene.go @@ -8,7 +8,6 @@ import ( "path/filepath" "strconv" "strings" - "time" "github.com/doug-martin/goqu/v9" "github.com/doug-martin/goqu/v9/exp" @@ -53,17 +52,17 @@ ORDER BY files.size DESC ` type sceneRow struct { - ID int `db:"id" goqu:"skipinsert"` - Title zero.String `db:"title"` - Details zero.String `db:"details"` - URL zero.String `db:"url"` - Date models.SQLiteDate `db:"date"` - Rating null.Int `db:"rating"` - Organized bool `db:"organized"` - OCounter int `db:"o_counter"` - StudioID null.Int `db:"studio_id,omitempty"` - CreatedAt time.Time `db:"created_at"` - UpdatedAt time.Time `db:"updated_at"` + ID int `db:"id" goqu:"skipinsert"` + Title zero.String `db:"title"` + Details zero.String `db:"details"` + URL zero.String `db:"url"` + Date models.SQLiteDate `db:"date"` + Rating null.Int `db:"rating"` + Organized bool `db:"organized"` + OCounter int `db:"o_counter"` + StudioID null.Int `db:"studio_id,omitempty"` + CreatedAt models.SQLiteTimestamp `db:"created_at"` + UpdatedAt models.SQLiteTimestamp `db:"updated_at"` } func (r *sceneRow) fromScene(o models.Scene) { @@ -78,8 +77,8 @@ func (r *sceneRow) fromScene(o models.Scene) { r.Organized = o.Organized r.OCounter = o.OCounter r.StudioID = intFromPtr(o.StudioID) - r.CreatedAt = o.CreatedAt - r.UpdatedAt = o.UpdatedAt + r.CreatedAt = models.SQLiteTimestamp{Timestamp: o.CreatedAt} + r.UpdatedAt = models.SQLiteTimestamp{Timestamp: o.UpdatedAt} } type sceneQueryRow struct { @@ -107,8 +106,8 @@ func (r *sceneQueryRow) resolve() *models.Scene { OSHash: r.PrimaryFileOshash.String, Checksum: r.PrimaryFileChecksum.String, - CreatedAt: r.CreatedAt, - UpdatedAt: r.UpdatedAt, + CreatedAt: r.CreatedAt.Timestamp, + UpdatedAt: r.UpdatedAt.Timestamp, } if r.PrimaryFileFolderPath.Valid && r.PrimaryFileBasename.Valid {