More timestamp corrections (#2933)

* Fix incorrect timestamp updates
* Correct folder time fields
* Add migration with new indexes
* Correct mod_time format
* Add mod_time to data massage

pkg/sqlite/database.go

@@ -21,7 +21,7 @@ import (
 	"github.com/stashapp/stash/pkg/logger"
 )
 
-var appSchemaVersion uint = 33
+var appSchemaVersion uint = 34
 
 //go:embed migrations/*.sql
 var migrationsBox embed.FS

pkg/sqlite/file.go

@@ -7,7 +7,6 @@ import (
 	"fmt"
 	"path/filepath"
 	"strings"
-	"time"
 
 	"github.com/doug-martin/goqu/v9"
 	"github.com/doug-martin/goqu/v9/exp"
@@ -35,9 +34,9 @@ type basicFileRow struct {
 	ZipFileID      null.Int               `db:"zip_file_id"`
 	ParentFolderID file.FolderID          `db:"parent_folder_id"`
 	Size           int64                  `db:"size"`
-	ModTime        time.Time     `db:"mod_time"`
+	ModTime        models.SQLiteTimestamp `db:"mod_time"`
-	CreatedAt      time.Time     `db:"created_at"`
+	CreatedAt      models.SQLiteTimestamp `db:"created_at"`
-	UpdatedAt      time.Time     `db:"updated_at"`
+	UpdatedAt      models.SQLiteTimestamp `db:"updated_at"`
 }
 
 func (r *basicFileRow) fromBasicFile(o file.BaseFile) {
@@ -46,9 +45,9 @@ func (r *basicFileRow) fromBasicFile(o file.BaseFile) {
 	r.ZipFileID = nullIntFromFileIDPtr(o.ZipFileID)
 	r.ParentFolderID = o.ParentFolderID
 	r.Size = o.Size
-	r.ModTime = o.ModTime
+	r.ModTime = models.SQLiteTimestamp{Timestamp: o.ModTime}
-	r.CreatedAt = o.CreatedAt
+	r.CreatedAt = models.SQLiteTimestamp{Timestamp: o.CreatedAt}
-	r.UpdatedAt = o.UpdatedAt
+	r.UpdatedAt = models.SQLiteTimestamp{Timestamp: o.UpdatedAt}
 }
 
 type videoFileRow struct {
@@ -172,9 +171,9 @@ type fileQueryRow struct {
 	ZipFileID      null.Int                   `db:"zip_file_id"`
 	ParentFolderID null.Int                   `db:"parent_folder_id"`
 	Size           null.Int                   `db:"size"`
-	ModTime        null.Time   `db:"mod_time"`
+	ModTime        models.NullSQLiteTimestamp `db:"mod_time"`
-	CreatedAt      null.Time   `db:"file_created_at"`
+	CreatedAt      models.NullSQLiteTimestamp `db:"file_created_at"`
-	UpdatedAt      null.Time   `db:"file_updated_at"`
+	UpdatedAt      models.NullSQLiteTimestamp `db:"file_updated_at"`
 
 	ZipBasename   null.String `db:"zip_basename"`
 	ZipFolderPath null.String `db:"zip_folder_path"`
@@ -190,14 +189,14 @@ func (r *fileQueryRow) resolve() file.File {
 		ID: file.ID(r.FileID.Int64),
 		DirEntry: file.DirEntry{
 			ZipFileID: nullIntFileIDPtr(r.ZipFileID),
-			ModTime:   r.ModTime.Time,
+			ModTime:   r.ModTime.Timestamp,
 		},
 		Path:           filepath.Join(r.FolderPath.String, r.Basename.String),
 		ParentFolderID: file.FolderID(r.ParentFolderID.Int64),
 		Basename:       r.Basename.String,
 		Size:           r.Size.Int64,
-		CreatedAt:      r.CreatedAt.Time,
+		CreatedAt:      r.CreatedAt.Timestamp,
-		UpdatedAt:      r.UpdatedAt.Time,
+		UpdatedAt:      r.UpdatedAt.Timestamp,
 	}
 
 	if basic.ZipFileID != nil && r.ZipFolderPath.Valid && r.ZipBasename.Valid {

pkg/sqlite/folder.go

@@ -6,12 +6,12 @@ import (
 	"errors"
 	"fmt"
 	"path/filepath"
-	"time"
 
 	"github.com/doug-martin/goqu/v9"
 	"github.com/doug-martin/goqu/v9/exp"
 	"github.com/jmoiron/sqlx"
 	"github.com/stashapp/stash/pkg/file"
+	"github.com/stashapp/stash/pkg/models"
 	"gopkg.in/guregu/null.v4"
 )
 
@@ -22,9 +22,9 @@ type folderRow struct {
 	Path           string                 `db:"path"`
 	ZipFileID      null.Int               `db:"zip_file_id"`
 	ParentFolderID null.Int               `db:"parent_folder_id"`
-	ModTime        time.Time     `db:"mod_time"`
+	ModTime        models.SQLiteTimestamp `db:"mod_time"`
-	CreatedAt      time.Time     `db:"created_at"`
+	CreatedAt      models.SQLiteTimestamp `db:"created_at"`
-	UpdatedAt      time.Time     `db:"updated_at"`
+	UpdatedAt      models.SQLiteTimestamp `db:"updated_at"`
 }
 
 func (r *folderRow) fromFolder(o file.Folder) {
@@ -32,9 +32,9 @@ func (r *folderRow) fromFolder(o file.Folder) {
 	r.Path = o.Path
 	r.ZipFileID = nullIntFromFileIDPtr(o.ZipFileID)
 	r.ParentFolderID = nullIntFromFolderIDPtr(o.ParentFolderID)
-	r.ModTime = o.ModTime
+	r.ModTime = models.SQLiteTimestamp{Timestamp: o.ModTime}
-	r.CreatedAt = o.CreatedAt
+	r.CreatedAt = models.SQLiteTimestamp{Timestamp: o.CreatedAt}
-	r.UpdatedAt = o.UpdatedAt
+	r.UpdatedAt = models.SQLiteTimestamp{Timestamp: o.UpdatedAt}
 }
 
 type folderQueryRow struct {
@@ -49,12 +49,12 @@ func (r *folderQueryRow) resolve() *file.Folder {
 		ID: r.ID,
 		DirEntry: file.DirEntry{
 			ZipFileID: nullIntFileIDPtr(r.ZipFileID),
-			ModTime:   r.ModTime,
+			ModTime:   r.ModTime.Timestamp,
 		},
 		Path:           string(r.Path),
 		ParentFolderID: nullIntFolderIDPtr(r.ParentFolderID),
-		CreatedAt:      r.CreatedAt,
+		CreatedAt:      r.CreatedAt.Timestamp,
-		UpdatedAt:      r.UpdatedAt,
+		UpdatedAt:      r.UpdatedAt.Timestamp,
 	}
 
 	if ret.ZipFileID != nil && r.ZipFolderPath.Valid && r.ZipBasename.Valid {

pkg/sqlite/folder_test.go

@@ -102,9 +102,9 @@ func Test_FolderStore_Create(t *testing.T) {
 func Test_FolderStore_Update(t *testing.T) {
 	var (
 		path        = "path"
-		fileModTime = time.Date(2000, 1, 2, 3, 4, 5, 6, time.UTC)
+		fileModTime = time.Date(2000, 1, 2, 3, 4, 5, 0, time.UTC)
-		createdAt   = time.Date(2001, 1, 2, 3, 4, 5, 6, time.UTC)
+		createdAt   = time.Date(2001, 1, 2, 3, 4, 5, 0, time.UTC)
-		updatedAt   = time.Date(2002, 1, 2, 3, 4, 5, 6, time.UTC)
+		updatedAt   = time.Date(2002, 1, 2, 3, 4, 5, 0, time.UTC)
 	)
 
 	tests := []struct {

pkg/sqlite/gallery.go

@@ -105,8 +105,8 @@ func (r *galleryRowRecord) fromPartial(o models.GalleryPartial) {
 	r.setNullInt("rating", o.Rating)
 	r.setBool("organized", o.Organized)
 	r.setNullInt("studio_id", o.StudioID)
-	r.setTime("created_at", o.CreatedAt)
+	r.setSQLiteTimestamp("created_at", o.CreatedAt)
-	r.setTime("updated_at", o.UpdatedAt)
+	r.setSQLiteTimestamp("updated_at", o.UpdatedAt)
 }
 
 type GalleryStore struct {

pkg/sqlite/image.go

@@ -89,8 +89,8 @@ func (r *imageRowRecord) fromPartial(i models.ImagePartial) {
 	r.setBool("organized", i.Organized)
 	r.setInt("o_counter", i.OCounter)
 	r.setNullInt("studio_id", i.StudioID)
-	r.setTime("created_at", i.CreatedAt)
+	r.setSQLiteTimestamp("created_at", i.CreatedAt)
-	r.setTime("updated_at", i.UpdatedAt)
+	r.setSQLiteTimestamp("updated_at", i.UpdatedAt)
 }
 
 type ImageStore struct {

pkg/sqlite/migrations/33_postmigrate.go

@@ -1,116 +0,0 @@
-package migrations
-
-import (
-	"context"
-	"fmt"
-	"time"
-
-	"github.com/jmoiron/sqlx"
-	"github.com/stashapp/stash/pkg/logger"
-	"github.com/stashapp/stash/pkg/sqlite"
-)
-
-type schema33Migrator struct {
-	migrator
-}
-
-func post33(ctx context.Context, db *sqlx.DB) error {
-	logger.Info("Running post-migration for schema version 33")
-
-	m := schema33Migrator{
-		migrator: migrator{
-			db: db,
-		},
-	}
-
-	if err := m.migrateObjects(ctx, "scenes"); err != nil {
-		return fmt.Errorf("migrating scenes: %w", err)
-	}
-	if err := m.migrateObjects(ctx, "images"); err != nil {
-		return fmt.Errorf("migrating images: %w", err)
-	}
-	if err := m.migrateObjects(ctx, "galleries"); err != nil {
-		return fmt.Errorf("migrating galleries: %w", err)
-	}
-
-	return nil
-}
-
-func (m *schema33Migrator) migrateObjects(ctx context.Context, table string) error {
-	logger.Infof("Migrating %s table", table)
-
-	const (
-		limit    = 1000
-		logEvery = 10000
-	)
-
-	lastID := 0
-	count := 0
-
-	for {
-		gotSome := false
-
-		if err := m.withTxn(ctx, func(tx *sqlx.Tx) error {
-			query := fmt.Sprintf("SELECT `id`, `created_at`, `updated_at` FROM `%s` WHERE `created_at` like '%% %%' OR `updated_at` like '%% %%'", table)
-
-			if lastID != 0 {
-				query += fmt.Sprintf("AND `id` > %d ", lastID)
-			}
-
-			query += fmt.Sprintf("ORDER BY `id` LIMIT %d", limit)
-
-			rows, err := m.db.Query(query)
-			if err != nil {
-				return err
-			}
-			defer rows.Close()
-
-			for rows.Next() {
-				var (
-					id        int
-					createdAt time.Time
-					updatedAt time.Time
-				)
-
-				err := rows.Scan(&id, &createdAt, &updatedAt)
-				if err != nil {
-					return err
-				}
-
-				lastID = id
-				gotSome = true
-				count++
-
-				// convert incorrect timestamp string to correct one
-				// based on models.SQLTimestamp
-				fixedCreated := createdAt.Format(time.RFC3339)
-				fixedUpdated := updatedAt.Format(time.RFC3339)
-
-				updateSQL := fmt.Sprintf("UPDATE `%s` SET `created_at` = ?, `updated_at` = ? WHERE `id` = ?", table)
-
-				_, err = m.db.Exec(updateSQL, fixedCreated, fixedUpdated, id)
-				if err != nil {
-					return err
-				}
-			}
-
-			return rows.Err()
-		}); err != nil {
-			return err
-		}
-
-		if !gotSome {
-			break
-		}
-
-		if count%logEvery == 0 {
-			logger.Infof("Migrated %d rows", count, table)
-		}
-	}
-
-	return nil
-}
-
-func init() {
-	sqlite.RegisterPostMigration(33, post33)
-}

pkg/sqlite/migrations/34_indexes.up.sql

@@ -0,0 +1,3 @@
+CREATE INDEX `index_performer_stash_ids_on_performer_id` ON `performer_stash_ids` (`performer_id`);
+CREATE INDEX `index_scene_stash_ids_on_scene_id` ON `scene_stash_ids` (`scene_id`);
+CREATE INDEX `index_studio_stash_ids_on_studio_id` ON `studio_stash_ids` (`studio_id`);

pkg/sqlite/migrations/34_postmigrate.go

@@ -0,0 +1,154 @@
+package migrations
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/jmoiron/sqlx"
+	"github.com/stashapp/stash/pkg/logger"
+	"github.com/stashapp/stash/pkg/sqlite"
+)
+
+type schema34Migrator struct {
+	migrator
+}
+
+func post34(ctx context.Context, db *sqlx.DB) error {
+	logger.Info("Running post-migration for schema version 34")
+
+	m := schema34Migrator{
+		migrator: migrator{
+			db: db,
+		},
+	}
+
+	objectCols := []string{
+		"created_at",
+		"updated_at",
+	}
+
+	filesystemCols := objectCols
+	filesystemCols = append(filesystemCols, "mod_time")
+
+	if err := m.migrateObjects(ctx, "scenes", objectCols); err != nil {
+		return fmt.Errorf("migrating scenes: %w", err)
+	}
+	if err := m.migrateObjects(ctx, "images", objectCols); err != nil {
+		return fmt.Errorf("migrating images: %w", err)
+	}
+	if err := m.migrateObjects(ctx, "galleries", objectCols); err != nil {
+		return fmt.Errorf("migrating galleries: %w", err)
+	}
+	if err := m.migrateObjects(ctx, "files", filesystemCols); err != nil {
+		return fmt.Errorf("migrating files: %w", err)
+	}
+	if err := m.migrateObjects(ctx, "folders", filesystemCols); err != nil {
+		return fmt.Errorf("migrating folders: %w", err)
+	}
+
+	return nil
+}
+
+func (m *schema34Migrator) migrateObjects(ctx context.Context, table string, cols []string) error {
+	logger.Infof("Migrating %s table", table)
+
+	quotedCols := make([]string, len(cols)+1)
+	quotedCols[0] = "`id`"
+	whereClauses := make([]string, len(cols))
+	updateClauses := make([]string, len(cols))
+	for i, v := range cols {
+		quotedCols[i+1] = "`" + v + "`"
+		whereClauses[i] = "`" + v + "` like '% %'"
+		updateClauses[i] = "`" + v + "` = ?"
+	}
+
+	colList := strings.Join(quotedCols, ", ")
+	clauseList := strings.Join(whereClauses, " OR ")
+	updateList := strings.Join(updateClauses, ", ")
+
+	const (
+		limit    = 1000
+		logEvery = 10000
+	)
+
+	lastID := 0
+	count := 0
+
+	for {
+		gotSome := false
+
+		if err := m.withTxn(ctx, func(tx *sqlx.Tx) error {
+			query := fmt.Sprintf("SELECT %s FROM `%s` WHERE (%s)", colList, table, clauseList)
+
+			if lastID != 0 {
+				query += fmt.Sprintf(" AND `id` > %d ", lastID)
+			}
+
+			query += fmt.Sprintf(" ORDER BY `id` LIMIT %d", limit)
+
+			rows, err := m.db.Query(query)
+			if err != nil {
+				return err
+			}
+			defer rows.Close()
+
+			for rows.Next() {
+				var (
+					id int
+				)
+
+				timeValues := make([]interface{}, len(cols)+1)
+				timeValues[0] = &id
+				for i := range cols {
+					v := time.Time{}
+					timeValues[i+1] = &v
+				}
+
+				err := rows.Scan(timeValues...)
+				if err != nil {
+					return err
+				}
+
+				lastID = id
+				gotSome = true
+				count++
+
+				// convert incorrect timestamp string to correct one
+				// based on models.SQLTimestamp
+				args := make([]interface{}, len(cols)+1)
+				for i := range cols {
+					tv := timeValues[i+1].(*time.Time)
+					args[i] = tv.Format(time.RFC3339)
+				}
+				args[len(cols)] = id
+
+				updateSQL := fmt.Sprintf("UPDATE `%s` SET %s WHERE `id` = ?", table, updateList)
+
+				_, err = m.db.Exec(updateSQL, args...)
+				if err != nil {
+					return err
+				}
+			}
+
+			return rows.Err()
+		}); err != nil {
+			return err
+		}
+
+		if !gotSome {
+			break
+		}
+
+		if count%logEvery == 0 {
+			logger.Infof("Migrated %d rows", count)
+		}
+	}
+
+	return nil
+}
+
+func init() {
+	sqlite.RegisterPostMigration(34, post34)
+}

pkg/sqlite/record.go

@@ -83,12 +83,12 @@ func (r *updateRecord) setNullInt(destField string, v models.OptionalInt) {
 // 	}
 // }
 
-func (r *updateRecord) setTime(destField string, v models.OptionalTime) {
+func (r *updateRecord) setSQLiteTimestamp(destField string, v models.OptionalTime) {
 	if v.Set {
 		if v.Null {
 			panic("null value not allowed in optional time")
 		}
-		r.set(destField, v.Value)
+		r.set(destField, models.SQLiteTimestamp{Timestamp: v.Value})
 	}
 }
 

pkg/sqlite/scene.go

@@ -130,8 +130,8 @@ func (r *sceneRowRecord) fromPartial(o models.ScenePartial) {
 	r.setBool("organized", o.Organized)
 	r.setInt("o_counter", o.OCounter)
 	r.setNullInt("studio_id", o.StudioID)
-	r.setTime("created_at", o.CreatedAt)
+	r.setSQLiteTimestamp("created_at", o.CreatedAt)
-	r.setTime("updated_at", o.UpdatedAt)
+	r.setSQLiteTimestamp("updated_at", o.UpdatedAt)
 }
 
 type SceneStore struct {