Mirrors/stash
1
0
Fork 0
mirror of https://github.com/stashapp/stash.git synced 2025-12-17 04:14:39 +03:00
Code Issues Packages Projects Releases Wiki Activity

More timestamp corrections (#2933)

Browse Source 
* Fix incorrect timestamp updates
* Correct folder time fields
* Add migration with new indexes
* Correct mod_time format
* Add mod_time to data massage
This commit is contained in:
WithoutPants
2022-09-20 13:52:37 +10:00
committed by GitHub
parent 98e3610ade
commit 3fa7b470e7
12 changed files with 205 additions and 165 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
pkg/sqlite/database.go
View File

@@ -21,7 +21,7 @@ import (
"github.com/stashapp/stash/pkg/logger"
)
var appSchemaVersion uint = 33
var appSchemaVersion uint = 34
//go:embed migrations/*.sql
var migrationsBox embed.FS

25
pkg/sqlite/file.go
View File

@@ -7,7 +7,6 @@ import (
"fmt"
"path/filepath"
"strings"
"time"
"github.com/doug-martin/goqu/v9"
"github.com/doug-martin/goqu/v9/exp"
@@ -35,9 +34,9 @@ type basicFileRow struct {
ZipFileID null.Int `db:"zip_file_id"`
ParentFolderID file.FolderID `db:"parent_folder_id"`
Size int64 `db:"size"`
ModTime time.Time `db:"mod_time"`
CreatedAt time.Time `db:"created_at"`
UpdatedAt time.Time `db:"updated_at"`
ModTime models.SQLiteTimestamp `db:"mod_time"`
CreatedAt models.SQLiteTimestamp `db:"created_at"`
UpdatedAt models.SQLiteTimestamp `db:"updated_at"`
}
func (r *basicFileRow) fromBasicFile(o file.BaseFile) {
@@ -46,9 +45,9 @@ func (r *basicFileRow) fromBasicFile(o file.BaseFile) {
r.ZipFileID = nullIntFromFileIDPtr(o.ZipFileID)
r.ParentFolderID = o.ParentFolderID
r.Size = o.Size
r.ModTime = o.ModTime
r.CreatedAt = o.CreatedAt
r.UpdatedAt = o.UpdatedAt
r.ModTime = models.SQLiteTimestamp{Timestamp: o.ModTime}
r.CreatedAt = models.SQLiteTimestamp{Timestamp: o.CreatedAt}
r.UpdatedAt = models.SQLiteTimestamp{Timestamp: o.UpdatedAt}
}
type videoFileRow struct {
@@ -172,9 +171,9 @@ type fileQueryRow struct {
ZipFileID null.Int `db:"zip_file_id"`
ParentFolderID null.Int `db:"parent_folder_id"`
Size null.Int `db:"size"`
ModTime null.Time `db:"mod_time"`
CreatedAt null.Time `db:"file_created_at"`
UpdatedAt null.Time `db:"file_updated_at"`
ModTime models.NullSQLiteTimestamp `db:"mod_time"`
CreatedAt models.NullSQLiteTimestamp `db:"file_created_at"`
UpdatedAt models.NullSQLiteTimestamp `db:"file_updated_at"`
ZipBasename null.String `db:"zip_basename"`
ZipFolderPath null.String `db:"zip_folder_path"`
@@ -190,14 +189,14 @@ func (r *fileQueryRow) resolve() file.File {
ID: file.ID(r.FileID.Int64),
DirEntry: file.DirEntry{
ZipFileID: nullIntFileIDPtr(r.ZipFileID),
ModTime: r.ModTime.Time,
ModTime: r.ModTime.Timestamp,
},
Path: filepath.Join(r.FolderPath.String, r.Basename.String),
ParentFolderID: file.FolderID(r.ParentFolderID.Int64),
Basename: r.Basename.String,
Size: r.Size.Int64,
CreatedAt: r.CreatedAt.Time,
UpdatedAt: r.UpdatedAt.Time,
CreatedAt: r.CreatedAt.Timestamp,
UpdatedAt: r.UpdatedAt.Timestamp,
}
if basic.ZipFileID != nil && r.ZipFolderPath.Valid && r.ZipBasename.Valid {

20
pkg/sqlite/folder.go
View File

@@ -6,12 +6,12 @@ import (
"errors"
"fmt"
"path/filepath"
"time"
"github.com/doug-martin/goqu/v9"
"github.com/doug-martin/goqu/v9/exp"
"github.com/jmoiron/sqlx"
"github.com/stashapp/stash/pkg/file"
"github.com/stashapp/stash/pkg/models"
"gopkg.in/guregu/null.v4"
)
@@ -22,9 +22,9 @@ type folderRow struct {
Path string `db:"path"`
ZipFileID null.Int `db:"zip_file_id"`
ParentFolderID null.Int `db:"parent_folder_id"`
ModTime time.Time `db:"mod_time"`
CreatedAt time.Time `db:"created_at"`
UpdatedAt time.Time `db:"updated_at"`
ModTime models.SQLiteTimestamp `db:"mod_time"`
CreatedAt models.SQLiteTimestamp `db:"created_at"`
UpdatedAt models.SQLiteTimestamp `db:"updated_at"`
}
func (r *folderRow) fromFolder(o file.Folder) {
@@ -32,9 +32,9 @@ func (r *folderRow) fromFolder(o file.Folder) {
r.Path = o.Path
r.ZipFileID = nullIntFromFileIDPtr(o.ZipFileID)
r.ParentFolderID = nullIntFromFolderIDPtr(o.ParentFolderID)
r.ModTime = o.ModTime
r.CreatedAt = o.CreatedAt
r.UpdatedAt = o.UpdatedAt
r.ModTime = models.SQLiteTimestamp{Timestamp: o.ModTime}
r.CreatedAt = models.SQLiteTimestamp{Timestamp: o.CreatedAt}
r.UpdatedAt = models.SQLiteTimestamp{Timestamp: o.UpdatedAt}
}
type folderQueryRow struct {
@@ -49,12 +49,12 @@ func (r *folderQueryRow) resolve() *file.Folder {
ID: r.ID,
DirEntry: file.DirEntry{
ZipFileID: nullIntFileIDPtr(r.ZipFileID),
ModTime: r.ModTime,
ModTime: r.ModTime.Timestamp,
},
Path: string(r.Path),
ParentFolderID: nullIntFolderIDPtr(r.ParentFolderID),
CreatedAt: r.CreatedAt,
UpdatedAt: r.UpdatedAt,
CreatedAt: r.CreatedAt.Timestamp,
UpdatedAt: r.UpdatedAt.Timestamp,
}
if ret.ZipFileID != nil && r.ZipFolderPath.Valid && r.ZipBasename.Valid {

6
pkg/sqlite/folder_test.go
View File

@@ -102,9 +102,9 @@ func Test_FolderStore_Create(t *testing.T) {
func Test_FolderStore_Update(t *testing.T) {
var (
path = "path"
fileModTime = time.Date(2000, 1, 2, 3, 4, 5, 6, time.UTC)
createdAt = time.Date(2001, 1, 2, 3, 4, 5, 6, time.UTC)
updatedAt = time.Date(2002, 1, 2, 3, 4, 5, 6, time.UTC)
fileModTime = time.Date(2000, 1, 2, 3, 4, 5, 0, time.UTC)
createdAt = time.Date(2001, 1, 2, 3, 4, 5, 0, time.UTC)
updatedAt = time.Date(2002, 1, 2, 3, 4, 5, 0, time.UTC)
)
tests := []struct {

4
pkg/sqlite/gallery.go
View File

@@ -105,8 +105,8 @@ func (r *galleryRowRecord) fromPartial(o models.GalleryPartial) {
r.setNullInt("rating", o.Rating)
r.setBool("organized", o.Organized)
r.setNullInt("studio_id", o.StudioID)
r.setTime("created_at", o.CreatedAt)
r.setTime("updated_at", o.UpdatedAt)
r.setSQLiteTimestamp("created_at", o.CreatedAt)
r.setSQLiteTimestamp("updated_at", o.UpdatedAt)
}
type GalleryStore struct {

4
pkg/sqlite/image.go
View File

@@ -89,8 +89,8 @@ func (r *imageRowRecord) fromPartial(i models.ImagePartial) {
r.setBool("organized", i.Organized)
r.setInt("o_counter", i.OCounter)
r.setNullInt("studio_id", i.StudioID)
r.setTime("created_at", i.CreatedAt)
r.setTime("updated_at", i.UpdatedAt)
r.setSQLiteTimestamp("created_at", i.CreatedAt)
r.setSQLiteTimestamp("updated_at", i.UpdatedAt)
}
type ImageStore struct {

0
pkg/sqlite/migrations/33_time_fix.up.sql → pkg/sqlite/migrations/33_noop.up.sql
View File

116
pkg/sqlite/migrations/33_postmigrate.go
View File

@@ -1,116 +0,0 @@
package migrations
import (
"context"
"fmt"
"time"
"github.com/jmoiron/sqlx"
"github.com/stashapp/stash/pkg/logger"
"github.com/stashapp/stash/pkg/sqlite"
)
type schema33Migrator struct {
migrator
}
func post33(ctx context.Context, db *sqlx.DB) error {
logger.Info("Running post-migration for schema version 33")
m := schema33Migrator{
migrator: migrator{
db: db,
},
}
if err := m.migrateObjects(ctx, "scenes"); err != nil {
return fmt.Errorf("migrating scenes: %w", err)
}
if err := m.migrateObjects(ctx, "images"); err != nil {
return fmt.Errorf("migrating images: %w", err)
}
if err := m.migrateObjects(ctx, "galleries"); err != nil {
return fmt.Errorf("migrating galleries: %w", err)
}
return nil
}
func (m *schema33Migrator) migrateObjects(ctx context.Context, table string) error {
logger.Infof("Migrating %s table", table)
const (
limit = 1000
logEvery = 10000
)
lastID := 0
count := 0
for {
gotSome := false
if err := m.withTxn(ctx, func(tx *sqlx.Tx) error {
query := fmt.Sprintf("SELECT `id`, `created_at`, `updated_at` FROM `%s` WHERE `created_at` like '%% %%' OR `updated_at` like '%% %%'", table)
if lastID != 0 {
query += fmt.Sprintf("AND `id` > %d ", lastID)
}
query += fmt.Sprintf("ORDER BY `id` LIMIT %d", limit)
rows, err := m.db.Query(query)
if err != nil {
return err
}
defer rows.Close()
for rows.Next() {
var (
id int
createdAt time.Time
updatedAt time.Time
)
err := rows.Scan(&id, &createdAt, &updatedAt)
if err != nil {
return err
}
lastID = id
gotSome = true
count++
// convert incorrect timestamp string to correct one
// based on models.SQLTimestamp
fixedCreated := createdAt.Format(time.RFC3339)
fixedUpdated := updatedAt.Format(time.RFC3339)
updateSQL := fmt.Sprintf("UPDATE `%s` SET `created_at` = ?, `updated_at` = ? WHERE `id` = ?", table)
_, err = m.db.Exec(updateSQL, fixedCreated, fixedUpdated, id)
if err != nil {
return err
}
}
return rows.Err()
}); err != nil {
return err
}
if !gotSome {
break
}
if count%logEvery == 0 {
logger.Infof("Migrated %d rows", count, table)
}
}
return nil
}
func init() {
sqlite.RegisterPostMigration(33, post33)
}

3
pkg/sqlite/migrations/34_indexes.up.sql Normal file
View File

@@ -0,0 +1,3 @@
CREATE INDEX `index_performer_stash_ids_on_performer_id` ON `performer_stash_ids` (`performer_id`);
CREATE INDEX `index_scene_stash_ids_on_scene_id` ON `scene_stash_ids` (`scene_id`);
CREATE INDEX `index_studio_stash_ids_on_studio_id` ON `studio_stash_ids` (`studio_id`);

154
pkg/sqlite/migrations/34_postmigrate.go Normal file
View File

@@ -0,0 +1,154 @@
package migrations
import (
"context"
"fmt"
"strings"
"time"
"github.com/jmoiron/sqlx"
"github.com/stashapp/stash/pkg/logger"
"github.com/stashapp/stash/pkg/sqlite"
)
type schema34Migrator struct {
migrator
}
func post34(ctx context.Context, db *sqlx.DB) error {
logger.Info("Running post-migration for schema version 34")
m := schema34Migrator{
migrator: migrator{
db: db,
},
}
objectCols := []string{
"created_at",
"updated_at",
}
filesystemCols := objectCols
filesystemCols = append(filesystemCols, "mod_time")
if err := m.migrateObjects(ctx, "scenes", objectCols); err != nil {
return fmt.Errorf("migrating scenes: %w", err)
}
if err := m.migrateObjects(ctx, "images", objectCols); err != nil {
return fmt.Errorf("migrating images: %w", err)
}
if err := m.migrateObjects(ctx, "galleries", objectCols); err != nil {
return fmt.Errorf("migrating galleries: %w", err)
}
if err := m.migrateObjects(ctx, "files", filesystemCols); err != nil {
return fmt.Errorf("migrating files: %w", err)
}
if err := m.migrateObjects(ctx, "folders", filesystemCols); err != nil {
return fmt.Errorf("migrating folders: %w", err)
}
return nil
}
func (m *schema34Migrator) migrateObjects(ctx context.Context, table string, cols []string) error {
logger.Infof("Migrating %s table", table)
quotedCols := make([]string, len(cols)+1)
quotedCols[0] = "`id`"
whereClauses := make([]string, len(cols))
updateClauses := make([]string, len(cols))
for i, v := range cols {
quotedCols[i+1] = "`" + v + "`"
whereClauses[i] = "`" + v + "` like '% %'"
updateClauses[i] = "`" + v + "` = ?"
}
colList := strings.Join(quotedCols, ", ")
clauseList := strings.Join(whereClauses, " OR ")
updateList := strings.Join(updateClauses, ", ")
const (
limit = 1000
logEvery = 10000
)
lastID := 0
count := 0
for {
gotSome := false
if err := m.withTxn(ctx, func(tx *sqlx.Tx) error {
query := fmt.Sprintf("SELECT %s FROM `%s` WHERE (%s)", colList, table, clauseList)
if lastID != 0 {
query += fmt.Sprintf(" AND `id` > %d ", lastID)
}
query += fmt.Sprintf(" ORDER BY `id` LIMIT %d", limit)
rows, err := m.db.Query(query)
if err != nil {
return err
}
defer rows.Close()
for rows.Next() {
var (
id int
)
timeValues := make([]interface{}, len(cols)+1)
timeValues[0] = &id
for i := range cols {
v := time.Time{}
timeValues[i+1] = &v
}
err := rows.Scan(timeValues...)
if err != nil {
return err
}
lastID = id
gotSome = true
count++
// convert incorrect timestamp string to correct one
// based on models.SQLTimestamp
args := make([]interface{}, len(cols)+1)
for i := range cols {
tv := timeValues[i+1].(*time.Time)
args[i] = tv.Format(time.RFC3339)
}
args[len(cols)] = id
updateSQL := fmt.Sprintf("UPDATE `%s` SET %s WHERE `id` = ?", table, updateList)
_, err = m.db.Exec(updateSQL, args...)
if err != nil {
return err
}
}
return rows.Err()
}); err != nil {
return err
}
if !gotSome {
break
}
if count%logEvery == 0 {
logger.Infof("Migrated %d rows", count)
}
}
return nil
}
func init() {
sqlite.RegisterPostMigration(34, post34)
}

4
pkg/sqlite/record.go
View File

@@ -83,12 +83,12 @@ func (r *updateRecord) setNullInt(destField string, v models.OptionalInt) {
// }
// }
func (r *updateRecord) setTime(destField string, v models.OptionalTime) {
func (r *updateRecord) setSQLiteTimestamp(destField string, v models.OptionalTime) {
if v.Set {
if v.Null {
panic("null value not allowed in optional time")
}
r.set(destField, v.Value)
r.set(destField, models.SQLiteTimestamp{Timestamp: v.Value})
}
}

4
pkg/sqlite/scene.go
View File

@@ -130,8 +130,8 @@ func (r *sceneRowRecord) fromPartial(o models.ScenePartial) {
r.setBool("organized", o.Organized)
r.setInt("o_counter", o.OCounter)
r.setNullInt("studio_id", o.StudioID)
r.setTime("created_at", o.CreatedAt)
r.setTime("updated_at", o.UpdatedAt)
r.setSQLiteTimestamp("created_at", o.CreatedAt)
r.setSQLiteTimestamp("updated_at", o.UpdatedAt)
}
type SceneStore struct {