mirror of
https://github.com/stashapp/stash.git
synced 2025-12-17 20:34:37 +03:00
More timestamp corrections (#2933)
* Fix incorrect timestamp updates * Correct folder time fields * Add migration with new indexes * Correct mod_time format * Add mod_time to data massage
This commit is contained in:
WithoutPants
@@ -1,116 +0,0 @@
|
||||
package migrations
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/jmoiron/sqlx"
|
||||
"github.com/stashapp/stash/pkg/logger"
|
||||
"github.com/stashapp/stash/pkg/sqlite"
|
||||
)
|
||||
|
||||
type schema33Migrator struct {
|
||||
migrator
|
||||
}
|
||||
|
||||
func post33(ctx context.Context, db *sqlx.DB) error {
|
||||
logger.Info("Running post-migration for schema version 33")
|
||||
|
||||
m := schema33Migrator{
|
||||
migrator: migrator{
|
||||
db: db,
|
||||
},
|
||||
}
|
||||
|
||||
if err := m.migrateObjects(ctx, "scenes"); err != nil {
|
||||
return fmt.Errorf("migrating scenes: %w", err)
|
||||
}
|
||||
if err := m.migrateObjects(ctx, "images"); err != nil {
|
||||
return fmt.Errorf("migrating images: %w", err)
|
||||
}
|
||||
if err := m.migrateObjects(ctx, "galleries"); err != nil {
|
||||
return fmt.Errorf("migrating galleries: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *schema33Migrator) migrateObjects(ctx context.Context, table string) error {
|
||||
logger.Infof("Migrating %s table", table)
|
||||
|
||||
const (
|
||||
limit = 1000
|
||||
logEvery = 10000
|
||||
)
|
||||
|
||||
lastID := 0
|
||||
count := 0
|
||||
|
||||
for {
|
||||
gotSome := false
|
||||
|
||||
if err := m.withTxn(ctx, func(tx *sqlx.Tx) error {
|
||||
query := fmt.Sprintf("SELECT `id`, `created_at`, `updated_at` FROM `%s` WHERE `created_at` like '%% %%' OR `updated_at` like '%% %%'", table)
|
||||
|
||||
if lastID != 0 {
|
||||
query += fmt.Sprintf("AND `id` > %d ", lastID)
|
||||
}
|
||||
|
||||
query += fmt.Sprintf("ORDER BY `id` LIMIT %d", limit)
|
||||
|
||||
rows, err := m.db.Query(query)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
for rows.Next() {
|
||||
var (
|
||||
id int
|
||||
createdAt time.Time
|
||||
updatedAt time.Time
|
||||
)
|
||||
|
||||
err := rows.Scan(&id, &createdAt, &updatedAt)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
lastID = id
|
||||
gotSome = true
|
||||
count++
|
||||
|
||||
// convert incorrect timestamp string to correct one
|
||||
// based on models.SQLTimestamp
|
||||
fixedCreated := createdAt.Format(time.RFC3339)
|
||||
fixedUpdated := updatedAt.Format(time.RFC3339)
|
||||
|
||||
updateSQL := fmt.Sprintf("UPDATE `%s` SET `created_at` = ?, `updated_at` = ? WHERE `id` = ?", table)
|
||||
|
||||
_, err = m.db.Exec(updateSQL, fixedCreated, fixedUpdated, id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return rows.Err()
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if !gotSome {
|
||||
break
|
||||
}
|
||||
|
||||
if count%logEvery == 0 {
|
||||
logger.Infof("Migrated %d rows", count, table)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func init() {
|
||||
sqlite.RegisterPostMigration(33, post33)
|
||||
}
|
||||
3
pkg/sqlite/migrations/34_indexes.up.sql
Normal file
3
pkg/sqlite/migrations/34_indexes.up.sql
Normal file
@@ -0,0 +1,3 @@
|
||||
CREATE INDEX `index_performer_stash_ids_on_performer_id` ON `performer_stash_ids` (`performer_id`);
|
||||
CREATE INDEX `index_scene_stash_ids_on_scene_id` ON `scene_stash_ids` (`scene_id`);
|
||||
CREATE INDEX `index_studio_stash_ids_on_studio_id` ON `studio_stash_ids` (`studio_id`);
|
||||
154
pkg/sqlite/migrations/34_postmigrate.go
Normal file
154
pkg/sqlite/migrations/34_postmigrate.go
Normal file
@@ -0,0 +1,154 @@
|
||||
package migrations
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/jmoiron/sqlx"
|
||||
"github.com/stashapp/stash/pkg/logger"
|
||||
"github.com/stashapp/stash/pkg/sqlite"
|
||||
)
|
||||
|
||||
type schema34Migrator struct {
|
||||
migrator
|
||||
}
|
||||
|
||||
func post34(ctx context.Context, db *sqlx.DB) error {
|
||||
logger.Info("Running post-migration for schema version 34")
|
||||
|
||||
m := schema34Migrator{
|
||||
migrator: migrator{
|
||||
db: db,
|
||||
},
|
||||
}
|
||||
|
||||
objectCols := []string{
|
||||
"created_at",
|
||||
"updated_at",
|
||||
}
|
||||
|
||||
filesystemCols := objectCols
|
||||
filesystemCols = append(filesystemCols, "mod_time")
|
||||
|
||||
if err := m.migrateObjects(ctx, "scenes", objectCols); err != nil {
|
||||
return fmt.Errorf("migrating scenes: %w", err)
|
||||
}
|
||||
if err := m.migrateObjects(ctx, "images", objectCols); err != nil {
|
||||
return fmt.Errorf("migrating images: %w", err)
|
||||
}
|
||||
if err := m.migrateObjects(ctx, "galleries", objectCols); err != nil {
|
||||
return fmt.Errorf("migrating galleries: %w", err)
|
||||
}
|
||||
if err := m.migrateObjects(ctx, "files", filesystemCols); err != nil {
|
||||
return fmt.Errorf("migrating files: %w", err)
|
||||
}
|
||||
if err := m.migrateObjects(ctx, "folders", filesystemCols); err != nil {
|
||||
return fmt.Errorf("migrating folders: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *schema34Migrator) migrateObjects(ctx context.Context, table string, cols []string) error {
|
||||
logger.Infof("Migrating %s table", table)
|
||||
|
||||
quotedCols := make([]string, len(cols)+1)
|
||||
quotedCols[0] = "`id`"
|
||||
whereClauses := make([]string, len(cols))
|
||||
updateClauses := make([]string, len(cols))
|
||||
for i, v := range cols {
|
||||
quotedCols[i+1] = "`" + v + "`"
|
||||
whereClauses[i] = "`" + v + "` like '% %'"
|
||||
updateClauses[i] = "`" + v + "` = ?"
|
||||
}
|
||||
|
||||
colList := strings.Join(quotedCols, ", ")
|
||||
clauseList := strings.Join(whereClauses, " OR ")
|
||||
updateList := strings.Join(updateClauses, ", ")
|
||||
|
||||
const (
|
||||
limit = 1000
|
||||
logEvery = 10000
|
||||
)
|
||||
|
||||
lastID := 0
|
||||
count := 0
|
||||
|
||||
for {
|
||||
gotSome := false
|
||||
|
||||
if err := m.withTxn(ctx, func(tx *sqlx.Tx) error {
|
||||
query := fmt.Sprintf("SELECT %s FROM `%s` WHERE (%s)", colList, table, clauseList)
|
||||
|
||||
if lastID != 0 {
|
||||
query += fmt.Sprintf(" AND `id` > %d ", lastID)
|
||||
}
|
||||
|
||||
query += fmt.Sprintf(" ORDER BY `id` LIMIT %d", limit)
|
||||
|
||||
rows, err := m.db.Query(query)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
for rows.Next() {
|
||||
var (
|
||||
id int
|
||||
)
|
||||
|
||||
timeValues := make([]interface{}, len(cols)+1)
|
||||
timeValues[0] = &id
|
||||
for i := range cols {
|
||||
v := time.Time{}
|
||||
timeValues[i+1] = &v
|
||||
}
|
||||
|
||||
err := rows.Scan(timeValues...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
lastID = id
|
||||
gotSome = true
|
||||
count++
|
||||
|
||||
// convert incorrect timestamp string to correct one
|
||||
// based on models.SQLTimestamp
|
||||
args := make([]interface{}, len(cols)+1)
|
||||
for i := range cols {
|
||||
tv := timeValues[i+1].(*time.Time)
|
||||
args[i] = tv.Format(time.RFC3339)
|
||||
}
|
||||
args[len(cols)] = id
|
||||
|
||||
updateSQL := fmt.Sprintf("UPDATE `%s` SET %s WHERE `id` = ?", table, updateList)
|
||||
|
||||
_, err = m.db.Exec(updateSQL, args...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return rows.Err()
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if !gotSome {
|
||||
break
|
||||
}
|
||||
|
||||
if count%logEvery == 0 {
|
||||
logger.Infof("Migrated %d rows", count)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func init() {
|
||||
sqlite.RegisterPostMigration(34, post34)
|
||||
}
|
||||
Reference in New Issue
Block a user