[Files Refactor] Use batching for pre/post-migration (#2906)

* Use batching for pre/post-migration
* Clarify release notes

pkg/sqlite/migrations/32_postmigrate.go

@@ -54,7 +54,25 @@ type schema32Migrator struct {
 func (m *schema32Migrator) migrateFolders(ctx context.Context) error {
 	logger.Infof("Migrating folders")
 
-	const query = "SELECT `folders`.`id`, `folders`.`path` FROM `folders` INNER JOIN `galleries` ON `galleries`.`folder_id` = `folders`.`id`"
+	const (
+		limit    = 1000
+		logEvery = 10000
+	)
+
+	lastID := 0
+	count := 0
+
+	for {
+		gotSome := false
+
+		if err := m.withTxn(ctx, func(tx *sqlx.Tx) error {
+			query := "SELECT `folders`.`id`, `folders`.`path` FROM `folders` INNER JOIN `galleries` ON `galleries`.`folder_id` = `folders`.`id`"
+
+			if lastID != 0 {
+				query += fmt.Sprintf("AND `folders`.`id` > %d ", lastID)
+			}
+
+			query += fmt.Sprintf("ORDER BY `folders`.`id` LIMIT %d", limit)
 
 			rows, err := m.db.Query(query)
 			if err != nil {
@@ -71,6 +89,10 @@ func (m *schema32Migrator) migrateFolders(ctx context.Context) error {
 					return err
 				}
 
+				lastID = id
+				gotSome = true
+				count++
+
 				parent := filepath.Dir(p)
 				parentID, zipFileID, err := m.createFolderHierarchy(parent)
 				if err != nil {
@@ -83,10 +105,20 @@ func (m *schema32Migrator) migrateFolders(ctx context.Context) error {
 				}
 			}
 
-	if err := rows.Err(); err != nil {
+			return rows.Err()
+		}); err != nil {
 			return err
 		}
 
+		if !gotSome {
+			break
+		}
+
+		if count%logEvery == 0 {
+			logger.Infof("Migrated %d folders", count)
+		}
+	}
+
 	return nil
 }
 

pkg/sqlite/migrations/32_premigrate.go

@@ -2,6 +2,7 @@ package migrations
 
 import (
 	"context"
+	"fmt"
 	"os"
 
 	"github.com/jmoiron/sqlx"
@@ -30,6 +31,11 @@ type schema32PreMigrator struct {
 }
 
 func (m *schema32PreMigrator) migrate(ctx context.Context) error {
+	const (
+		limit    = 1000
+		logEvery = 10000
+	)
+
 	// query for galleries with zip = 0 and path not null
 	result := struct {
 		Count int `db:"count"`
@@ -45,8 +51,20 @@ func (m *schema32PreMigrator) migrate(ctx context.Context) error {
 
 	logger.Infof("Checking %d galleries for incorrect zip value...", result.Count)
 
+	lastID := 0
+	count := 0
+
+	for {
+		gotSome := false
+
 		if err := m.withTxn(ctx, func(tx *sqlx.Tx) error {
-		const query = "SELECT `id`, `path` FROM `galleries` WHERE `zip` = '0' AND `path` IS NOT NULL ORDER BY `id`"
+			query := "SELECT `id`, `path` FROM `galleries` WHERE `zip` = '0' AND `path` IS NOT NULL "
+			if lastID != 0 {
+				query += fmt.Sprintf("AND `id` > %d ", lastID)
+			}
+
+			query += fmt.Sprintf("ORDER BY `id` LIMIT %d", limit)
+
 			rows, err := m.db.Query(query)
 			if err != nil {
 				return err
@@ -62,12 +80,16 @@ func (m *schema32PreMigrator) migrate(ctx context.Context) error {
 					return err
 				}
 
-			// if path does not exist, assume that it is a file and not a folder
+				gotSome = true
+				lastID = id
+				count++
+
+				// if path does not exist, make no changes
 				// if it does exist and is a folder, then we ignore it
 				// otherwise set zip to 1
 				info, err := os.Stat(p)
 				if err != nil {
-				logger.Warnf("unable to verify if %q is a folder due to error %v. Not migrating.", p, err)
+					logger.Warnf("unable to verify if %q is a folder due to error %v. Assuming folder-based.", p, err)
 					continue
 				}
 
@@ -89,6 +111,15 @@ func (m *schema32PreMigrator) migrate(ctx context.Context) error {
 			return err
 		}
 
+		if !gotSome {
+			break
+		}
+
+		if count%logEvery == 0 {
+			logger.Infof("Checked %d galleries", count)
+		}
+	}
+
 	return nil
 }
 

ui/v2.5/src/docs/en/MigrationNotes/32.md

@@ -1,3 +1,5 @@
+**For best results, ensure that zip-based gallery paths are correct by performing a scan and clean of your library using v0.16.1 prior to running this migration.**
+
 This migration significantly changes the way that stash stores information about your files. This migration is not reversible.
 
 After migrating, please run a scan on your entire library to populate missing data, and to ingest identical files which were previously ignored.