[Files Refactor] Use batching for pre/post-migration (#2906)

* Use batching for pre/post-migration
* Clarify release notes

pkg/sqlite/migrations/32_postmigrate.go

@@ -54,39 +54,71 @@ type schema32Migrator struct {
 func (m *schema32Migrator) migrateFolders(ctx context.Context) error {
 	logger.Infof("Migrating folders")
 
-	const query = "SELECT `folders`.`id`, `folders`.`path` FROM `folders` INNER JOIN `galleries` ON `galleries`.`folder_id` = `folders`.`id`"
+	const (
+		limit    = 1000
+		logEvery = 10000
+	)
 
-	rows, err := m.db.Query(query)
+	lastID := 0
-	if err != nil {
+	count := 0
-		return err
-	}
-	defer rows.Close()
 
-	for rows.Next() {
+	for {
-		var id int
+		gotSome := false
-		var p string
 
-		err := rows.Scan(&id, &p)
+		if err := m.withTxn(ctx, func(tx *sqlx.Tx) error {
-		if err != nil {
+			query := "SELECT `folders`.`id`, `folders`.`path` FROM `folders` INNER JOIN `galleries` ON `galleries`.`folder_id` = `folders`.`id`"
+
+			if lastID != 0 {
+				query += fmt.Sprintf("AND `folders`.`id` > %d ", lastID)
+			}
+
+			query += fmt.Sprintf("ORDER BY `folders`.`id` LIMIT %d", limit)
+
+			rows, err := m.db.Query(query)
+			if err != nil {
+				return err
+			}
+			defer rows.Close()
+
+			for rows.Next() {
+				var id int
+				var p string
+
+				err := rows.Scan(&id, &p)
+				if err != nil {
+					return err
+				}
+
+				lastID = id
+				gotSome = true
+				count++
+
+				parent := filepath.Dir(p)
+				parentID, zipFileID, err := m.createFolderHierarchy(parent)
+				if err != nil {
+					return err
+				}
+
+				_, err = m.db.Exec("UPDATE `folders` SET `parent_folder_id` = ?, `zip_file_id` = ? WHERE `id` = ?", parentID, zipFileID, id)
+				if err != nil {
+					return err
+				}
+			}
+
+			return rows.Err()
+		}); err != nil {
 			return err
 		}
 
-		parent := filepath.Dir(p)
+		if !gotSome {
-		parentID, zipFileID, err := m.createFolderHierarchy(parent)
+			break
-		if err != nil {
-			return err
 		}
 
-		_, err = m.db.Exec("UPDATE `folders` SET `parent_folder_id` = ?, `zip_file_id` = ? WHERE `id` = ?", parentID, zipFileID, id)
+		if count%logEvery == 0 {
-		if err != nil {
+			logger.Infof("Migrated %d folders", count)
-			return err
 		}
 	}
 
-	if err := rows.Err(); err != nil {
-		return err
-	}
-
 	return nil
 }
 

pkg/sqlite/migrations/32_premigrate.go

@@ -2,6 +2,7 @@ package migrations
 
 import (
 	"context"
+	"fmt"
 	"os"
 
 	"github.com/jmoiron/sqlx"
@@ -30,6 +31,11 @@ type schema32PreMigrator struct {
 }
 
 func (m *schema32PreMigrator) migrate(ctx context.Context) error {
+	const (
+		limit    = 1000
+		logEvery = 10000
+	)
+
 	// query for galleries with zip = 0 and path not null
 	result := struct {
 		Count int `db:"count"`
@@ -45,48 +51,73 @@ func (m *schema32PreMigrator) migrate(ctx context.Context) error {
 
 	logger.Infof("Checking %d galleries for incorrect zip value...", result.Count)
 
-	if err := m.withTxn(ctx, func(tx *sqlx.Tx) error {
+	lastID := 0
-		const query = "SELECT `id`, `path` FROM `galleries` WHERE `zip` = '0' AND `path` IS NOT NULL ORDER BY `id`"
+	count := 0
-		rows, err := m.db.Query(query)
+
-		if err != nil {
+	for {
+		gotSome := false
+
+		if err := m.withTxn(ctx, func(tx *sqlx.Tx) error {
+			query := "SELECT `id`, `path` FROM `galleries` WHERE `zip` = '0' AND `path` IS NOT NULL "
+			if lastID != 0 {
+				query += fmt.Sprintf("AND `id` > %d ", lastID)
+			}
+
+			query += fmt.Sprintf("ORDER BY `id` LIMIT %d", limit)
+
+			rows, err := m.db.Query(query)
+			if err != nil {
+				return err
+			}
+			defer rows.Close()
+
+			for rows.Next() {
+				var id int
+				var p string
+
+				err := rows.Scan(&id, &p)
+				if err != nil {
+					return err
+				}
+
+				gotSome = true
+				lastID = id
+				count++
+
+				// if path does not exist, make no changes
+				// if it does exist and is a folder, then we ignore it
+				// otherwise set zip to 1
+				info, err := os.Stat(p)
+				if err != nil {
+					logger.Warnf("unable to verify if %q is a folder due to error %v. Assuming folder-based.", p, err)
+					continue
+				}
+
+				if info.IsDir() {
+					// ignore it
+					continue
+				}
+
+				logger.Infof("Correcting %q gallery to be zip-based.", p)
+
+				_, err = m.db.Exec("UPDATE `galleries` SET `zip` = '1' WHERE `id` = ?", id)
+				if err != nil {
+					return err
+				}
+			}
+
+			return rows.Err()
+		}); err != nil {
 			return err
 		}
-		defer rows.Close()
 
-		for rows.Next() {
+		if !gotSome {
-			var id int
+			break
-			var p string
-
-			err := rows.Scan(&id, &p)
-			if err != nil {
-				return err
-			}
-
-			// if path does not exist, assume that it is a file and not a folder
-			// if it does exist and is a folder, then we ignore it
-			// otherwise set zip to 1
-			info, err := os.Stat(p)
-			if err != nil {
-				logger.Warnf("unable to verify if %q is a folder due to error %v. Not migrating.", p, err)
-				continue
-			}
-
-			if info.IsDir() {
-				// ignore it
-				continue
-			}
-
-			logger.Infof("Correcting %q gallery to be zip-based.", p)
-
-			_, err = m.db.Exec("UPDATE `galleries` SET `zip` = '1' WHERE `id` = ?", id)
-			if err != nil {
-				return err
-			}
 		}
 
-		return rows.Err()
+		if count%logEvery == 0 {
-	}); err != nil {
+			logger.Infof("Checked %d galleries", count)
-		return err
+		}
 	}
 
 	return nil

ui/v2.5/src/docs/en/MigrationNotes/32.md

@@ -1,3 +1,5 @@
+**For best results, ensure that zip-based gallery paths are correct by performing a scan and clean of your library using v0.16.1 prior to running this migration.**
+
 This migration significantly changes the way that stash stores information about your files. This migration is not reversible.
 
 After migrating, please run a scan on your entire library to populate missing data, and to ingest identical files which were previously ignored.