Mirrors/stash
1
0
Fork 0
mirror of https://github.com/stashapp/stash.git synced 2025-12-16 20:07:05 +03:00
Code Issues Packages Projects Releases Wiki Activity

Make migration an asynchronous task (#4666)

Browse Source 
* Add failed state and error to Job
* Move migration code
* Add websocket monitor
* Make migrate a job managed task
This commit is contained in:
WithoutPants
2024-03-14 11:06:23 +11:00
committed by GitHub
parent fa172c2dfd
commit e5929389b4
36 changed files with 693 additions and 304 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
pkg/job/job.go
View File

@@ -5,22 +5,24 @@ import (
"time"
)
type JobExecFn func(ctx context.Context, progress *Progress) error
// JobExec represents the implementation of a Job to be executed.
type JobExec interface {
Execute(ctx context.Context, progress *Progress)
Execute(ctx context.Context, progress *Progress) error
}
type jobExecImpl struct {
fn func(ctx context.Context, progress *Progress)
fn JobExecFn
}
func (j *jobExecImpl) Execute(ctx context.Context, progress *Progress) {
j.fn(ctx, progress)
func (j *jobExecImpl) Execute(ctx context.Context, progress *Progress) error {
return j.fn(ctx, progress)
}
// MakeJobExec returns a simple JobExec implementation using the provided
// function.
func MakeJobExec(fn func(ctx context.Context, progress *Progress)) JobExec {
func MakeJobExec(fn JobExecFn) JobExec {
return &jobExecImpl{
fn: fn,
}
@@ -56,6 +58,7 @@ type Job struct {
StartTime *time.Time
EndTime *time.Time
AddTime time.Time
Error *string
outerCtx context.Context
exec JobExec
@@ -87,6 +90,12 @@ func (j *Job) cancel() {
}
}
func (j *Job) error(err error) {
errStr := err.Error()
j.Error = &errStr
j.Status = StatusFailed
}
// IsCancelled returns true if cancel has been called on the context.
func IsCancelled(ctx context.Context) bool {
select {

5
pkg/job/manager.go
View File

@@ -206,7 +206,10 @@ func (m *Manager) executeJob(ctx context.Context, j *Job, done chan struct{}) {
}()
progress := m.newProgress(j)
j.exec.Execute(ctx, progress)
if err := j.exec.Execute(ctx, progress); err != nil {
logger.Errorf("task failed due to error: %v", err)
j.error(err)
}
}
func (m *Manager) onJobFinish(job *Job) {

4
pkg/job/manager_test.go
View File

@@ -24,7 +24,7 @@ func newTestExec(finish chan struct{}) *testExec {
}
}
func (e *testExec) Execute(ctx context.Context, p *Progress) {
func (e *testExec) Execute(ctx context.Context, p *Progress) error {
e.progress = p
close(e.started)
@@ -38,6 +38,8 @@ func (e *testExec) Execute(ctx context.Context, p *Progress) {
// fall through
}
}
return nil
}
func TestAdd(t *testing.T) {

129
pkg/sqlite/database.go
View File

@@ -10,9 +10,6 @@ import (
"path/filepath"
"time"
"github.com/golang-migrate/migrate/v4"
sqlite3mig "github.com/golang-migrate/migrate/v4/database/sqlite3"
"github.com/golang-migrate/migrate/v4/source/iofs"
"github.com/jmoiron/sqlx"
"github.com/stashapp/stash/pkg/fsutil"
@@ -144,7 +141,7 @@ func (db *Database) Open(dbPath string) error {
if databaseSchemaVersion == 0 {
// new database, just run the migrations
if err := db.RunMigrations(); err != nil {
if err := db.RunAllMigrations(); err != nil {
return fmt.Errorf("error running initial schema migrations: %w", err)
}
} else {
@@ -312,11 +309,6 @@ func (db *Database) RestoreFromBackup(backupPath string) error {
return os.Rename(backupPath, db.dbPath)
}
// Migrate the database
func (db *Database) needsMigration() bool {
return db.schemaVersion != appSchemaVersion
}
func (db *Database) AppSchemaVersion() uint {
return appSchemaVersion
}
@@ -349,100 +341,6 @@ func (db *Database) Version() uint {
return db.schemaVersion
}
func (db *Database) getMigrate() (*migrate.Migrate, error) {
migrations, err := iofs.New(migrationsBox, "migrations")
if err != nil {
return nil, err
}
const disableForeignKeys = true
conn, err := db.open(disableForeignKeys)
if err != nil {
return nil, err
}
driver, err := sqlite3mig.WithInstance(conn.DB, &sqlite3mig.Config{})
if err != nil {
return nil, err
}
// use sqlite3Driver so that migration has access to durationToTinyInt
return migrate.NewWithInstance(
"iofs",
migrations,
db.dbPath,
driver,
)
}
func (db *Database) getDatabaseSchemaVersion() (uint, error) {
m, err := db.getMigrate()
if err != nil {
return 0, err
}
defer m.Close()
ret, _, _ := m.Version()
return ret, nil
}
// Migrate the database
func (db *Database) RunMigrations() error {
ctx := context.Background()
m, err := db.getMigrate()
if err != nil {
return err
}
defer m.Close()
databaseSchemaVersion, _, _ := m.Version()
stepNumber := appSchemaVersion - databaseSchemaVersion
if stepNumber != 0 {
logger.Infof("Migrating database from version %d to %d", databaseSchemaVersion, appSchemaVersion)
// run each migration individually, and run custom migrations as needed
var i uint = 1
for ; i <= stepNumber; i++ {
newVersion := databaseSchemaVersion + i
// run pre migrations as needed
if err := db.runCustomMigrations(ctx, preMigrations[newVersion]); err != nil {
return fmt.Errorf("running pre migrations for schema version %d: %w", newVersion, err)
}
err = m.Steps(1)
if err != nil {
// migration failed
return err
}
// run post migrations as needed
if err := db.runCustomMigrations(ctx, postMigrations[newVersion]); err != nil {
return fmt.Errorf("running post migrations for schema version %d: %w", newVersion, err)
}
}
}
// update the schema version
db.schemaVersion, _, _ = m.Version()
// re-initialise the database
const disableForeignKeys = false
db.db, err = db.open(disableForeignKeys)
if err != nil {
return fmt.Errorf("re-initializing the database: %w", err)
}
// optimize database after migration
err = db.Optimise(ctx)
if err != nil {
logger.Warnf("error while performing post-migration optimisation: %v", err)
}
return nil
}
func (db *Database) Optimise(ctx context.Context) error {
logger.Info("Optimising database")
@@ -524,28 +422,3 @@ func (db *Database) QuerySQL(ctx context.Context, query string, args []interface
return cols, ret, nil
}
func (db *Database) runCustomMigrations(ctx context.Context, fns []customMigrationFunc) error {
for _, fn := range fns {
if err := db.runCustomMigration(ctx, fn); err != nil {
return err
}
}
return nil
}
func (db *Database) runCustomMigration(ctx context.Context, fn customMigrationFunc) error {
const disableForeignKeys = false
d, err := db.open(disableForeignKeys)
if err != nil {
return err
}
defer d.Close()
if err := fn(ctx, d); err != nil {
return err
}
return nil
}

188
pkg/sqlite/migrate.go Normal file
View File

@@ -0,0 +1,188 @@
package sqlite
import (
"context"
"fmt"
"github.com/golang-migrate/migrate/v4"
sqlite3mig "github.com/golang-migrate/migrate/v4/database/sqlite3"
"github.com/golang-migrate/migrate/v4/source/iofs"
"github.com/stashapp/stash/pkg/logger"
)
func (db *Database) needsMigration() bool {
return db.schemaVersion != appSchemaVersion
}
type Migrator struct {
db *Database
m *migrate.Migrate
}
func NewMigrator(db *Database) (*Migrator, error) {
m := &Migrator{
db: db,
}
var err error
m.m, err = m.getMigrate()
return m, err
}
func (m *Migrator) Close() {
if m.m != nil {
m.m.Close()
m.m = nil
}
}
func (m *Migrator) CurrentSchemaVersion() uint {
databaseSchemaVersion, _, _ := m.m.Version()
return databaseSchemaVersion
}
func (m *Migrator) RequiredSchemaVersion() uint {
return appSchemaVersion
}
func (m *Migrator) getMigrate() (*migrate.Migrate, error) {
migrations, err := iofs.New(migrationsBox, "migrations")
if err != nil {
return nil, err
}
const disableForeignKeys = true
conn, err := m.db.open(disableForeignKeys)
if err != nil {
return nil, err
}
driver, err := sqlite3mig.WithInstance(conn.DB, &sqlite3mig.Config{})
if err != nil {
return nil, err
}
// use sqlite3Driver so that migration has access to durationToTinyInt
return migrate.NewWithInstance(
"iofs",
migrations,
m.db.dbPath,
driver,
)
}
func (m *Migrator) RunMigration(ctx context.Context, newVersion uint) error {
databaseSchemaVersion, _, _ := m.m.Version()
if newVersion != databaseSchemaVersion+1 {
return fmt.Errorf("invalid migration version %d, expected %d", newVersion, databaseSchemaVersion+1)
}
// run pre migrations as needed
if err := m.runCustomMigrations(ctx, preMigrations[newVersion]); err != nil {
return fmt.Errorf("running pre migrations for schema version %d: %w", newVersion, err)
}
if err := m.m.Steps(1); err != nil {
// migration failed
return err
}
// run post migrations as needed
if err := m.runCustomMigrations(ctx, postMigrations[newVersion]); err != nil {
return fmt.Errorf("running post migrations for schema version %d: %w", newVersion, err)
}
// update the schema version
m.db.schemaVersion, _, _ = m.m.Version()
return nil
}
func (m *Migrator) runCustomMigrations(ctx context.Context, fns []customMigrationFunc) error {
for _, fn := range fns {
if err := m.runCustomMigration(ctx, fn); err != nil {
return err
}
}
return nil
}
func (m *Migrator) runCustomMigration(ctx context.Context, fn customMigrationFunc) error {
const disableForeignKeys = false
d, err := m.db.open(disableForeignKeys)
if err != nil {
return err
}
defer d.Close()
if err := fn(ctx, d); err != nil {
return err
}
return nil
}
func (db *Database) getDatabaseSchemaVersion() (uint, error) {
m, err := NewMigrator(db)
if err != nil {
return 0, err
}
defer m.Close()
ret, _, _ := m.m.Version()
return ret, nil
}
func (db *Database) ReInitialise() error {
const disableForeignKeys = false
var err error
db.db, err = db.open(disableForeignKeys)
if err != nil {
return fmt.Errorf("re-initializing the database: %w", err)
}
return nil
}
// RunAllMigrations runs all migrations to bring the database up to the current schema version
func (db *Database) RunAllMigrations() error {
ctx := context.Background()
m, err := NewMigrator(db)
if err != nil {
return err
}
defer m.Close()
databaseSchemaVersion, _, _ := m.m.Version()
stepNumber := appSchemaVersion - databaseSchemaVersion
if stepNumber != 0 {
logger.Infof("Migrating database from version %d to %d", databaseSchemaVersion, appSchemaVersion)
// run each migration individually, and run custom migrations as needed
var i uint = 1
for ; i <= stepNumber; i++ {
newVersion := databaseSchemaVersion + i
if err := m.RunMigration(ctx, newVersion); err != nil {
return err
}
}
}
// re-initialise the database
const disableForeignKeys = false
db.db, err = db.open(disableForeignKeys)
if err != nil {
return fmt.Errorf("re-initializing the database: %w", err)
}
// optimize database after migration
err = db.Optimise(ctx)
if err != nil {
logger.Warnf("error while performing post-migration optimisation: %v", err)
}
return nil
}