Add anonymise database task (#3186)

This commit is contained in:
WithoutPants
2022-12-23 09:15:27 +11:00
committed by GitHub
parent 0b4b100ecc
commit 9351a0b2a4
13 changed files with 1078 additions and 8 deletions

View File

@@ -212,7 +212,7 @@ func (db *Database) open(disableForeignKeys bool) (*sqlx.DB, error) {
return conn, nil
}
func (db *Database) Reset() error {
func (db *Database) Remove() error {
databasePath := db.dbPath
err := db.Close()
@@ -236,6 +236,15 @@ func (db *Database) Reset() error {
}
}
return nil
}
func (db *Database) Reset() error {
databasePath := db.dbPath
if err := db.Remove(); err != nil {
return err
}
if err := db.Open(databasePath); err != nil {
return fmt.Errorf("[reset DB] unable to initialize: %w", err)
}
@@ -265,6 +274,16 @@ func (db *Database) Backup(backupPath string) error {
return nil
}
func (db *Database) Anonymise(outPath string) error {
anon, err := NewAnonymiser(db, outPath)
if err != nil {
return err
}
return anon.Anonymise(context.Background())
}
func (db *Database) RestoreFromBackup(backupPath string) error {
logger.Infof("Restoring backup database %s into %s", backupPath, db.dbPath)
return os.Rename(backupPath, db.dbPath)
@@ -293,6 +312,16 @@ func (db *Database) DatabaseBackupPath(backupDirectoryPath string) string {
return fn
}
func (db *Database) AnonymousDatabasePath(backupDirectoryPath string) string {
fn := fmt.Sprintf("%s.anonymous.%d.%s", filepath.Base(db.dbPath), db.schemaVersion, time.Now().Format("20060102_150405"))
if backupDirectoryPath != "" {
return filepath.Join(backupDirectoryPath, fn)
}
return fn
}
func (db *Database) Version() uint {
return db.schemaVersion
}
@@ -383,8 +412,14 @@ func (db *Database) RunMigrations() error {
}
// optimize database after migration
db.optimise()
return nil
}
func (db *Database) optimise() {
logger.Info("Optimizing database")
_, err = db.db.Exec("ANALYZE")
_, err := db.db.Exec("ANALYZE")
if err != nil {
logger.Warnf("error while performing post-migration optimization: %v", err)
}
@@ -392,8 +427,6 @@ func (db *Database) RunMigrations() error {
if err != nil {
logger.Warnf("error while performing post-migration vacuum: %v", err)
}
return nil
}
func (db *Database) runCustomMigrations(ctx context.Context, fns []customMigrationFunc) error {