Mirrors/stash
1
0
Fork 0
mirror of https://github.com/stashapp/stash.git synced 2025-12-17 20:34:37 +03:00
Code Issues Packages Projects Releases Wiki Activity

[Files Refactor] Performance tuning (#2865)

Browse Source 
* Don't load image files by default
* Don't load gallery files by default
* Don't load scene files by default
* Retry locked transactions forever
* Don't show release notes if config not loaded
* Don't translate path slashes in export
This commit is contained in:
WithoutPants
2022-09-01 17:54:34 +10:00
parent 0b534d89c6
commit 273cf0383d
94 changed files with 2611 additions and 981 deletions
Download Patch File Download Diff File Expand all files Collapse all files

115
pkg/sqlite/scene.go
View File

@@ -82,8 +82,17 @@ func (r *sceneRow) fromScene(o models.Scene) {
r.UpdatedAt = o.UpdatedAt
}
func (r *sceneRow) resolve() *models.Scene {
return &models.Scene{
type sceneQueryRow struct {
sceneRow
PrimaryFileID null.Int `db:"primary_file_id"`
PrimaryFileFolderPath zero.String `db:"primary_file_folder_path"`
PrimaryFileBasename zero.String `db:"primary_file_basename"`
PrimaryFileOshash zero.String `db:"primary_file_oshash"`
PrimaryFileChecksum zero.String `db:"primary_file_checksum"`
}
func (r *sceneQueryRow) resolve() *models.Scene {
ret := &models.Scene{
ID: r.ID,
Title: r.Title.String,
Details: r.Details.String,
@@ -93,9 +102,20 @@ func (r *sceneRow) resolve() *models.Scene {
Organized: r.Organized,
OCounter: r.OCounter,
StudioID: nullIntPtr(r.StudioID),
PrimaryFileID: nullIntFileIDPtr(r.PrimaryFileID),
OSHash: r.PrimaryFileOshash.String,
Checksum: r.PrimaryFileChecksum.String,
CreatedAt: r.CreatedAt,
UpdatedAt: r.UpdatedAt,
}
if r.PrimaryFileFolderPath.Valid && r.PrimaryFileBasename.Valid {
ret.Path = filepath.Join(r.PrimaryFileFolderPath.String, r.PrimaryFileBasename.String)
}
return ret
}
type sceneRowRecord struct {
@@ -278,13 +298,15 @@ func (qb *SceneStore) Update(ctx context.Context, updatedObject *models.Scene) e
}
}
fileIDs := make([]file.ID, len(updatedObject.Files))
for i, f := range updatedObject.Files {
fileIDs[i] = f.ID
}
if updatedObject.Files.Loaded() {
fileIDs := make([]file.ID, len(updatedObject.Files.List()))
for i, f := range updatedObject.Files.List() {
fileIDs[i] = f.ID
}
if err := scenesFilesTableMgr.replaceJoins(ctx, updatedObject.ID, fileIDs); err != nil {
return err
if err := scenesFilesTableMgr.replaceJoins(ctx, updatedObject.ID, fileIDs); err != nil {
return err
}
}
return nil
@@ -333,7 +355,43 @@ func (qb *SceneStore) FindMany(ctx context.Context, ids []int) ([]*models.Scene,
func (qb *SceneStore) selectDataset() *goqu.SelectDataset {
table := qb.table()
return dialect.From(table).Select(table.All())
files := fileTableMgr.table
folders := folderTableMgr.table
checksum := fingerprintTableMgr.table.As("fingerprint_md5")
oshash := fingerprintTableMgr.table.As("fingerprint_oshash")
return dialect.From(table).LeftJoin(
scenesFilesJoinTable,
goqu.On(
scenesFilesJoinTable.Col(sceneIDColumn).Eq(table.Col(idColumn)),
scenesFilesJoinTable.Col("primary").Eq(1),
),
).LeftJoin(
files,
goqu.On(files.Col(idColumn).Eq(scenesFilesJoinTable.Col(fileIDColumn))),
).LeftJoin(
folders,
goqu.On(folders.Col(idColumn).Eq(files.Col("parent_folder_id"))),
).LeftJoin(
checksum,
goqu.On(
checksum.Col(fileIDColumn).Eq(scenesFilesJoinTable.Col(fileIDColumn)),
checksum.Col("type").Eq(file.FingerprintTypeMD5),
),
).LeftJoin(
oshash,
goqu.On(
oshash.Col(fileIDColumn).Eq(scenesFilesJoinTable.Col(fileIDColumn)),
oshash.Col("type").Eq(file.FingerprintTypeOshash),
),
).Select(
qb.table().All(),
scenesFilesJoinTable.Col(fileIDColumn).As("primary_file_id"),
folders.Col("path").As("primary_file_folder_path"),
files.Col("basename").As("primary_file_basename"),
checksum.Col("fingerprint").As("primary_file_checksum"),
oshash.Col("fingerprint").As("primary_file_oshash"),
)
}
func (qb *SceneStore) get(ctx context.Context, q *goqu.SelectDataset) (*models.Scene, error) {
@@ -353,7 +411,7 @@ func (qb *SceneStore) getMany(ctx context.Context, q *goqu.SelectDataset) ([]*mo
const single = false
var ret []*models.Scene
if err := queryFunc(ctx, q, single, func(r *sqlx.Rows) error {
var f sceneRow
var f sceneQueryRow
if err := r.StructScan(&f); err != nil {
return err
}
@@ -366,28 +424,10 @@ func (qb *SceneStore) getMany(ctx context.Context, q *goqu.SelectDataset) ([]*mo
return nil, err
}
for _, s := range ret {
if err := qb.resolveRelationships(ctx, s); err != nil {
return nil, err
}
}
return ret, nil
}
func (qb *SceneStore) resolveRelationships(ctx context.Context, s *models.Scene) error {
var err error
// files
s.Files, err = qb.getFiles(ctx, s.ID)
if err != nil {
return fmt.Errorf("resolving scene files: %w", err)
}
return nil
}
func (qb *SceneStore) getFiles(ctx context.Context, id int) ([]*file.VideoFile, error) {
func (qb *SceneStore) GetFiles(ctx context.Context, id int) ([]*file.VideoFile, error) {
fileIDs, err := qb.filesRepository().get(ctx, id)
if err != nil {
return nil, err
@@ -411,6 +451,11 @@ func (qb *SceneStore) getFiles(ctx context.Context, id int) ([]*file.VideoFile,
return ret, nil
}
func (qb *SceneStore) GetManyFileIDs(ctx context.Context, ids []int) ([][]file.ID, error) {
const primaryOnly = false
return qb.filesRepository().getMany(ctx, ids, primaryOnly)
}
func (qb *SceneStore) find(ctx context.Context, id int) (*models.Scene, error) {
q := qb.selectDataset().Where(qb.tableMgr.byID(id))
@@ -680,16 +725,8 @@ func (qb *SceneStore) All(ctx context.Context) ([]*models.Scene, error) {
table := qb.table()
fileTable := fileTableMgr.table
folderTable := folderTableMgr.table
return qb.getMany(ctx, qb.selectDataset().LeftJoin(
scenesFilesJoinTable,
goqu.On(scenesFilesJoinTable.Col(sceneIDColumn).Eq(table.Col(idColumn))),
).LeftJoin(
fileTable,
goqu.On(fileTable.Col(idColumn).Eq(scenesFilesJoinTable.Col(fileIDColumn))),
).LeftJoin(
folderTable,
goqu.On(folderTable.Col(idColumn).Eq(fileTable.Col("parent_folder_id"))),
).Order(
return qb.getMany(ctx, qb.selectDataset().Order(
folderTable.Col("path").Asc(),
fileTable.Col("basename").Asc(),
table.Col("date").Asc(),