Mirrors/stash
1
0
Fork 0
mirror of https://github.com/stashapp/stash.git synced 2025-12-17 04:14:39 +03:00
Code Issues Packages Projects Releases Wiki Activity

[Files Refactor] Performance tuning (#2784)

Browse Source 
* Improve image query performance
* Tune queries
* Fix db generator
* Don't show release notes in setup
* Further tune indexes
* Log when creating screenshot
This commit is contained in:
WithoutPants
2022-08-01 11:01:29 +10:00
parent abb574205a
commit bc47932343
20 changed files with 810 additions and 354 deletions
Download Patch File Download Diff File Expand all files Collapse all files

90
pkg/sqlite/scene.go
View File

@@ -456,10 +456,8 @@ func (qb *SceneStore) find(ctx context.Context, id int) (*models.Scene, error) {
}
func (qb *SceneStore) FindByFileID(ctx context.Context, fileID file.ID) ([]*models.Scene, error) {
table := qb.queryTable()
sq := dialect.From(table).Select(table.Col(idColumn)).Where(
table.Col("file_id").Eq(fileID),
sq := dialect.From(scenesFilesJoinTable).Select(scenesFilesJoinTable.Col(sceneIDColumn)).Where(
scenesFilesJoinTable.Col(fileIDColumn).Eq(fileID),
)
ret, err := qb.findBySubquery(ctx, sq)
@@ -478,18 +476,23 @@ func (qb *SceneStore) CountByFileID(ctx context.Context, fileID file.ID) (int, e
}
func (qb *SceneStore) FindByFingerprints(ctx context.Context, fp []file.Fingerprint) ([]*models.Scene, error) {
table := qb.queryTable()
fingerprintTable := fingerprintTableMgr.table
var ex []exp.Expression
for _, v := range fp {
ex = append(ex, goqu.And(
table.Col("fingerprint_type").Eq(v.Type),
table.Col("fingerprint").Eq(v.Fingerprint),
fingerprintTable.Col("type").Eq(v.Type),
fingerprintTable.Col("fingerprint").Eq(v.Fingerprint),
))
}
sq := dialect.From(table).Select(table.Col(idColumn)).Where(goqu.Or(ex...))
sq := dialect.From(scenesFilesJoinTable).
InnerJoin(
fingerprintTable,
goqu.On(fingerprintTable.Col(fileIDColumn).Eq(scenesFilesJoinTable.Col(fileIDColumn))),
).
Select(scenesFilesJoinTable.Col(sceneIDColumn)).Where(goqu.Or(ex...))
ret, err := qb.findBySubquery(ctx, sq)
if err != nil {
@@ -500,39 +503,26 @@ func (qb *SceneStore) FindByFingerprints(ctx context.Context, fp []file.Fingerpr
}
func (qb *SceneStore) FindByChecksum(ctx context.Context, checksum string) ([]*models.Scene, error) {
table := qb.queryTable()
sq := dialect.From(table).Select(table.Col(idColumn)).Where(
table.Col("fingerprint_type").Eq(file.FingerprintTypeMD5),
table.Col("fingerprint").Eq(checksum),
)
ret, err := qb.findBySubquery(ctx, sq)
if err != nil {
return nil, fmt.Errorf("getting scenes by checksum %s: %w", checksum, err)
}
return ret, nil
return qb.FindByFingerprints(ctx, []file.Fingerprint{
{
Type: file.FingerprintTypeMD5,
Fingerprint: checksum,
},
})
}
func (qb *SceneStore) FindByOSHash(ctx context.Context, oshash string) ([]*models.Scene, error) {
table := qb.queryTable()
sq := dialect.From(table).Select(table.Col(idColumn)).Where(
table.Col("fingerprint_type").Eq(file.FingerprintTypeOshash),
table.Col("fingerprint").Eq(oshash),
)
ret, err := qb.findBySubquery(ctx, sq)
if err != nil {
return nil, fmt.Errorf("getting scenes by oshash %s: %w", oshash, err)
}
return ret, nil
return qb.FindByFingerprints(ctx, []file.Fingerprint{
{
Type: file.FingerprintTypeOshash,
Fingerprint: oshash,
},
})
}
func (qb *SceneStore) FindByPath(ctx context.Context, p string) ([]*models.Scene, error) {
table := scenesQueryTable
filesTable := fileTableMgr.table
foldersTable := folderTableMgr.table
basename := filepath.Base(p)
dirStr := filepath.Dir(p)
@@ -542,9 +532,15 @@ func (qb *SceneStore) FindByPath(ctx context.Context, p string) ([]*models.Scene
dir, _ := path(dirStr).Value()
sq := dialect.From(table).Select(table.Col(idColumn)).Where(
table.Col("parent_folder_path").Like(dir),
table.Col("basename").Like(basename),
sq := dialect.From(scenesFilesJoinTable).InnerJoin(
filesTable,
goqu.On(filesTable.Col(idColumn).Eq(scenesFilesJoinTable.Col(fileIDColumn))),
).InnerJoin(
foldersTable,
goqu.On(foldersTable.Col(idColumn).Eq(filesTable.Col("parent_folder_id"))),
).Select(scenesFilesJoinTable.Col(sceneIDColumn)).Where(
foldersTable.Col("path").Like(dir),
filesTable.Col("basename").Like(basename),
)
ret, err := qb.findBySubquery(ctx, sq)
@@ -670,18 +666,16 @@ func (qb *SceneStore) CountByTagID(ctx context.Context, tagID int) (int, error)
}
func (qb *SceneStore) countMissingFingerprints(ctx context.Context, fpType string) (int, error) {
table := qb.queryTable()
fpTable := fingerprintTableMgr.table.As("fingerprints_temp")
q := dialect.Select(goqu.COUNT(goqu.DISTINCT(table.Col(idColumn)))).From(table).LeftJoin(
q := dialect.From(scenesFilesJoinTable).LeftJoin(
fpTable,
goqu.On(
table.Col("file_id").Eq(fpTable.Col("file_id")),
scenesFilesJoinTable.Col(fileIDColumn).Eq(fpTable.Col(fileIDColumn)),
fpTable.Col("type").Eq(fpType),
),
)
).Select(goqu.COUNT(goqu.DISTINCT(scenesFilesJoinTable.Col(sceneIDColumn)))).Where(fpTable.Col("fingerprint").IsNull())
q.Where(fpTable.Col("fingerprint").IsNull())
return count(ctx, q)
}
@@ -762,7 +756,7 @@ func (qb *SceneStore) makeFilter(ctx context.Context, sceneFilter *models.SceneF
query.not(qb.makeFilter(ctx, sceneFilter.Not))
}
query.handleCriterion(ctx, pathCriterionHandler(sceneFilter.Path, "scenes_query.parent_folder_path", "scenes_query.basename"))
query.handleCriterion(ctx, pathCriterionHandler(sceneFilter.Path, "scenes_query.parent_folder_path", "scenes_query.basename", nil))
query.handleCriterion(ctx, sceneFileCountCriterionHandler(qb, sceneFilter.FileCount))
query.handleCriterion(ctx, stringCriterionHandler(sceneFilter.Title, "scenes.title"))
query.handleCriterion(ctx, stringCriterionHandler(sceneFilter.Details, "scenes.details"))
@@ -799,7 +793,7 @@ func (qb *SceneStore) makeFilter(ctx context.Context, sceneFilter *models.SceneF
query.handleCriterion(ctx, boolCriterionHandler(sceneFilter.Organized, "scenes.organized"))
query.handleCriterion(ctx, durationCriterionHandler(sceneFilter.Duration, "scenes_query.duration"))
query.handleCriterion(ctx, resolutionCriterionHandler(sceneFilter.Resolution, "scenes_query.video_height", "scenes_query.video_width"))
query.handleCriterion(ctx, resolutionCriterionHandler(sceneFilter.Resolution, "scenes_query.video_height", "scenes_query.video_width", nil))
query.handleCriterion(ctx, hasMarkersCriterionHandler(sceneFilter.HasMarkers))
query.handleCriterion(ctx, sceneIsMissingCriterionHandler(qb, sceneFilter.IsMissing))
@@ -958,9 +952,13 @@ func durationCriterionHandler(durationFilter *models.IntCriterionInput, column s
}
}
func resolutionCriterionHandler(resolution *models.ResolutionCriterionInput, heightColumn string, widthColumn string) criterionHandlerFunc {
func resolutionCriterionHandler(resolution *models.ResolutionCriterionInput, heightColumn string, widthColumn string, addJoinFn func(f *filterBuilder)) criterionHandlerFunc {
return func(ctx context.Context, f *filterBuilder) {
if resolution != nil && resolution.Value.IsValid() {
if addJoinFn != nil {
addJoinFn(f)
}
min := resolution.Value.GetMinResolution()
max := resolution.Value.GetMaxResolution()