Mirrors/stash
1
0
Fork 0
mirror of https://github.com/stashapp/stash.git synced 2025-12-18 04:44:37 +03:00
Code Issues Packages Projects Releases Wiki Activity

[Files Refactor] Performance tuning (#2813)

Browse Source 
* Do database txn in same thread. Retry on locked db
* Remove captions from slimscenedata
* Fix tracing
* Use where in instead of individual selects
* Remove scenes_query view
* Remove image query view
* Remove gallery query view
* Use where in for FindMany
* Don't interrupt scanning zip files
* Fix image filesize sort
This commit is contained in:
WithoutPants
2022-08-11 16:14:57 +10:00
parent 87167736f6
commit 9b31b20fed
19 changed files with 715 additions and 680 deletions
Download Patch File Download Diff File Expand all files Collapse all files

239
pkg/sqlite/image.go
View File

@@ -48,6 +48,19 @@ func (r *imageRow) fromImage(i models.Image) {
r.UpdatedAt = i.UpdatedAt
}
func (r *imageRow) resolve() *models.Image {
return &models.Image{
ID: r.ID,
Title: r.Title.String,
Rating: nullIntPtr(r.Rating),
Organized: r.Organized,
OCounter: r.OCounter,
StudioID: nullIntPtr(r.StudioID),
CreatedAt: r.CreatedAt,
UpdatedAt: r.UpdatedAt,
}
}
type imageRowRecord struct {
updateRecord
}
@@ -62,109 +75,24 @@ func (r *imageRowRecord) fromPartial(i models.ImagePartial) {
r.setTime("updated_at", i.UpdatedAt)
}
type imageQueryRow struct {
imageRow
relatedFileQueryRow
GalleryID null.Int `db:"gallery_id"`
TagID null.Int `db:"tag_id"`
PerformerID null.Int `db:"performer_id"`
}
func (r *imageQueryRow) resolve() *models.Image {
ret := &models.Image{
ID: r.ID,
Title: r.Title.String,
Rating: nullIntPtr(r.Rating),
Organized: r.Organized,
OCounter: r.OCounter,
StudioID: nullIntPtr(r.StudioID),
CreatedAt: r.CreatedAt,
UpdatedAt: r.UpdatedAt,
}
r.appendRelationships(ret)
return ret
}
func appendImageFileUnique(vs []*file.ImageFile, toAdd *file.ImageFile, isPrimary bool) []*file.ImageFile {
// check in reverse, since it's most likely to be the last one
for i := len(vs) - 1; i >= 0; i-- {
if vs[i].Base().ID == toAdd.Base().ID {
// merge the two
mergeFiles(vs[i], toAdd)
return vs
}
}
if !isPrimary {
return append(vs, toAdd)
}
// primary should be first
return append([]*file.ImageFile{toAdd}, vs...)
}
func (r *imageQueryRow) appendRelationships(i *models.Image) {
if r.GalleryID.Valid {
i.GalleryIDs = intslice.IntAppendUnique(i.GalleryIDs, int(r.GalleryID.Int64))
}
if r.TagID.Valid {
i.TagIDs = intslice.IntAppendUnique(i.TagIDs, int(r.TagID.Int64))
}
if r.PerformerID.Valid {
i.PerformerIDs = intslice.IntAppendUnique(i.PerformerIDs, int(r.PerformerID.Int64))
}
if r.relatedFileQueryRow.FileID.Valid {
f := r.fileQueryRow.resolve().(*file.ImageFile)
i.Files = appendImageFileUnique(i.Files, f, r.Primary.Bool)
}
}
type imageQueryRows []imageQueryRow
func (r imageQueryRows) resolve() []*models.Image {
var ret []*models.Image
var last *models.Image
var lastID int
for _, row := range r {
if last == nil || lastID != row.ID {
f := row.resolve()
last = f
lastID = row.ID
ret = append(ret, last)
continue
}
// must be merging with previous row
row.appendRelationships(last)
}
return ret
}
type ImageStore struct {
repository
tableMgr *table
queryTableMgr *table
tableMgr *table
oCounterManager
fileStore *FileStore
}
func NewImageStore() *ImageStore {
func NewImageStore(fileStore *FileStore) *ImageStore {
return &ImageStore{
repository: repository{
tableName: imageTable,
idColumn: idColumn,
},
tableMgr: imageTableMgr,
queryTableMgr: imageQueryTableMgr,
oCounterManager: oCounterManager{imageTableMgr},
fileStore: fileStore,
}
}
@@ -172,10 +100,6 @@ func (qb *ImageStore) table() exp.IdentifierExpression {
return qb.tableMgr.table
}
func (qb *ImageStore) queryTable() exp.IdentifierExpression {
return qb.queryTableMgr.table
}
func (qb *ImageStore) Create(ctx context.Context, newObject *models.ImageCreateInput) error {
var r imageRow
r.fromImage(*newObject.Image)
@@ -291,21 +215,30 @@ func (qb *ImageStore) Find(ctx context.Context, id int) (*models.Image, error) {
}
func (qb *ImageStore) FindMany(ctx context.Context, ids []int) ([]*models.Image, error) {
var images []*models.Image
for _, id := range ids {
image, err := qb.Find(ctx, id)
if err != nil {
return nil, err
}
q := qb.selectDataset().Prepared(true).Where(qb.table().Col(idColumn).In(ids))
unsorted, err := qb.getMany(ctx, q)
if err != nil {
return nil, err
}
images = append(images, image)
images := make([]*models.Image, len(ids))
for _, s := range unsorted {
i := intslice.IntIndex(ids, s.ID)
images[i] = s
}
for i := range images {
if images[i] == nil {
return nil, fmt.Errorf("image with id %d not found", ids[i])
}
}
return images, nil
}
func (qb *ImageStore) selectDataset() *goqu.SelectDataset {
return dialect.From(imagesQueryTable).Select(imagesQueryTable.All())
return dialect.From(qb.table()).Select(qb.table().All())
}
func (qb *ImageStore) get(ctx context.Context, q *goqu.SelectDataset) (*models.Image, error) {
@@ -323,24 +256,84 @@ func (qb *ImageStore) get(ctx context.Context, q *goqu.SelectDataset) (*models.I
func (qb *ImageStore) getMany(ctx context.Context, q *goqu.SelectDataset) ([]*models.Image, error) {
const single = false
var rows imageQueryRows
var ret []*models.Image
if err := queryFunc(ctx, q, single, func(r *sqlx.Rows) error {
var f imageQueryRow
var f imageRow
if err := r.StructScan(&f); err != nil {
return err
}
rows = append(rows, f)
i := f.resolve()
if err := qb.resolveRelationships(ctx, i); err != nil {
return err
}
ret = append(ret, i)
return nil
}); err != nil {
return nil, err
}
return rows.resolve(), nil
return ret, nil
}
func (qb *ImageStore) resolveRelationships(ctx context.Context, i *models.Image) error {
var err error
// files
i.Files, err = qb.getFiles(ctx, i.ID)
if err != nil {
return fmt.Errorf("resolving image files: %w", err)
}
// performers
i.PerformerIDs, err = qb.performersRepository().getIDs(ctx, i.ID)
if err != nil {
return fmt.Errorf("resolving image performers: %w", err)
}
// tags
i.TagIDs, err = qb.tagsRepository().getIDs(ctx, i.ID)
if err != nil {
return fmt.Errorf("resolving image tags: %w", err)
}
// galleries
i.GalleryIDs, err = qb.galleriesRepository().getIDs(ctx, i.ID)
if err != nil {
return fmt.Errorf("resolving image galleries: %w", err)
}
return nil
}
func (qb *ImageStore) getFiles(ctx context.Context, id int) ([]*file.ImageFile, error) {
fileIDs, err := qb.filesRepository().get(ctx, id)
if err != nil {
return nil, err
}
// use fileStore to load files
files, err := qb.fileStore.Find(ctx, fileIDs...)
if err != nil {
return nil, err
}
ret := make([]*file.ImageFile, len(files))
for i, f := range files {
var ok bool
ret[i], ok = f.(*file.ImageFile)
if !ok {
return nil, fmt.Errorf("expected file to be *file.ImageFile not %T", f)
}
}
return ret, nil
}
func (qb *ImageStore) find(ctx context.Context, id int) (*models.Image, error) {
q := qb.selectDataset().Where(qb.queryTableMgr.byID(id))
q := qb.selectDataset().Where(qb.tableMgr.byID(id))
ret, err := qb.get(ctx, q)
if err != nil {
@@ -351,7 +344,7 @@ func (qb *ImageStore) find(ctx context.Context, id int) (*models.Image, error) {
}
func (qb *ImageStore) findBySubquery(ctx context.Context, sq *goqu.SelectDataset) ([]*models.Image, error) {
table := qb.queryTable()
table := qb.table()
q := qb.selectDataset().Prepared(true).Where(
table.Col(idColumn).Eq(
@@ -430,7 +423,8 @@ func (qb *ImageStore) FindByChecksum(ctx context.Context, checksum string) ([]*m
func (qb *ImageStore) FindByGalleryID(ctx context.Context, galleryID int) ([]*models.Image, error) {
table := qb.table()
queryTable := qb.queryTable()
fileTable := fileTableMgr.table
folderTable := folderTableMgr.table
sq := dialect.From(table).
InnerJoin(
@@ -441,11 +435,20 @@ func (qb *ImageStore) FindByGalleryID(ctx context.Context, galleryID int) ([]*mo
galleriesImagesJoinTable.Col("gallery_id").Eq(galleryID),
)
q := qb.selectDataset().Prepared(true).Where(
queryTable.Col(idColumn).Eq(
q := qb.selectDataset().Prepared(true).LeftJoin(
imagesFilesJoinTable,
goqu.On(imagesFilesJoinTable.Col(imageIDColumn).Eq(table.Col(idColumn))),
).LeftJoin(
fileTable,
goqu.On(fileTable.Col(idColumn).Eq(imagesFilesJoinTable.Col(fileIDColumn))),
).LeftJoin(
folderTable,
goqu.On(folderTable.Col(idColumn).Eq(fileTable.Col("parent_folder_id"))),
).Where(
table.Col(idColumn).Eq(
sq,
),
).Order(queryTable.Col("parent_folder_path").Asc(), queryTable.Col("basename").Asc())
).Order(folderTable.Col("path").Asc(), fileTable.Col("basename").Asc())
ret, err := qb.getMany(ctx, q)
if err != nil {
@@ -969,7 +972,7 @@ func (qb *ImageStore) setImageSortAndPagination(q *queryBuilder, findFilter *mod
sortClause = getCountSort(imageTable, imagesTagsTable, imageIDColumn, direction)
case "performer_count":
sortClause = getCountSort(imageTable, performersImagesTable, imageIDColumn, direction)
case "mod_time", "size":
case "mod_time", "filesize":
addFilesJoin()
sortClause = getSort(sort, direction, "files")
default:
@@ -991,6 +994,16 @@ func (qb *ImageStore) galleriesRepository() *joinRepository {
}
}
func (qb *ImageStore) filesRepository() *filesRepository {
return &filesRepository{
repository: repository{
tx: qb.tx,
tableName: imagesFilesTable,
idColumn: imageIDColumn,
},
}
}
// func (qb *imageQueryBuilder) GetGalleryIDs(ctx context.Context, imageID int) ([]int, error) {
// return qb.galleriesRepository().getIDs(ctx, imageID)
// }