Autotag optimisation (#2368)

* Add duration to autotag finish message
* No sorting scene/image/gallery where not specified
* Use an LRU cache for sqlite regexp function
* Compile path separator regex once
* Cache objects with single letter first names
* Move finished auto-tag log
* Add more verbose logging
* Add new changelog
This commit is contained in:
WithoutPants
2022-03-09 12:01:56 +11:00
committed by GitHub
parent 9ef3060452
commit d88515abcd
28 changed files with 440 additions and 199 deletions

View File

@@ -2,21 +2,23 @@ package autotag
import ( import (
"github.com/stashapp/stash/pkg/gallery" "github.com/stashapp/stash/pkg/gallery"
"github.com/stashapp/stash/pkg/match"
"github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/models"
) )
func getGalleryFileTagger(s *models.Gallery) tagger { func getGalleryFileTagger(s *models.Gallery, cache *match.Cache) tagger {
return tagger{ return tagger{
ID: s.ID, ID: s.ID,
Type: "gallery", Type: "gallery",
Name: s.GetTitle(), Name: s.GetTitle(),
Path: s.Path.String, Path: s.Path.String,
cache: cache,
} }
} }
// GalleryPerformers tags the provided gallery with performers whose name matches the gallery's path. // GalleryPerformers tags the provided gallery with performers whose name matches the gallery's path.
func GalleryPerformers(s *models.Gallery, rw models.GalleryReaderWriter, performerReader models.PerformerReader) error { func GalleryPerformers(s *models.Gallery, rw models.GalleryReaderWriter, performerReader models.PerformerReader, cache *match.Cache) error {
t := getGalleryFileTagger(s) t := getGalleryFileTagger(s, cache)
return t.tagPerformers(performerReader, func(subjectID, otherID int) (bool, error) { return t.tagPerformers(performerReader, func(subjectID, otherID int) (bool, error) {
return gallery.AddPerformer(rw, subjectID, otherID) return gallery.AddPerformer(rw, subjectID, otherID)
@@ -26,13 +28,13 @@ func GalleryPerformers(s *models.Gallery, rw models.GalleryReaderWriter, perform
// GalleryStudios tags the provided gallery with the first studio whose name matches the gallery's path. // GalleryStudios tags the provided gallery with the first studio whose name matches the gallery's path.
// //
// Gallerys will not be tagged if studio is already set. // Gallerys will not be tagged if studio is already set.
func GalleryStudios(s *models.Gallery, rw models.GalleryReaderWriter, studioReader models.StudioReader) error { func GalleryStudios(s *models.Gallery, rw models.GalleryReaderWriter, studioReader models.StudioReader, cache *match.Cache) error {
if s.StudioID.Valid { if s.StudioID.Valid {
// don't modify // don't modify
return nil return nil
} }
t := getGalleryFileTagger(s) t := getGalleryFileTagger(s, cache)
return t.tagStudios(studioReader, func(subjectID, otherID int) (bool, error) { return t.tagStudios(studioReader, func(subjectID, otherID int) (bool, error) {
return addGalleryStudio(rw, subjectID, otherID) return addGalleryStudio(rw, subjectID, otherID)
@@ -40,8 +42,8 @@ func GalleryStudios(s *models.Gallery, rw models.GalleryReaderWriter, studioRead
} }
// GalleryTags tags the provided gallery with tags whose name matches the gallery's path. // GalleryTags tags the provided gallery with tags whose name matches the gallery's path.
func GalleryTags(s *models.Gallery, rw models.GalleryReaderWriter, tagReader models.TagReader) error { func GalleryTags(s *models.Gallery, rw models.GalleryReaderWriter, tagReader models.TagReader, cache *match.Cache) error {
t := getGalleryFileTagger(s) t := getGalleryFileTagger(s, cache)
return t.tagTags(tagReader, func(subjectID, otherID int) (bool, error) { return t.tagTags(tagReader, func(subjectID, otherID int) (bool, error) {
return gallery.AddTag(rw, subjectID, otherID) return gallery.AddTag(rw, subjectID, otherID)

View File

@@ -37,6 +37,7 @@ func TestGalleryPerformers(t *testing.T) {
mockPerformerReader := &mocks.PerformerReaderWriter{} mockPerformerReader := &mocks.PerformerReaderWriter{}
mockGalleryReader := &mocks.GalleryReaderWriter{} mockGalleryReader := &mocks.GalleryReaderWriter{}
mockPerformerReader.On("Query", mock.Anything, mock.Anything).Return(nil, 0, nil)
mockPerformerReader.On("QueryForAutoTag", mock.Anything).Return([]*models.Performer{&performer, &reversedPerformer}, nil).Once() mockPerformerReader.On("QueryForAutoTag", mock.Anything).Return([]*models.Performer{&performer, &reversedPerformer}, nil).Once()
if test.Matches { if test.Matches {
@@ -48,7 +49,7 @@ func TestGalleryPerformers(t *testing.T) {
ID: galleryID, ID: galleryID,
Path: models.NullString(test.Path), Path: models.NullString(test.Path),
} }
err := GalleryPerformers(&gallery, mockGalleryReader, mockPerformerReader) err := GalleryPerformers(&gallery, mockGalleryReader, mockPerformerReader, nil)
assert.Nil(err) assert.Nil(err)
mockPerformerReader.AssertExpectations(t) mockPerformerReader.AssertExpectations(t)
@@ -92,7 +93,7 @@ func TestGalleryStudios(t *testing.T) {
ID: galleryID, ID: galleryID,
Path: models.NullString(test.Path), Path: models.NullString(test.Path),
} }
err := GalleryStudios(&gallery, mockGalleryReader, mockStudioReader) err := GalleryStudios(&gallery, mockGalleryReader, mockStudioReader, nil)
assert.Nil(err) assert.Nil(err)
mockStudioReader.AssertExpectations(t) mockStudioReader.AssertExpectations(t)
@@ -103,6 +104,7 @@ func TestGalleryStudios(t *testing.T) {
mockStudioReader := &mocks.StudioReaderWriter{} mockStudioReader := &mocks.StudioReaderWriter{}
mockGalleryReader := &mocks.GalleryReaderWriter{} mockGalleryReader := &mocks.GalleryReaderWriter{}
mockStudioReader.On("Query", mock.Anything, mock.Anything).Return(nil, 0, nil)
mockStudioReader.On("QueryForAutoTag", mock.Anything).Return([]*models.Studio{&studio, &reversedStudio}, nil).Once() mockStudioReader.On("QueryForAutoTag", mock.Anything).Return([]*models.Studio{&studio, &reversedStudio}, nil).Once()
mockStudioReader.On("GetAliases", mock.Anything).Return([]string{}, nil).Maybe() mockStudioReader.On("GetAliases", mock.Anything).Return([]string{}, nil).Maybe()
@@ -117,6 +119,7 @@ func TestGalleryStudios(t *testing.T) {
mockStudioReader := &mocks.StudioReaderWriter{} mockStudioReader := &mocks.StudioReaderWriter{}
mockGalleryReader := &mocks.GalleryReaderWriter{} mockGalleryReader := &mocks.GalleryReaderWriter{}
mockStudioReader.On("Query", mock.Anything, mock.Anything).Return(nil, 0, nil)
mockStudioReader.On("QueryForAutoTag", mock.Anything).Return([]*models.Studio{&studio, &reversedStudio}, nil).Once() mockStudioReader.On("QueryForAutoTag", mock.Anything).Return([]*models.Studio{&studio, &reversedStudio}, nil).Once()
mockStudioReader.On("GetAliases", studioID).Return([]string{ mockStudioReader.On("GetAliases", studioID).Return([]string{
studioName, studioName,
@@ -159,7 +162,7 @@ func TestGalleryTags(t *testing.T) {
ID: galleryID, ID: galleryID,
Path: models.NullString(test.Path), Path: models.NullString(test.Path),
} }
err := GalleryTags(&gallery, mockGalleryReader, mockTagReader) err := GalleryTags(&gallery, mockGalleryReader, mockTagReader, nil)
assert.Nil(err) assert.Nil(err)
mockTagReader.AssertExpectations(t) mockTagReader.AssertExpectations(t)
@@ -170,6 +173,7 @@ func TestGalleryTags(t *testing.T) {
mockTagReader := &mocks.TagReaderWriter{} mockTagReader := &mocks.TagReaderWriter{}
mockGalleryReader := &mocks.GalleryReaderWriter{} mockGalleryReader := &mocks.GalleryReaderWriter{}
mockTagReader.On("Query", mock.Anything, mock.Anything).Return(nil, 0, nil)
mockTagReader.On("QueryForAutoTag", mock.Anything).Return([]*models.Tag{&tag, &reversedTag}, nil).Once() mockTagReader.On("QueryForAutoTag", mock.Anything).Return([]*models.Tag{&tag, &reversedTag}, nil).Once()
mockTagReader.On("GetAliases", mock.Anything).Return([]string{}, nil).Maybe() mockTagReader.On("GetAliases", mock.Anything).Return([]string{}, nil).Maybe()
@@ -183,6 +187,7 @@ func TestGalleryTags(t *testing.T) {
mockTagReader := &mocks.TagReaderWriter{} mockTagReader := &mocks.TagReaderWriter{}
mockGalleryReader := &mocks.GalleryReaderWriter{} mockGalleryReader := &mocks.GalleryReaderWriter{}
mockTagReader.On("Query", mock.Anything, mock.Anything).Return(nil, 0, nil)
mockTagReader.On("QueryForAutoTag", mock.Anything).Return([]*models.Tag{&tag, &reversedTag}, nil).Once() mockTagReader.On("QueryForAutoTag", mock.Anything).Return([]*models.Tag{&tag, &reversedTag}, nil).Once()
mockTagReader.On("GetAliases", tagID).Return([]string{ mockTagReader.On("GetAliases", tagID).Return([]string{
tagName, tagName,

View File

@@ -2,21 +2,23 @@ package autotag
import ( import (
"github.com/stashapp/stash/pkg/image" "github.com/stashapp/stash/pkg/image"
"github.com/stashapp/stash/pkg/match"
"github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/models"
) )
func getImageFileTagger(s *models.Image) tagger { func getImageFileTagger(s *models.Image, cache *match.Cache) tagger {
return tagger{ return tagger{
ID: s.ID, ID: s.ID,
Type: "image", Type: "image",
Name: s.GetTitle(), Name: s.GetTitle(),
Path: s.Path, Path: s.Path,
cache: cache,
} }
} }
// ImagePerformers tags the provided image with performers whose name matches the image's path. // ImagePerformers tags the provided image with performers whose name matches the image's path.
func ImagePerformers(s *models.Image, rw models.ImageReaderWriter, performerReader models.PerformerReader) error { func ImagePerformers(s *models.Image, rw models.ImageReaderWriter, performerReader models.PerformerReader, cache *match.Cache) error {
t := getImageFileTagger(s) t := getImageFileTagger(s, cache)
return t.tagPerformers(performerReader, func(subjectID, otherID int) (bool, error) { return t.tagPerformers(performerReader, func(subjectID, otherID int) (bool, error) {
return image.AddPerformer(rw, subjectID, otherID) return image.AddPerformer(rw, subjectID, otherID)
@@ -26,13 +28,13 @@ func ImagePerformers(s *models.Image, rw models.ImageReaderWriter, performerRead
// ImageStudios tags the provided image with the first studio whose name matches the image's path. // ImageStudios tags the provided image with the first studio whose name matches the image's path.
// //
// Images will not be tagged if studio is already set. // Images will not be tagged if studio is already set.
func ImageStudios(s *models.Image, rw models.ImageReaderWriter, studioReader models.StudioReader) error { func ImageStudios(s *models.Image, rw models.ImageReaderWriter, studioReader models.StudioReader, cache *match.Cache) error {
if s.StudioID.Valid { if s.StudioID.Valid {
// don't modify // don't modify
return nil return nil
} }
t := getImageFileTagger(s) t := getImageFileTagger(s, cache)
return t.tagStudios(studioReader, func(subjectID, otherID int) (bool, error) { return t.tagStudios(studioReader, func(subjectID, otherID int) (bool, error) {
return addImageStudio(rw, subjectID, otherID) return addImageStudio(rw, subjectID, otherID)
@@ -40,8 +42,8 @@ func ImageStudios(s *models.Image, rw models.ImageReaderWriter, studioReader mod
} }
// ImageTags tags the provided image with tags whose name matches the image's path. // ImageTags tags the provided image with tags whose name matches the image's path.
func ImageTags(s *models.Image, rw models.ImageReaderWriter, tagReader models.TagReader) error { func ImageTags(s *models.Image, rw models.ImageReaderWriter, tagReader models.TagReader, cache *match.Cache) error {
t := getImageFileTagger(s) t := getImageFileTagger(s, cache)
return t.tagTags(tagReader, func(subjectID, otherID int) (bool, error) { return t.tagTags(tagReader, func(subjectID, otherID int) (bool, error) {
return image.AddTag(rw, subjectID, otherID) return image.AddTag(rw, subjectID, otherID)

View File

@@ -37,6 +37,7 @@ func TestImagePerformers(t *testing.T) {
mockPerformerReader := &mocks.PerformerReaderWriter{} mockPerformerReader := &mocks.PerformerReaderWriter{}
mockImageReader := &mocks.ImageReaderWriter{} mockImageReader := &mocks.ImageReaderWriter{}
mockPerformerReader.On("Query", mock.Anything, mock.Anything).Return(nil, 0, nil)
mockPerformerReader.On("QueryForAutoTag", mock.Anything).Return([]*models.Performer{&performer, &reversedPerformer}, nil).Once() mockPerformerReader.On("QueryForAutoTag", mock.Anything).Return([]*models.Performer{&performer, &reversedPerformer}, nil).Once()
if test.Matches { if test.Matches {
@@ -48,7 +49,7 @@ func TestImagePerformers(t *testing.T) {
ID: imageID, ID: imageID,
Path: test.Path, Path: test.Path,
} }
err := ImagePerformers(&image, mockImageReader, mockPerformerReader) err := ImagePerformers(&image, mockImageReader, mockPerformerReader, nil)
assert.Nil(err) assert.Nil(err)
mockPerformerReader.AssertExpectations(t) mockPerformerReader.AssertExpectations(t)
@@ -92,7 +93,7 @@ func TestImageStudios(t *testing.T) {
ID: imageID, ID: imageID,
Path: test.Path, Path: test.Path,
} }
err := ImageStudios(&image, mockImageReader, mockStudioReader) err := ImageStudios(&image, mockImageReader, mockStudioReader, nil)
assert.Nil(err) assert.Nil(err)
mockStudioReader.AssertExpectations(t) mockStudioReader.AssertExpectations(t)
@@ -103,6 +104,7 @@ func TestImageStudios(t *testing.T) {
mockStudioReader := &mocks.StudioReaderWriter{} mockStudioReader := &mocks.StudioReaderWriter{}
mockImageReader := &mocks.ImageReaderWriter{} mockImageReader := &mocks.ImageReaderWriter{}
mockStudioReader.On("Query", mock.Anything, mock.Anything).Return(nil, 0, nil)
mockStudioReader.On("QueryForAutoTag", mock.Anything).Return([]*models.Studio{&studio, &reversedStudio}, nil).Once() mockStudioReader.On("QueryForAutoTag", mock.Anything).Return([]*models.Studio{&studio, &reversedStudio}, nil).Once()
mockStudioReader.On("GetAliases", mock.Anything).Return([]string{}, nil).Maybe() mockStudioReader.On("GetAliases", mock.Anything).Return([]string{}, nil).Maybe()
@@ -117,6 +119,7 @@ func TestImageStudios(t *testing.T) {
mockStudioReader := &mocks.StudioReaderWriter{} mockStudioReader := &mocks.StudioReaderWriter{}
mockImageReader := &mocks.ImageReaderWriter{} mockImageReader := &mocks.ImageReaderWriter{}
mockStudioReader.On("Query", mock.Anything, mock.Anything).Return(nil, 0, nil)
mockStudioReader.On("QueryForAutoTag", mock.Anything).Return([]*models.Studio{&studio, &reversedStudio}, nil).Once() mockStudioReader.On("QueryForAutoTag", mock.Anything).Return([]*models.Studio{&studio, &reversedStudio}, nil).Once()
mockStudioReader.On("GetAliases", studioID).Return([]string{ mockStudioReader.On("GetAliases", studioID).Return([]string{
studioName, studioName,
@@ -159,7 +162,7 @@ func TestImageTags(t *testing.T) {
ID: imageID, ID: imageID,
Path: test.Path, Path: test.Path,
} }
err := ImageTags(&image, mockImageReader, mockTagReader) err := ImageTags(&image, mockImageReader, mockTagReader, nil)
assert.Nil(err) assert.Nil(err)
mockTagReader.AssertExpectations(t) mockTagReader.AssertExpectations(t)
@@ -170,6 +173,7 @@ func TestImageTags(t *testing.T) {
mockTagReader := &mocks.TagReaderWriter{} mockTagReader := &mocks.TagReaderWriter{}
mockImageReader := &mocks.ImageReaderWriter{} mockImageReader := &mocks.ImageReaderWriter{}
mockTagReader.On("Query", mock.Anything, mock.Anything).Return(nil, 0, nil)
mockTagReader.On("QueryForAutoTag", mock.Anything).Return([]*models.Tag{&tag, &reversedTag}, nil).Once() mockTagReader.On("QueryForAutoTag", mock.Anything).Return([]*models.Tag{&tag, &reversedTag}, nil).Once()
mockTagReader.On("GetAliases", mock.Anything).Return([]string{}, nil).Maybe() mockTagReader.On("GetAliases", mock.Anything).Return([]string{}, nil).Maybe()
@@ -184,6 +188,7 @@ func TestImageTags(t *testing.T) {
mockTagReader := &mocks.TagReaderWriter{} mockTagReader := &mocks.TagReaderWriter{}
mockImageReader := &mocks.ImageReaderWriter{} mockImageReader := &mocks.ImageReaderWriter{}
mockTagReader.On("Query", mock.Anything, mock.Anything).Return(nil, 0, nil)
mockTagReader.On("QueryForAutoTag", mock.Anything).Return([]*models.Tag{&tag, &reversedTag}, nil).Once() mockTagReader.On("QueryForAutoTag", mock.Anything).Return([]*models.Tag{&tag, &reversedTag}, nil).Once()
mockTagReader.On("GetAliases", tagID).Return([]string{ mockTagReader.On("GetAliases", tagID).Return([]string{
tagName, tagName,

View File

@@ -361,7 +361,7 @@ func TestParsePerformerScenes(t *testing.T) {
for _, p := range performers { for _, p := range performers {
if err := withTxn(func(r models.Repository) error { if err := withTxn(func(r models.Repository) error {
return PerformerScenes(p, nil, r.Scene()) return PerformerScenes(p, nil, r.Scene(), nil)
}); err != nil { }); err != nil {
t.Errorf("Error auto-tagging performers: %s", err) t.Errorf("Error auto-tagging performers: %s", err)
} }
@@ -413,7 +413,7 @@ func TestParseStudioScenes(t *testing.T) {
return err return err
} }
return StudioScenes(s, nil, aliases, r.Scene()) return StudioScenes(s, nil, aliases, r.Scene(), nil)
}); err != nil { }); err != nil {
t.Errorf("Error auto-tagging performers: %s", err) t.Errorf("Error auto-tagging performers: %s", err)
} }
@@ -469,7 +469,7 @@ func TestParseTagScenes(t *testing.T) {
return err return err
} }
return TagScenes(s, nil, aliases, r.Scene()) return TagScenes(s, nil, aliases, r.Scene(), nil)
}); err != nil { }); err != nil {
t.Errorf("Error auto-tagging performers: %s", err) t.Errorf("Error auto-tagging performers: %s", err)
} }
@@ -516,7 +516,7 @@ func TestParsePerformerImages(t *testing.T) {
for _, p := range performers { for _, p := range performers {
if err := withTxn(func(r models.Repository) error { if err := withTxn(func(r models.Repository) error {
return PerformerImages(p, nil, r.Image()) return PerformerImages(p, nil, r.Image(), nil)
}); err != nil { }); err != nil {
t.Errorf("Error auto-tagging performers: %s", err) t.Errorf("Error auto-tagging performers: %s", err)
} }
@@ -568,7 +568,7 @@ func TestParseStudioImages(t *testing.T) {
return err return err
} }
return StudioImages(s, nil, aliases, r.Image()) return StudioImages(s, nil, aliases, r.Image(), nil)
}); err != nil { }); err != nil {
t.Errorf("Error auto-tagging performers: %s", err) t.Errorf("Error auto-tagging performers: %s", err)
} }
@@ -624,7 +624,7 @@ func TestParseTagImages(t *testing.T) {
return err return err
} }
return TagImages(s, nil, aliases, r.Image()) return TagImages(s, nil, aliases, r.Image(), nil)
}); err != nil { }); err != nil {
t.Errorf("Error auto-tagging performers: %s", err) t.Errorf("Error auto-tagging performers: %s", err)
} }
@@ -671,7 +671,7 @@ func TestParsePerformerGalleries(t *testing.T) {
for _, p := range performers { for _, p := range performers {
if err := withTxn(func(r models.Repository) error { if err := withTxn(func(r models.Repository) error {
return PerformerGalleries(p, nil, r.Gallery()) return PerformerGalleries(p, nil, r.Gallery(), nil)
}); err != nil { }); err != nil {
t.Errorf("Error auto-tagging performers: %s", err) t.Errorf("Error auto-tagging performers: %s", err)
} }
@@ -723,7 +723,7 @@ func TestParseStudioGalleries(t *testing.T) {
return err return err
} }
return StudioGalleries(s, nil, aliases, r.Gallery()) return StudioGalleries(s, nil, aliases, r.Gallery(), nil)
}); err != nil { }); err != nil {
t.Errorf("Error auto-tagging performers: %s", err) t.Errorf("Error auto-tagging performers: %s", err)
} }
@@ -779,7 +779,7 @@ func TestParseTagGalleries(t *testing.T) {
return err return err
} }
return TagGalleries(s, nil, aliases, r.Gallery()) return TagGalleries(s, nil, aliases, r.Gallery(), nil)
}); err != nil { }); err != nil {
t.Errorf("Error auto-tagging performers: %s", err) t.Errorf("Error auto-tagging performers: %s", err)
} }

View File

@@ -3,21 +3,23 @@ package autotag
import ( import (
"github.com/stashapp/stash/pkg/gallery" "github.com/stashapp/stash/pkg/gallery"
"github.com/stashapp/stash/pkg/image" "github.com/stashapp/stash/pkg/image"
"github.com/stashapp/stash/pkg/match"
"github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/models"
"github.com/stashapp/stash/pkg/scene" "github.com/stashapp/stash/pkg/scene"
) )
func getPerformerTagger(p *models.Performer) tagger { func getPerformerTagger(p *models.Performer, cache *match.Cache) tagger {
return tagger{ return tagger{
ID: p.ID, ID: p.ID,
Type: "performer", Type: "performer",
Name: p.Name.String, Name: p.Name.String,
cache: cache,
} }
} }
// PerformerScenes searches for scenes whose path matches the provided performer name and tags the scene with the performer. // PerformerScenes searches for scenes whose path matches the provided performer name and tags the scene with the performer.
func PerformerScenes(p *models.Performer, paths []string, rw models.SceneReaderWriter) error { func PerformerScenes(p *models.Performer, paths []string, rw models.SceneReaderWriter, cache *match.Cache) error {
t := getPerformerTagger(p) t := getPerformerTagger(p, cache)
return t.tagScenes(paths, rw, func(subjectID, otherID int) (bool, error) { return t.tagScenes(paths, rw, func(subjectID, otherID int) (bool, error) {
return scene.AddPerformer(rw, otherID, subjectID) return scene.AddPerformer(rw, otherID, subjectID)
@@ -25,8 +27,8 @@ func PerformerScenes(p *models.Performer, paths []string, rw models.SceneReaderW
} }
// PerformerImages searches for images whose path matches the provided performer name and tags the image with the performer. // PerformerImages searches for images whose path matches the provided performer name and tags the image with the performer.
func PerformerImages(p *models.Performer, paths []string, rw models.ImageReaderWriter) error { func PerformerImages(p *models.Performer, paths []string, rw models.ImageReaderWriter, cache *match.Cache) error {
t := getPerformerTagger(p) t := getPerformerTagger(p, cache)
return t.tagImages(paths, rw, func(subjectID, otherID int) (bool, error) { return t.tagImages(paths, rw, func(subjectID, otherID int) (bool, error) {
return image.AddPerformer(rw, otherID, subjectID) return image.AddPerformer(rw, otherID, subjectID)
@@ -34,8 +36,8 @@ func PerformerImages(p *models.Performer, paths []string, rw models.ImageReaderW
} }
// PerformerGalleries searches for galleries whose path matches the provided performer name and tags the gallery with the performer. // PerformerGalleries searches for galleries whose path matches the provided performer name and tags the gallery with the performer.
func PerformerGalleries(p *models.Performer, paths []string, rw models.GalleryReaderWriter) error { func PerformerGalleries(p *models.Performer, paths []string, rw models.GalleryReaderWriter, cache *match.Cache) error {
t := getPerformerTagger(p) t := getPerformerTagger(p, cache)
return t.tagGalleries(paths, rw, func(subjectID, otherID int) (bool, error) { return t.tagGalleries(paths, rw, func(subjectID, otherID int) (bool, error) {
return gallery.AddPerformer(rw, otherID, subjectID) return gallery.AddPerformer(rw, otherID, subjectID)

View File

@@ -21,15 +21,15 @@ func TestPerformerScenes(t *testing.T) {
performerNames := []test{ performerNames := []test{
{ {
"performer name", "performer name",
`(?i)(?:^|_|[^\w\d])performer[.\-_ ]*name(?:$|_|[^\w\d])`, `(?i)(?:^|_|[^\p{L}\d])performer[.\-_ ]*name(?:$|_|[^\p{L}\d])`,
}, },
{ {
"performer + name", "performer + name",
`(?i)(?:^|_|[^\w\d])performer[.\-_ ]*\+[.\-_ ]*name(?:$|_|[^\w\d])`, `(?i)(?:^|_|[^\p{L}\d])performer[.\-_ ]*\+[.\-_ ]*name(?:$|_|[^\p{L}\d])`,
}, },
{ {
`performer + name\`, `performer + name\`,
`(?i)(?:^|_|[^\w\d])performer[.\-_ ]*\+[.\-_ ]*name\\(?:$|_|[^\w\d])`, `(?i)(?:^|_|[^\p{L}\d])performer[.\-_ ]*\+[.\-_ ]*name\\(?:$|_|[^\p{L}\d])`,
}, },
} }
@@ -81,7 +81,7 @@ func testPerformerScenes(t *testing.T, performerName, expectedRegex string) {
mockSceneReader.On("UpdatePerformers", sceneID, []int{performerID}).Return(nil).Once() mockSceneReader.On("UpdatePerformers", sceneID, []int{performerID}).Return(nil).Once()
} }
err := PerformerScenes(&performer, nil, mockSceneReader) err := PerformerScenes(&performer, nil, mockSceneReader, nil)
assert := assert.New(t) assert := assert.New(t)
@@ -100,11 +100,11 @@ func TestPerformerImages(t *testing.T) {
performerNames := []test{ performerNames := []test{
{ {
"performer name", "performer name",
`(?i)(?:^|_|[^\w\d])performer[.\-_ ]*name(?:$|_|[^\w\d])`, `(?i)(?:^|_|[^\p{L}\d])performer[.\-_ ]*name(?:$|_|[^\p{L}\d])`,
}, },
{ {
"performer + name", "performer + name",
`(?i)(?:^|_|[^\w\d])performer[.\-_ ]*\+[.\-_ ]*name(?:$|_|[^\w\d])`, `(?i)(?:^|_|[^\p{L}\d])performer[.\-_ ]*\+[.\-_ ]*name(?:$|_|[^\p{L}\d])`,
}, },
} }
@@ -156,7 +156,7 @@ func testPerformerImages(t *testing.T, performerName, expectedRegex string) {
mockImageReader.On("UpdatePerformers", imageID, []int{performerID}).Return(nil).Once() mockImageReader.On("UpdatePerformers", imageID, []int{performerID}).Return(nil).Once()
} }
err := PerformerImages(&performer, nil, mockImageReader) err := PerformerImages(&performer, nil, mockImageReader, nil)
assert := assert.New(t) assert := assert.New(t)
@@ -175,11 +175,11 @@ func TestPerformerGalleries(t *testing.T) {
performerNames := []test{ performerNames := []test{
{ {
"performer name", "performer name",
`(?i)(?:^|_|[^\w\d])performer[.\-_ ]*name(?:$|_|[^\w\d])`, `(?i)(?:^|_|[^\p{L}\d])performer[.\-_ ]*name(?:$|_|[^\p{L}\d])`,
}, },
{ {
"performer + name", "performer + name",
`(?i)(?:^|_|[^\w\d])performer[.\-_ ]*\+[.\-_ ]*name(?:$|_|[^\w\d])`, `(?i)(?:^|_|[^\p{L}\d])performer[.\-_ ]*\+[.\-_ ]*name(?:$|_|[^\p{L}\d])`,
}, },
} }
@@ -230,7 +230,7 @@ func testPerformerGalleries(t *testing.T, performerName, expectedRegex string) {
mockGalleryReader.On("UpdatePerformers", galleryID, []int{performerID}).Return(nil).Once() mockGalleryReader.On("UpdatePerformers", galleryID, []int{performerID}).Return(nil).Once()
} }
err := PerformerGalleries(&performer, nil, mockGalleryReader) err := PerformerGalleries(&performer, nil, mockGalleryReader, nil)
assert := assert.New(t) assert := assert.New(t)

View File

@@ -1,22 +1,24 @@
package autotag package autotag
import ( import (
"github.com/stashapp/stash/pkg/match"
"github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/models"
"github.com/stashapp/stash/pkg/scene" "github.com/stashapp/stash/pkg/scene"
) )
func getSceneFileTagger(s *models.Scene) tagger { func getSceneFileTagger(s *models.Scene, cache *match.Cache) tagger {
return tagger{ return tagger{
ID: s.ID, ID: s.ID,
Type: "scene", Type: "scene",
Name: s.GetTitle(), Name: s.GetTitle(),
Path: s.Path, Path: s.Path,
cache: cache,
} }
} }
// ScenePerformers tags the provided scene with performers whose name matches the scene's path. // ScenePerformers tags the provided scene with performers whose name matches the scene's path.
func ScenePerformers(s *models.Scene, rw models.SceneReaderWriter, performerReader models.PerformerReader) error { func ScenePerformers(s *models.Scene, rw models.SceneReaderWriter, performerReader models.PerformerReader, cache *match.Cache) error {
t := getSceneFileTagger(s) t := getSceneFileTagger(s, cache)
return t.tagPerformers(performerReader, func(subjectID, otherID int) (bool, error) { return t.tagPerformers(performerReader, func(subjectID, otherID int) (bool, error) {
return scene.AddPerformer(rw, subjectID, otherID) return scene.AddPerformer(rw, subjectID, otherID)
@@ -26,13 +28,13 @@ func ScenePerformers(s *models.Scene, rw models.SceneReaderWriter, performerRead
// SceneStudios tags the provided scene with the first studio whose name matches the scene's path. // SceneStudios tags the provided scene with the first studio whose name matches the scene's path.
// //
// Scenes will not be tagged if studio is already set. // Scenes will not be tagged if studio is already set.
func SceneStudios(s *models.Scene, rw models.SceneReaderWriter, studioReader models.StudioReader) error { func SceneStudios(s *models.Scene, rw models.SceneReaderWriter, studioReader models.StudioReader, cache *match.Cache) error {
if s.StudioID.Valid { if s.StudioID.Valid {
// don't modify // don't modify
return nil return nil
} }
t := getSceneFileTagger(s) t := getSceneFileTagger(s, cache)
return t.tagStudios(studioReader, func(subjectID, otherID int) (bool, error) { return t.tagStudios(studioReader, func(subjectID, otherID int) (bool, error) {
return addSceneStudio(rw, subjectID, otherID) return addSceneStudio(rw, subjectID, otherID)
@@ -40,8 +42,8 @@ func SceneStudios(s *models.Scene, rw models.SceneReaderWriter, studioReader mod
} }
// SceneTags tags the provided scene with tags whose name matches the scene's path. // SceneTags tags the provided scene with tags whose name matches the scene's path.
func SceneTags(s *models.Scene, rw models.SceneReaderWriter, tagReader models.TagReader) error { func SceneTags(s *models.Scene, rw models.SceneReaderWriter, tagReader models.TagReader, cache *match.Cache) error {
t := getSceneFileTagger(s) t := getSceneFileTagger(s, cache)
return t.tagTags(tagReader, func(subjectID, otherID int) (bool, error) { return t.tagTags(tagReader, func(subjectID, otherID int) (bool, error) {
return scene.AddTag(rw, subjectID, otherID) return scene.AddTag(rw, subjectID, otherID)

View File

@@ -172,6 +172,7 @@ func TestScenePerformers(t *testing.T) {
mockPerformerReader := &mocks.PerformerReaderWriter{} mockPerformerReader := &mocks.PerformerReaderWriter{}
mockSceneReader := &mocks.SceneReaderWriter{} mockSceneReader := &mocks.SceneReaderWriter{}
mockPerformerReader.On("Query", mock.Anything, mock.Anything).Return(nil, 0, nil)
mockPerformerReader.On("QueryForAutoTag", mock.Anything).Return([]*models.Performer{&performer, &reversedPerformer}, nil).Once() mockPerformerReader.On("QueryForAutoTag", mock.Anything).Return([]*models.Performer{&performer, &reversedPerformer}, nil).Once()
if test.Matches { if test.Matches {
@@ -183,7 +184,7 @@ func TestScenePerformers(t *testing.T) {
ID: sceneID, ID: sceneID,
Path: test.Path, Path: test.Path,
} }
err := ScenePerformers(&scene, mockSceneReader, mockPerformerReader) err := ScenePerformers(&scene, mockSceneReader, mockPerformerReader, nil)
assert.Nil(err) assert.Nil(err)
mockPerformerReader.AssertExpectations(t) mockPerformerReader.AssertExpectations(t)
@@ -227,7 +228,7 @@ func TestSceneStudios(t *testing.T) {
ID: sceneID, ID: sceneID,
Path: test.Path, Path: test.Path,
} }
err := SceneStudios(&scene, mockSceneReader, mockStudioReader) err := SceneStudios(&scene, mockSceneReader, mockStudioReader, nil)
assert.Nil(err) assert.Nil(err)
mockStudioReader.AssertExpectations(t) mockStudioReader.AssertExpectations(t)
@@ -238,6 +239,7 @@ func TestSceneStudios(t *testing.T) {
mockStudioReader := &mocks.StudioReaderWriter{} mockStudioReader := &mocks.StudioReaderWriter{}
mockSceneReader := &mocks.SceneReaderWriter{} mockSceneReader := &mocks.SceneReaderWriter{}
mockStudioReader.On("Query", mock.Anything, mock.Anything).Return(nil, 0, nil)
mockStudioReader.On("QueryForAutoTag", mock.Anything).Return([]*models.Studio{&studio, &reversedStudio}, nil).Once() mockStudioReader.On("QueryForAutoTag", mock.Anything).Return([]*models.Studio{&studio, &reversedStudio}, nil).Once()
mockStudioReader.On("GetAliases", mock.Anything).Return([]string{}, nil).Maybe() mockStudioReader.On("GetAliases", mock.Anything).Return([]string{}, nil).Maybe()
@@ -252,6 +254,7 @@ func TestSceneStudios(t *testing.T) {
mockStudioReader := &mocks.StudioReaderWriter{} mockStudioReader := &mocks.StudioReaderWriter{}
mockSceneReader := &mocks.SceneReaderWriter{} mockSceneReader := &mocks.SceneReaderWriter{}
mockStudioReader.On("Query", mock.Anything, mock.Anything).Return(nil, 0, nil)
mockStudioReader.On("QueryForAutoTag", mock.Anything).Return([]*models.Studio{&studio, &reversedStudio}, nil).Once() mockStudioReader.On("QueryForAutoTag", mock.Anything).Return([]*models.Studio{&studio, &reversedStudio}, nil).Once()
mockStudioReader.On("GetAliases", studioID).Return([]string{ mockStudioReader.On("GetAliases", studioID).Return([]string{
studioName, studioName,
@@ -294,7 +297,7 @@ func TestSceneTags(t *testing.T) {
ID: sceneID, ID: sceneID,
Path: test.Path, Path: test.Path,
} }
err := SceneTags(&scene, mockSceneReader, mockTagReader) err := SceneTags(&scene, mockSceneReader, mockTagReader, nil)
assert.Nil(err) assert.Nil(err)
mockTagReader.AssertExpectations(t) mockTagReader.AssertExpectations(t)
@@ -305,6 +308,7 @@ func TestSceneTags(t *testing.T) {
mockTagReader := &mocks.TagReaderWriter{} mockTagReader := &mocks.TagReaderWriter{}
mockSceneReader := &mocks.SceneReaderWriter{} mockSceneReader := &mocks.SceneReaderWriter{}
mockTagReader.On("Query", mock.Anything, mock.Anything).Return(nil, 0, nil)
mockTagReader.On("QueryForAutoTag", mock.Anything).Return([]*models.Tag{&tag, &reversedTag}, nil).Once() mockTagReader.On("QueryForAutoTag", mock.Anything).Return([]*models.Tag{&tag, &reversedTag}, nil).Once()
mockTagReader.On("GetAliases", mock.Anything).Return([]string{}, nil).Maybe() mockTagReader.On("GetAliases", mock.Anything).Return([]string{}, nil).Maybe()
@@ -319,6 +323,7 @@ func TestSceneTags(t *testing.T) {
mockTagReader := &mocks.TagReaderWriter{} mockTagReader := &mocks.TagReaderWriter{}
mockSceneReader := &mocks.SceneReaderWriter{} mockSceneReader := &mocks.SceneReaderWriter{}
mockTagReader.On("Query", mock.Anything, mock.Anything).Return(nil, 0, nil)
mockTagReader.On("QueryForAutoTag", mock.Anything).Return([]*models.Tag{&tag, &reversedTag}, nil).Once() mockTagReader.On("QueryForAutoTag", mock.Anything).Return([]*models.Tag{&tag, &reversedTag}, nil).Once()
mockTagReader.On("GetAliases", tagID).Return([]string{ mockTagReader.On("GetAliases", tagID).Return([]string{
tagName, tagName,

View File

@@ -3,6 +3,7 @@ package autotag
import ( import (
"database/sql" "database/sql"
"github.com/stashapp/stash/pkg/match"
"github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/models"
) )
@@ -78,11 +79,12 @@ func addGalleryStudio(galleryWriter models.GalleryReaderWriter, galleryID, studi
return true, nil return true, nil
} }
func getStudioTagger(p *models.Studio, aliases []string) []tagger { func getStudioTagger(p *models.Studio, aliases []string, cache *match.Cache) []tagger {
ret := []tagger{{ ret := []tagger{{
ID: p.ID, ID: p.ID,
Type: "studio", Type: "studio",
Name: p.Name.String, Name: p.Name.String,
cache: cache,
}} }}
for _, a := range aliases { for _, a := range aliases {
@@ -97,8 +99,8 @@ func getStudioTagger(p *models.Studio, aliases []string) []tagger {
} }
// StudioScenes searches for scenes whose path matches the provided studio name and tags the scene with the studio, if studio is not already set on the scene. // StudioScenes searches for scenes whose path matches the provided studio name and tags the scene with the studio, if studio is not already set on the scene.
func StudioScenes(p *models.Studio, paths []string, aliases []string, rw models.SceneReaderWriter) error { func StudioScenes(p *models.Studio, paths []string, aliases []string, rw models.SceneReaderWriter, cache *match.Cache) error {
t := getStudioTagger(p, aliases) t := getStudioTagger(p, aliases, cache)
for _, tt := range t { for _, tt := range t {
if err := tt.tagScenes(paths, rw, func(subjectID, otherID int) (bool, error) { if err := tt.tagScenes(paths, rw, func(subjectID, otherID int) (bool, error) {
@@ -112,8 +114,8 @@ func StudioScenes(p *models.Studio, paths []string, aliases []string, rw models.
} }
// StudioImages searches for images whose path matches the provided studio name and tags the image with the studio, if studio is not already set on the image. // StudioImages searches for images whose path matches the provided studio name and tags the image with the studio, if studio is not already set on the image.
func StudioImages(p *models.Studio, paths []string, aliases []string, rw models.ImageReaderWriter) error { func StudioImages(p *models.Studio, paths []string, aliases []string, rw models.ImageReaderWriter, cache *match.Cache) error {
t := getStudioTagger(p, aliases) t := getStudioTagger(p, aliases, cache)
for _, tt := range t { for _, tt := range t {
if err := tt.tagImages(paths, rw, func(subjectID, otherID int) (bool, error) { if err := tt.tagImages(paths, rw, func(subjectID, otherID int) (bool, error) {
@@ -127,8 +129,8 @@ func StudioImages(p *models.Studio, paths []string, aliases []string, rw models.
} }
// StudioGalleries searches for galleries whose path matches the provided studio name and tags the gallery with the studio, if studio is not already set on the gallery. // StudioGalleries searches for galleries whose path matches the provided studio name and tags the gallery with the studio, if studio is not already set on the gallery.
func StudioGalleries(p *models.Studio, paths []string, aliases []string, rw models.GalleryReaderWriter) error { func StudioGalleries(p *models.Studio, paths []string, aliases []string, rw models.GalleryReaderWriter, cache *match.Cache) error {
t := getStudioTagger(p, aliases) t := getStudioTagger(p, aliases, cache)
for _, tt := range t { for _, tt := range t {
if err := tt.tagGalleries(paths, rw, func(subjectID, otherID int) (bool, error) { if err := tt.tagGalleries(paths, rw, func(subjectID, otherID int) (bool, error) {

View File

@@ -20,39 +20,39 @@ type testStudioCase struct {
var testStudioCases = []testStudioCase{ var testStudioCases = []testStudioCase{
{ {
"studio name", "studio name",
`(?i)(?:^|_|[^\w\d])studio[.\-_ ]*name(?:$|_|[^\w\d])`, `(?i)(?:^|_|[^\p{L}\d])studio[.\-_ ]*name(?:$|_|[^\p{L}\d])`,
"", "",
"", "",
}, },
{ {
"studio + name", "studio + name",
`(?i)(?:^|_|[^\w\d])studio[.\-_ ]*\+[.\-_ ]*name(?:$|_|[^\w\d])`, `(?i)(?:^|_|[^\p{L}\d])studio[.\-_ ]*\+[.\-_ ]*name(?:$|_|[^\p{L}\d])`,
"", "",
"", "",
}, },
{ {
`studio + name\`, `studio + name\`,
`(?i)(?:^|_|[^\w\d])studio[.\-_ ]*\+[.\-_ ]*name\\(?:$|_|[^\w\d])`, `(?i)(?:^|_|[^\p{L}\d])studio[.\-_ ]*\+[.\-_ ]*name\\(?:$|_|[^\p{L}\d])`,
"", "",
"", "",
}, },
{ {
"studio name", "studio name",
`(?i)(?:^|_|[^\w\d])studio[.\-_ ]*name(?:$|_|[^\w\d])`, `(?i)(?:^|_|[^\p{L}\d])studio[.\-_ ]*name(?:$|_|[^\p{L}\d])`,
"alias name", "alias name",
`(?i)(?:^|_|[^\w\d])alias[.\-_ ]*name(?:$|_|[^\w\d])`, `(?i)(?:^|_|[^\p{L}\d])alias[.\-_ ]*name(?:$|_|[^\p{L}\d])`,
}, },
{ {
"studio + name", "studio + name",
`(?i)(?:^|_|[^\w\d])studio[.\-_ ]*\+[.\-_ ]*name(?:$|_|[^\w\d])`, `(?i)(?:^|_|[^\p{L}\d])studio[.\-_ ]*\+[.\-_ ]*name(?:$|_|[^\p{L}\d])`,
"alias + name", "alias + name",
`(?i)(?:^|_|[^\w\d])alias[.\-_ ]*\+[.\-_ ]*name(?:$|_|[^\w\d])`, `(?i)(?:^|_|[^\p{L}\d])alias[.\-_ ]*\+[.\-_ ]*name(?:$|_|[^\p{L}\d])`,
}, },
{ {
`studio + name\`, `studio + name\`,
`(?i)(?:^|_|[^\w\d])studio[.\-_ ]*\+[.\-_ ]*name\\(?:$|_|[^\w\d])`, `(?i)(?:^|_|[^\p{L}\d])studio[.\-_ ]*\+[.\-_ ]*name\\(?:$|_|[^\p{L}\d])`,
`alias + name\`, `alias + name\`,
`(?i)(?:^|_|[^\w\d])alias[.\-_ ]*\+[.\-_ ]*name\\(?:$|_|[^\w\d])`, `(?i)(?:^|_|[^\p{L}\d])alias[.\-_ ]*\+[.\-_ ]*name\\(?:$|_|[^\p{L}\d])`,
}, },
} }
@@ -142,7 +142,7 @@ func testStudioScenes(t *testing.T, tc testStudioCase) {
}).Return(nil, nil).Once() }).Return(nil, nil).Once()
} }
err := StudioScenes(&studio, nil, aliases, mockSceneReader) err := StudioScenes(&studio, nil, aliases, mockSceneReader, nil)
assert := assert.New(t) assert := assert.New(t)
@@ -234,7 +234,7 @@ func testStudioImages(t *testing.T, tc testStudioCase) {
}).Return(nil, nil).Once() }).Return(nil, nil).Once()
} }
err := StudioImages(&studio, nil, aliases, mockImageReader) err := StudioImages(&studio, nil, aliases, mockImageReader, nil)
assert := assert.New(t) assert := assert.New(t)
@@ -324,7 +324,7 @@ func testStudioGalleries(t *testing.T, tc testStudioCase) {
}).Return(nil, nil).Once() }).Return(nil, nil).Once()
} }
err := StudioGalleries(&studio, nil, aliases, mockGalleryReader) err := StudioGalleries(&studio, nil, aliases, mockGalleryReader, nil)
assert := assert.New(t) assert := assert.New(t)

View File

@@ -3,22 +3,25 @@ package autotag
import ( import (
"github.com/stashapp/stash/pkg/gallery" "github.com/stashapp/stash/pkg/gallery"
"github.com/stashapp/stash/pkg/image" "github.com/stashapp/stash/pkg/image"
"github.com/stashapp/stash/pkg/match"
"github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/models"
"github.com/stashapp/stash/pkg/scene" "github.com/stashapp/stash/pkg/scene"
) )
func getTagTaggers(p *models.Tag, aliases []string) []tagger { func getTagTaggers(p *models.Tag, aliases []string, cache *match.Cache) []tagger {
ret := []tagger{{ ret := []tagger{{
ID: p.ID, ID: p.ID,
Type: "tag", Type: "tag",
Name: p.Name, Name: p.Name,
cache: cache,
}} }}
for _, a := range aliases { for _, a := range aliases {
ret = append(ret, tagger{ ret = append(ret, tagger{
ID: p.ID, ID: p.ID,
Type: "tag", Type: "tag",
Name: a, Name: a,
cache: cache,
}) })
} }
@@ -26,8 +29,8 @@ func getTagTaggers(p *models.Tag, aliases []string) []tagger {
} }
// TagScenes searches for scenes whose path matches the provided tag name and tags the scene with the tag. // TagScenes searches for scenes whose path matches the provided tag name and tags the scene with the tag.
func TagScenes(p *models.Tag, paths []string, aliases []string, rw models.SceneReaderWriter) error { func TagScenes(p *models.Tag, paths []string, aliases []string, rw models.SceneReaderWriter, cache *match.Cache) error {
t := getTagTaggers(p, aliases) t := getTagTaggers(p, aliases, cache)
for _, tt := range t { for _, tt := range t {
if err := tt.tagScenes(paths, rw, func(subjectID, otherID int) (bool, error) { if err := tt.tagScenes(paths, rw, func(subjectID, otherID int) (bool, error) {
@@ -40,8 +43,8 @@ func TagScenes(p *models.Tag, paths []string, aliases []string, rw models.SceneR
} }
// TagImages searches for images whose path matches the provided tag name and tags the image with the tag. // TagImages searches for images whose path matches the provided tag name and tags the image with the tag.
func TagImages(p *models.Tag, paths []string, aliases []string, rw models.ImageReaderWriter) error { func TagImages(p *models.Tag, paths []string, aliases []string, rw models.ImageReaderWriter, cache *match.Cache) error {
t := getTagTaggers(p, aliases) t := getTagTaggers(p, aliases, cache)
for _, tt := range t { for _, tt := range t {
if err := tt.tagImages(paths, rw, func(subjectID, otherID int) (bool, error) { if err := tt.tagImages(paths, rw, func(subjectID, otherID int) (bool, error) {
@@ -54,8 +57,8 @@ func TagImages(p *models.Tag, paths []string, aliases []string, rw models.ImageR
} }
// TagGalleries searches for galleries whose path matches the provided tag name and tags the gallery with the tag. // TagGalleries searches for galleries whose path matches the provided tag name and tags the gallery with the tag.
func TagGalleries(p *models.Tag, paths []string, aliases []string, rw models.GalleryReaderWriter) error { func TagGalleries(p *models.Tag, paths []string, aliases []string, rw models.GalleryReaderWriter, cache *match.Cache) error {
t := getTagTaggers(p, aliases) t := getTagTaggers(p, aliases, cache)
for _, tt := range t { for _, tt := range t {
if err := tt.tagGalleries(paths, rw, func(subjectID, otherID int) (bool, error) { if err := tt.tagGalleries(paths, rw, func(subjectID, otherID int) (bool, error) {

View File

@@ -20,39 +20,39 @@ type testTagCase struct {
var testTagCases = []testTagCase{ var testTagCases = []testTagCase{
{ {
"tag name", "tag name",
`(?i)(?:^|_|[^\w\d])tag[.\-_ ]*name(?:$|_|[^\w\d])`, `(?i)(?:^|_|[^\p{L}\d])tag[.\-_ ]*name(?:$|_|[^\p{L}\d])`,
"", "",
"", "",
}, },
{ {
"tag + name", "tag + name",
`(?i)(?:^|_|[^\w\d])tag[.\-_ ]*\+[.\-_ ]*name(?:$|_|[^\w\d])`, `(?i)(?:^|_|[^\p{L}\d])tag[.\-_ ]*\+[.\-_ ]*name(?:$|_|[^\p{L}\d])`,
"", "",
"", "",
}, },
{ {
`tag + name\`, `tag + name\`,
`(?i)(?:^|_|[^\w\d])tag[.\-_ ]*\+[.\-_ ]*name\\(?:$|_|[^\w\d])`, `(?i)(?:^|_|[^\p{L}\d])tag[.\-_ ]*\+[.\-_ ]*name\\(?:$|_|[^\p{L}\d])`,
"", "",
"", "",
}, },
{ {
"tag name", "tag name",
`(?i)(?:^|_|[^\w\d])tag[.\-_ ]*name(?:$|_|[^\w\d])`, `(?i)(?:^|_|[^\p{L}\d])tag[.\-_ ]*name(?:$|_|[^\p{L}\d])`,
"alias name", "alias name",
`(?i)(?:^|_|[^\w\d])alias[.\-_ ]*name(?:$|_|[^\w\d])`, `(?i)(?:^|_|[^\p{L}\d])alias[.\-_ ]*name(?:$|_|[^\p{L}\d])`,
}, },
{ {
"tag + name", "tag + name",
`(?i)(?:^|_|[^\w\d])tag[.\-_ ]*\+[.\-_ ]*name(?:$|_|[^\w\d])`, `(?i)(?:^|_|[^\p{L}\d])tag[.\-_ ]*\+[.\-_ ]*name(?:$|_|[^\p{L}\d])`,
"alias + name", "alias + name",
`(?i)(?:^|_|[^\w\d])alias[.\-_ ]*\+[.\-_ ]*name(?:$|_|[^\w\d])`, `(?i)(?:^|_|[^\p{L}\d])alias[.\-_ ]*\+[.\-_ ]*name(?:$|_|[^\p{L}\d])`,
}, },
{ {
`tag + name\`, `tag + name\`,
`(?i)(?:^|_|[^\w\d])tag[.\-_ ]*\+[.\-_ ]*name\\(?:$|_|[^\w\d])`, `(?i)(?:^|_|[^\p{L}\d])tag[.\-_ ]*\+[.\-_ ]*name\\(?:$|_|[^\p{L}\d])`,
`alias + name\`, `alias + name\`,
`(?i)(?:^|_|[^\w\d])alias[.\-_ ]*\+[.\-_ ]*name\\(?:$|_|[^\w\d])`, `(?i)(?:^|_|[^\p{L}\d])alias[.\-_ ]*\+[.\-_ ]*name\\(?:$|_|[^\p{L}\d])`,
}, },
} }
@@ -137,7 +137,7 @@ func testTagScenes(t *testing.T, tc testTagCase) {
mockSceneReader.On("UpdateTags", sceneID, []int{tagID}).Return(nil).Once() mockSceneReader.On("UpdateTags", sceneID, []int{tagID}).Return(nil).Once()
} }
err := TagScenes(&tag, nil, aliases, mockSceneReader) err := TagScenes(&tag, nil, aliases, mockSceneReader, nil)
assert := assert.New(t) assert := assert.New(t)
@@ -225,7 +225,7 @@ func testTagImages(t *testing.T, tc testTagCase) {
mockImageReader.On("UpdateTags", imageID, []int{tagID}).Return(nil).Once() mockImageReader.On("UpdateTags", imageID, []int{tagID}).Return(nil).Once()
} }
err := TagImages(&tag, nil, aliases, mockImageReader) err := TagImages(&tag, nil, aliases, mockImageReader, nil)
assert := assert.New(t) assert := assert.New(t)
@@ -312,7 +312,7 @@ func testTagGalleries(t *testing.T, tc testTagCase) {
mockGalleryReader.On("UpdateTags", galleryID, []int{tagID}).Return(nil).Once() mockGalleryReader.On("UpdateTags", galleryID, []int{tagID}).Return(nil).Once()
} }
err := TagGalleries(&tag, nil, aliases, mockGalleryReader) err := TagGalleries(&tag, nil, aliases, mockGalleryReader, nil)
assert := assert.New(t) assert := assert.New(t)

View File

@@ -26,6 +26,8 @@ type tagger struct {
Type string Type string
Name string Name string
Path string Path string
cache *match.Cache
} }
type addLinkFunc func(subjectID, otherID int) (bool, error) type addLinkFunc func(subjectID, otherID int) (bool, error)
@@ -39,7 +41,7 @@ func (t *tagger) addLog(otherType, otherName string) {
} }
func (t *tagger) tagPerformers(performerReader models.PerformerReader, addFunc addLinkFunc) error { func (t *tagger) tagPerformers(performerReader models.PerformerReader, addFunc addLinkFunc) error {
others, err := match.PathToPerformers(t.Path, performerReader) others, err := match.PathToPerformers(t.Path, performerReader, t.cache)
if err != nil { if err != nil {
return err return err
} }
@@ -60,7 +62,7 @@ func (t *tagger) tagPerformers(performerReader models.PerformerReader, addFunc a
} }
func (t *tagger) tagStudios(studioReader models.StudioReader, addFunc addLinkFunc) error { func (t *tagger) tagStudios(studioReader models.StudioReader, addFunc addLinkFunc) error {
studio, err := match.PathToStudio(t.Path, studioReader) studio, err := match.PathToStudio(t.Path, studioReader, t.cache)
if err != nil { if err != nil {
return err return err
} }
@@ -81,7 +83,7 @@ func (t *tagger) tagStudios(studioReader models.StudioReader, addFunc addLinkFun
} }
func (t *tagger) tagTags(tagReader models.TagReader, addFunc addLinkFunc) error { func (t *tagger) tagTags(tagReader models.TagReader, addFunc addLinkFunc) error {
others, err := match.PathToTags(t.Path, tagReader) others, err := match.PathToTags(t.Path, tagReader, t.cache)
if err != nil { if err != nil {
return err return err
} }

View File

@@ -1,15 +1,10 @@
package database package database
import ( import (
"regexp"
"strconv" "strconv"
"strings" "strings"
) )
func regexFn(re, s string) (bool, error) {
return regexp.MatchString(re, s)
}
func durationToTinyIntFn(str string) (int64, error) { func durationToTinyIntFn(str string) (int64, error) {
splits := strings.Split(str, ":") splits := strings.Split(str, ":")

42
pkg/database/regex.go Normal file
View File

@@ -0,0 +1,42 @@
package database
import (
"regexp"
lru "github.com/hashicorp/golang-lru"
)
// size of the regex LRU cache in elements.
// A small number number was chosen because it's most likely use is for a
// single query - this function gets called for every row in the (filtered)
// results. It's likely to only need no more than 1 or 2 in any given query.
// After that point, it's just sitting in the cache and is unlikely to be used
// again.
const regexCacheSize = 10
var regexCache *lru.Cache
func init() {
regexCache, _ = lru.New(regexCacheSize)
}
// regexFn is registered as an SQLite function as "regexp"
// It uses an LRU cache to cache recent regex patterns to reduce CPU load over
// identical patterns.
func regexFn(re, s string) (bool, error) {
entry, ok := regexCache.Get(re)
var compiled *regexp.Regexp
if !ok {
var err error
compiled, err = regexp.Compile(re)
if err != nil {
return false, err
}
regexCache.Add(re, compiled)
} else {
compiled = entry.(*regexp.Regexp)
}
return compiled.MatchString(s), nil
}

View File

@@ -7,11 +7,13 @@ import (
"strconv" "strconv"
"strings" "strings"
"sync" "sync"
"time"
"github.com/stashapp/stash/pkg/autotag" "github.com/stashapp/stash/pkg/autotag"
"github.com/stashapp/stash/pkg/image" "github.com/stashapp/stash/pkg/image"
"github.com/stashapp/stash/pkg/job" "github.com/stashapp/stash/pkg/job"
"github.com/stashapp/stash/pkg/logger" "github.com/stashapp/stash/pkg/logger"
"github.com/stashapp/stash/pkg/match"
"github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/models"
"github.com/stashapp/stash/pkg/scene" "github.com/stashapp/stash/pkg/scene"
) )
@@ -19,9 +21,13 @@ import (
type autoTagJob struct { type autoTagJob struct {
txnManager models.TransactionManager txnManager models.TransactionManager
input models.AutoTagMetadataInput input models.AutoTagMetadataInput
cache match.Cache
} }
func (j *autoTagJob) Execute(ctx context.Context, progress *job.Progress) { func (j *autoTagJob) Execute(ctx context.Context, progress *job.Progress) {
begin := time.Now()
input := j.input input := j.input
if j.isFileBasedAutoTag(input) { if j.isFileBasedAutoTag(input) {
// doing file-based auto-tag // doing file-based auto-tag
@@ -30,6 +36,8 @@ func (j *autoTagJob) Execute(ctx context.Context, progress *job.Progress) {
// doing specific performer/studio/tag auto-tag // doing specific performer/studio/tag auto-tag
j.autoTagSpecific(ctx, progress) j.autoTagSpecific(ctx, progress)
} }
logger.Infof("Finished autotag after %s", time.Since(begin).String())
} }
func (j *autoTagJob) isFileBasedAutoTag(input models.AutoTagMetadataInput) bool { func (j *autoTagJob) isFileBasedAutoTag(input models.AutoTagMetadataInput) bool {
@@ -50,6 +58,7 @@ func (j *autoTagJob) autoTagFiles(ctx context.Context, progress *job.Progress, p
ctx: ctx, ctx: ctx,
progress: progress, progress: progress,
txnManager: j.txnManager, txnManager: j.txnManager,
cache: &j.cache,
} }
t.process() t.process()
@@ -105,8 +114,6 @@ func (j *autoTagJob) autoTagSpecific(ctx context.Context, progress *job.Progress
j.autoTagPerformers(ctx, progress, input.Paths, performerIds) j.autoTagPerformers(ctx, progress, input.Paths, performerIds)
j.autoTagStudios(ctx, progress, input.Paths, studioIds) j.autoTagStudios(ctx, progress, input.Paths, studioIds)
j.autoTagTags(ctx, progress, input.Paths, tagIds) j.autoTagTags(ctx, progress, input.Paths, tagIds)
logger.Info("Finished autotag")
} }
func (j *autoTagJob) autoTagPerformers(ctx context.Context, progress *job.Progress, paths []string, performerIds []string) { func (j *autoTagJob) autoTagPerformers(ctx context.Context, progress *job.Progress, paths []string, performerIds []string) {
@@ -150,13 +157,13 @@ func (j *autoTagJob) autoTagPerformers(ctx context.Context, progress *job.Progre
} }
if err := j.txnManager.WithTxn(context.TODO(), func(r models.Repository) error { if err := j.txnManager.WithTxn(context.TODO(), func(r models.Repository) error {
if err := autotag.PerformerScenes(performer, paths, r.Scene()); err != nil { if err := autotag.PerformerScenes(performer, paths, r.Scene(), &j.cache); err != nil {
return err return err
} }
if err := autotag.PerformerImages(performer, paths, r.Image()); err != nil { if err := autotag.PerformerImages(performer, paths, r.Image(), &j.cache); err != nil {
return err return err
} }
if err := autotag.PerformerGalleries(performer, paths, r.Gallery()); err != nil { if err := autotag.PerformerGalleries(performer, paths, r.Gallery(), &j.cache); err != nil {
return err return err
} }
@@ -222,13 +229,13 @@ func (j *autoTagJob) autoTagStudios(ctx context.Context, progress *job.Progress,
return err return err
} }
if err := autotag.StudioScenes(studio, paths, aliases, r.Scene()); err != nil { if err := autotag.StudioScenes(studio, paths, aliases, r.Scene(), &j.cache); err != nil {
return err return err
} }
if err := autotag.StudioImages(studio, paths, aliases, r.Image()); err != nil { if err := autotag.StudioImages(studio, paths, aliases, r.Image(), &j.cache); err != nil {
return err return err
} }
if err := autotag.StudioGalleries(studio, paths, aliases, r.Gallery()); err != nil { if err := autotag.StudioGalleries(studio, paths, aliases, r.Gallery(), &j.cache); err != nil {
return err return err
} }
@@ -288,13 +295,13 @@ func (j *autoTagJob) autoTagTags(ctx context.Context, progress *job.Progress, pa
return err return err
} }
if err := autotag.TagScenes(tag, paths, aliases, r.Scene()); err != nil { if err := autotag.TagScenes(tag, paths, aliases, r.Scene(), &j.cache); err != nil {
return err return err
} }
if err := autotag.TagImages(tag, paths, aliases, r.Image()); err != nil { if err := autotag.TagImages(tag, paths, aliases, r.Image(), &j.cache); err != nil {
return err return err
} }
if err := autotag.TagGalleries(tag, paths, aliases, r.Gallery()); err != nil { if err := autotag.TagGalleries(tag, paths, aliases, r.Gallery(), &j.cache); err != nil {
return err return err
} }
@@ -323,6 +330,7 @@ type autoTagFilesTask struct {
ctx context.Context ctx context.Context
progress *job.Progress progress *job.Progress
txnManager models.TransactionManager txnManager models.TransactionManager
cache *match.Cache
} }
func (t *autoTagFilesTask) makeSceneFilter() *models.SceneFilterType { func (t *autoTagFilesTask) makeSceneFilter() *models.SceneFilterType {
@@ -469,6 +477,7 @@ func (t *autoTagFilesTask) processScenes(r models.ReaderRepository) error {
performers: t.performers, performers: t.performers,
studios: t.studios, studios: t.studios,
tags: t.tags, tags: t.tags,
cache: t.cache,
} }
var wg sync.WaitGroup var wg sync.WaitGroup
@@ -483,6 +492,10 @@ func (t *autoTagFilesTask) processScenes(r models.ReaderRepository) error {
more = false more = false
} else { } else {
*findFilter.Page++ *findFilter.Page++
if *findFilter.Page%10 == 1 {
logger.Infof("Processed %d scenes...", (*findFilter.Page-1)*batchSize)
}
} }
} }
@@ -517,6 +530,7 @@ func (t *autoTagFilesTask) processImages(r models.ReaderRepository) error {
performers: t.performers, performers: t.performers,
studios: t.studios, studios: t.studios,
tags: t.tags, tags: t.tags,
cache: t.cache,
} }
var wg sync.WaitGroup var wg sync.WaitGroup
@@ -531,6 +545,10 @@ func (t *autoTagFilesTask) processImages(r models.ReaderRepository) error {
more = false more = false
} else { } else {
*findFilter.Page++ *findFilter.Page++
if *findFilter.Page%10 == 1 {
logger.Infof("Processed %d images...", (*findFilter.Page-1)*batchSize)
}
} }
} }
@@ -565,6 +583,7 @@ func (t *autoTagFilesTask) processGalleries(r models.ReaderRepository) error {
performers: t.performers, performers: t.performers,
studios: t.studios, studios: t.studios,
tags: t.tags, tags: t.tags,
cache: t.cache,
} }
var wg sync.WaitGroup var wg sync.WaitGroup
@@ -579,6 +598,10 @@ func (t *autoTagFilesTask) processGalleries(r models.ReaderRepository) error {
more = false more = false
} else { } else {
*findFilter.Page++ *findFilter.Page++
if *findFilter.Page%10 == 1 {
logger.Infof("Processed %d galleries...", (*findFilter.Page-1)*batchSize)
}
} }
} }
@@ -596,14 +619,17 @@ func (t *autoTagFilesTask) process() {
logger.Infof("Starting autotag of %d files", total) logger.Infof("Starting autotag of %d files", total)
logger.Info("Autotagging scenes...")
if err := t.processScenes(r); err != nil { if err := t.processScenes(r); err != nil {
return err return err
} }
logger.Info("Autotagging images...")
if err := t.processImages(r); err != nil { if err := t.processImages(r); err != nil {
return err return err
} }
logger.Info("Autotagging galleries...")
if err := t.processGalleries(r); err != nil { if err := t.processGalleries(r); err != nil {
return err return err
} }
@@ -616,8 +642,6 @@ func (t *autoTagFilesTask) process() {
}); err != nil { }); err != nil {
logger.Error(err.Error()) logger.Error(err.Error())
} }
logger.Info("Finished autotag")
} }
type autoTagSceneTask struct { type autoTagSceneTask struct {
@@ -627,23 +651,25 @@ type autoTagSceneTask struct {
performers bool performers bool
studios bool studios bool
tags bool tags bool
cache *match.Cache
} }
func (t *autoTagSceneTask) Start(wg *sync.WaitGroup) { func (t *autoTagSceneTask) Start(wg *sync.WaitGroup) {
defer wg.Done() defer wg.Done()
if err := t.txnManager.WithTxn(context.TODO(), func(r models.Repository) error { if err := t.txnManager.WithTxn(context.TODO(), func(r models.Repository) error {
if t.performers { if t.performers {
if err := autotag.ScenePerformers(t.scene, r.Scene(), r.Performer()); err != nil { if err := autotag.ScenePerformers(t.scene, r.Scene(), r.Performer(), t.cache); err != nil {
return fmt.Errorf("error tagging scene performers for %s: %v", t.scene.Path, err) return fmt.Errorf("error tagging scene performers for %s: %v", t.scene.Path, err)
} }
} }
if t.studios { if t.studios {
if err := autotag.SceneStudios(t.scene, r.Scene(), r.Studio()); err != nil { if err := autotag.SceneStudios(t.scene, r.Scene(), r.Studio(), t.cache); err != nil {
return fmt.Errorf("error tagging scene studio for %s: %v", t.scene.Path, err) return fmt.Errorf("error tagging scene studio for %s: %v", t.scene.Path, err)
} }
} }
if t.tags { if t.tags {
if err := autotag.SceneTags(t.scene, r.Scene(), r.Tag()); err != nil { if err := autotag.SceneTags(t.scene, r.Scene(), r.Tag(), t.cache); err != nil {
return fmt.Errorf("error tagging scene tags for %s: %v", t.scene.Path, err) return fmt.Errorf("error tagging scene tags for %s: %v", t.scene.Path, err)
} }
} }
@@ -661,23 +687,25 @@ type autoTagImageTask struct {
performers bool performers bool
studios bool studios bool
tags bool tags bool
cache *match.Cache
} }
func (t *autoTagImageTask) Start(wg *sync.WaitGroup) { func (t *autoTagImageTask) Start(wg *sync.WaitGroup) {
defer wg.Done() defer wg.Done()
if err := t.txnManager.WithTxn(context.TODO(), func(r models.Repository) error { if err := t.txnManager.WithTxn(context.TODO(), func(r models.Repository) error {
if t.performers { if t.performers {
if err := autotag.ImagePerformers(t.image, r.Image(), r.Performer()); err != nil { if err := autotag.ImagePerformers(t.image, r.Image(), r.Performer(), t.cache); err != nil {
return fmt.Errorf("error tagging image performers for %s: %v", t.image.Path, err) return fmt.Errorf("error tagging image performers for %s: %v", t.image.Path, err)
} }
} }
if t.studios { if t.studios {
if err := autotag.ImageStudios(t.image, r.Image(), r.Studio()); err != nil { if err := autotag.ImageStudios(t.image, r.Image(), r.Studio(), t.cache); err != nil {
return fmt.Errorf("error tagging image studio for %s: %v", t.image.Path, err) return fmt.Errorf("error tagging image studio for %s: %v", t.image.Path, err)
} }
} }
if t.tags { if t.tags {
if err := autotag.ImageTags(t.image, r.Image(), r.Tag()); err != nil { if err := autotag.ImageTags(t.image, r.Image(), r.Tag(), t.cache); err != nil {
return fmt.Errorf("error tagging image tags for %s: %v", t.image.Path, err) return fmt.Errorf("error tagging image tags for %s: %v", t.image.Path, err)
} }
} }
@@ -695,23 +723,25 @@ type autoTagGalleryTask struct {
performers bool performers bool
studios bool studios bool
tags bool tags bool
cache *match.Cache
} }
func (t *autoTagGalleryTask) Start(wg *sync.WaitGroup) { func (t *autoTagGalleryTask) Start(wg *sync.WaitGroup) {
defer wg.Done() defer wg.Done()
if err := t.txnManager.WithTxn(context.TODO(), func(r models.Repository) error { if err := t.txnManager.WithTxn(context.TODO(), func(r models.Repository) error {
if t.performers { if t.performers {
if err := autotag.GalleryPerformers(t.gallery, r.Gallery(), r.Performer()); err != nil { if err := autotag.GalleryPerformers(t.gallery, r.Gallery(), r.Performer(), t.cache); err != nil {
return fmt.Errorf("error tagging gallery performers for %s: %v", t.gallery.Path.String, err) return fmt.Errorf("error tagging gallery performers for %s: %v", t.gallery.Path.String, err)
} }
} }
if t.studios { if t.studios {
if err := autotag.GalleryStudios(t.gallery, r.Gallery(), r.Studio()); err != nil { if err := autotag.GalleryStudios(t.gallery, r.Gallery(), r.Studio(), t.cache); err != nil {
return fmt.Errorf("error tagging gallery studio for %s: %v", t.gallery.Path.String, err) return fmt.Errorf("error tagging gallery studio for %s: %v", t.gallery.Path.String, err)
} }
} }
if t.tags { if t.tags {
if err := autotag.GalleryTags(t.gallery, r.Gallery(), r.Tag()); err != nil { if err := autotag.GalleryTags(t.gallery, r.Gallery(), r.Tag(), t.cache); err != nil {
return fmt.Errorf("error tagging gallery tags for %s: %v", t.gallery.Path.String, err) return fmt.Errorf("error tagging gallery tags for %s: %v", t.gallery.Path.String, err)
} }
} }

120
pkg/match/cache.go Normal file
View File

@@ -0,0 +1,120 @@
package match
import "github.com/stashapp/stash/pkg/models"
const singleFirstCharacterRegex = `^[\p{L}][.\-_ ]`
// Cache is used to cache queries that should not change across an autotag process.
type Cache struct {
singleCharPerformers []*models.Performer
singleCharStudios []*models.Studio
singleCharTags []*models.Tag
}
// getSingleLetterPerformers returns all performers with names that start with single character words.
// The autotag query splits the words into two-character words to query
// against. This means that performers with single-letter words in their names could potentially
// be missed.
// This query is expensive, so it's queried once and cached, if the cache if provided.
func getSingleLetterPerformers(c *Cache, reader models.PerformerReader) ([]*models.Performer, error) {
if c == nil {
c = &Cache{}
}
if c.singleCharPerformers == nil {
pp := -1
performers, _, err := reader.Query(&models.PerformerFilterType{
Name: &models.StringCriterionInput{
Value: singleFirstCharacterRegex,
Modifier: models.CriterionModifierMatchesRegex,
},
}, &models.FindFilterType{
PerPage: &pp,
})
if err != nil {
return nil, err
}
if len(performers) == 0 {
// make singleWordPerformers not nil
c.singleCharPerformers = make([]*models.Performer, 0)
} else {
c.singleCharPerformers = performers
}
}
return c.singleCharPerformers, nil
}
// getSingleLetterStudios returns all studios with names that start with single character words.
// See getSingleLetterPerformers for details.
func getSingleLetterStudios(c *Cache, reader models.StudioReader) ([]*models.Studio, error) {
if c == nil {
c = &Cache{}
}
if c.singleCharStudios == nil {
pp := -1
studios, _, err := reader.Query(&models.StudioFilterType{
Name: &models.StringCriterionInput{
Value: singleFirstCharacterRegex,
Modifier: models.CriterionModifierMatchesRegex,
},
}, &models.FindFilterType{
PerPage: &pp,
})
if err != nil {
return nil, err
}
if len(studios) == 0 {
// make singleWordStudios not nil
c.singleCharStudios = make([]*models.Studio, 0)
} else {
c.singleCharStudios = studios
}
}
return c.singleCharStudios, nil
}
// getSingleLetterTags returns all tags with names that start with single character words.
// See getSingleLetterPerformers for details.
func getSingleLetterTags(c *Cache, reader models.TagReader) ([]*models.Tag, error) {
if c == nil {
c = &Cache{}
}
if c.singleCharTags == nil {
pp := -1
tags, _, err := reader.Query(&models.TagFilterType{
Name: &models.StringCriterionInput{
Value: singleFirstCharacterRegex,
Modifier: models.CriterionModifierMatchesRegex,
},
Or: &models.TagFilterType{
Aliases: &models.StringCriterionInput{
Value: singleFirstCharacterRegex,
Modifier: models.CriterionModifierMatchesRegex,
},
},
}, &models.FindFilterType{
PerPage: &pp,
})
if err != nil {
return nil, err
}
if len(tags) == 0 {
// make singleWordTags not nil
c.singleCharTags = make([]*models.Tag, 0)
} else {
c.singleCharTags = tags
}
}
return c.singleCharTags, nil
}

View File

@@ -14,12 +14,15 @@ import (
) )
const ( const (
separatorChars = `.\-_ ` separatorChars = `.\-_ `
separatorPattern = `(?:_|[^\p{L}\w\d])+`
reNotLetterWordUnicode = `[^\p{L}\w\d]` reNotLetterWordUnicode = `[^\p{L}\w\d]`
reNotLetterWord = `[^\w\d]` reNotLetterWord = `[^\w\d]`
) )
var separatorRE = regexp.MustCompile(separatorPattern)
func getPathQueryRegex(name string) string { func getPathQueryRegex(name string) string {
// escape specific regex characters // escape specific regex characters
name = regexp.QuoteMeta(name) name = regexp.QuoteMeta(name)
@@ -29,13 +32,7 @@ func getPathQueryRegex(name string) string {
ret := strings.ReplaceAll(name, " ", separator+"*") ret := strings.ReplaceAll(name, " ", separator+"*")
// \p{L} is specifically omitted here because of the performance hit when ret = `(?:^|_|[^\p{L}\d])` + ret + `(?:$|_|[^\p{L}\d])`
// including it. It does mean that paths where the name is bounded by
// unicode letters will be returned. However, the results should be tested
// by nameMatchesPath which does include \p{L}. The improvement in query
// performance should be outweigh the performance hit of testing any extra
// results.
ret = `(?:^|_|[^\w\d])` + ret + `(?:$|_|[^\w\d])`
return ret return ret
} }
@@ -49,9 +46,7 @@ func getPathWords(path string) []string {
} }
// handle path separators // handle path separators
const separator = `(?:_|[^\p{L}\w\d])+` retStr = separatorRE.ReplaceAllString(retStr, " ")
re := regexp.MustCompile(separator)
retStr = re.ReplaceAllString(retStr, " ")
words := strings.Split(retStr, " ") words := strings.Split(retStr, " ")
@@ -132,10 +127,24 @@ func regexpMatchesPath(r *regexp.Regexp, path string) int {
return found[len(found)-1][0] return found[len(found)-1][0]
} }
func PathToPerformers(path string, performerReader models.PerformerReader) ([]*models.Performer, error) { func getPerformers(words []string, performerReader models.PerformerReader, cache *Cache) ([]*models.Performer, error) {
words := getPathWords(path)
performers, err := performerReader.QueryForAutoTag(words) performers, err := performerReader.QueryForAutoTag(words)
if err != nil {
return nil, err
}
swPerformers, err := getSingleLetterPerformers(cache, performerReader)
if err != nil {
return nil, err
}
return append(performers, swPerformers...), nil
}
func PathToPerformers(path string, reader models.PerformerReader, cache *Cache) ([]*models.Performer, error) {
words := getPathWords(path)
performers, err := getPerformers(words, reader, cache)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -151,12 +160,26 @@ func PathToPerformers(path string, performerReader models.PerformerReader) ([]*m
return ret, nil return ret, nil
} }
func getStudios(words []string, reader models.StudioReader, cache *Cache) ([]*models.Studio, error) {
studios, err := reader.QueryForAutoTag(words)
if err != nil {
return nil, err
}
swStudios, err := getSingleLetterStudios(cache, reader)
if err != nil {
return nil, err
}
return append(studios, swStudios...), nil
}
// PathToStudio returns the Studio that matches the given path. // PathToStudio returns the Studio that matches the given path.
// Where multiple matching studios are found, the one that matches the latest // Where multiple matching studios are found, the one that matches the latest
// position in the path is returned. // position in the path is returned.
func PathToStudio(path string, reader models.StudioReader) (*models.Studio, error) { func PathToStudio(path string, reader models.StudioReader, cache *Cache) (*models.Studio, error) {
words := getPathWords(path) words := getPathWords(path)
candidates, err := reader.QueryForAutoTag(words) candidates, err := getStudios(words, reader, cache)
if err != nil { if err != nil {
return nil, err return nil, err
@@ -188,9 +211,23 @@ func PathToStudio(path string, reader models.StudioReader) (*models.Studio, erro
return ret, nil return ret, nil
} }
func PathToTags(path string, tagReader models.TagReader) ([]*models.Tag, error) { func getTags(words []string, reader models.TagReader, cache *Cache) ([]*models.Tag, error) {
tags, err := reader.QueryForAutoTag(words)
if err != nil {
return nil, err
}
swTags, err := getSingleLetterTags(cache, reader)
if err != nil {
return nil, err
}
return append(tags, swTags...), nil
}
func PathToTags(path string, reader models.TagReader, cache *Cache) ([]*models.Tag, error) {
words := getPathWords(path) words := getPathWords(path)
tags, err := tagReader.QueryForAutoTag(words) tags, err := getTags(words, reader, cache)
if err != nil { if err != nil {
return nil, err return nil, err
@@ -204,7 +241,7 @@ func PathToTags(path string, tagReader models.TagReader) ([]*models.Tag, error)
} }
if !matches { if !matches {
aliases, err := tagReader.GetAliases(t.ID) aliases, err := reader.GetAliases(t.ID)
if err != nil { if err != nil {
return nil, err return nil, err
} }

View File

@@ -22,7 +22,7 @@ type autotagScraper struct {
} }
func autotagMatchPerformers(path string, performerReader models.PerformerReader) ([]*models.ScrapedPerformer, error) { func autotagMatchPerformers(path string, performerReader models.PerformerReader) ([]*models.ScrapedPerformer, error) {
p, err := match.PathToPerformers(path, performerReader) p, err := match.PathToPerformers(path, performerReader, nil)
if err != nil { if err != nil {
return nil, fmt.Errorf("error matching performers: %w", err) return nil, fmt.Errorf("error matching performers: %w", err)
} }
@@ -46,7 +46,7 @@ func autotagMatchPerformers(path string, performerReader models.PerformerReader)
} }
func autotagMatchStudio(path string, studioReader models.StudioReader) (*models.ScrapedStudio, error) { func autotagMatchStudio(path string, studioReader models.StudioReader) (*models.ScrapedStudio, error) {
studio, err := match.PathToStudio(path, studioReader) studio, err := match.PathToStudio(path, studioReader, nil)
if err != nil { if err != nil {
return nil, fmt.Errorf("error matching studios: %w", err) return nil, fmt.Errorf("error matching studios: %w", err)
} }
@@ -63,7 +63,7 @@ func autotagMatchStudio(path string, studioReader models.StudioReader) (*models.
} }
func autotagMatchTags(path string, tagReader models.TagReader) ([]*models.ScrapedTag, error) { func autotagMatchTags(path string, tagReader models.TagReader) ([]*models.ScrapedTag, error) {
t, err := match.PathToTags(path, tagReader) t, err := match.PathToTags(path, tagReader, nil)
if err != nil { if err != nil {
return nil, fmt.Errorf("error matching tags: %w", err) return nil, fmt.Errorf("error matching tags: %w", err)
} }

View File

@@ -486,16 +486,13 @@ func galleryAverageResolutionCriterionHandler(qb *galleryQueryBuilder, resolutio
} }
func (qb *galleryQueryBuilder) getGallerySort(findFilter *models.FindFilterType) string { func (qb *galleryQueryBuilder) getGallerySort(findFilter *models.FindFilterType) string {
var sort string if findFilter == nil || findFilter.Sort == nil || *findFilter.Sort == "" {
var direction string return ""
if findFilter == nil {
sort = "path"
direction = "ASC"
} else {
sort = findFilter.GetSort("path")
direction = findFilter.GetDirection()
} }
sort := findFilter.GetSort("path")
direction := findFilter.GetDirection()
switch sort { switch sort {
case "images_count": case "images_count":
return getCountSort(galleryTable, galleriesImagesTable, galleryIDColumn, direction) return getCountSort(galleryTable, galleriesImagesTable, galleryIDColumn, direction)

View File

@@ -517,8 +517,8 @@ INNER JOIN (` + valuesClause + `) t ON t.column2 = pt.tag_id
} }
func (qb *imageQueryBuilder) getImageSort(findFilter *models.FindFilterType) string { func (qb *imageQueryBuilder) getImageSort(findFilter *models.FindFilterType) string {
if findFilter == nil { if findFilter == nil || findFilter.Sort == nil || *findFilter.Sort == "" {
return " ORDER BY images.path ASC " return ""
} }
sort := findFilter.GetSort("title") sort := findFilter.GetSort("title")
direction := findFilter.GetDirection() direction := findFilter.GetDirection()

View File

@@ -21,12 +21,6 @@ WHERE performers_tags.tag_id = ?
GROUP BY performers_tags.performer_id GROUP BY performers_tags.performer_id
` `
// KNOWN ISSUE: using \p{L} to find single unicode character names results in
// very slow queries.
// Suggested solution will be to cache single-character names and not include it
// in the autotag query.
const singleFirstCharacterRegex = `^[\w][.\-_ ]`
type performerQueryBuilder struct { type performerQueryBuilder struct {
repository repository
} }
@@ -189,9 +183,6 @@ func (qb *performerQueryBuilder) QueryForAutoTag(words []string) ([]*models.Perf
var whereClauses []string var whereClauses []string
var args []interface{} var args []interface{}
whereClauses = append(whereClauses, "name regexp ?")
args = append(args, singleFirstCharacterRegex)
for _, w := range words { for _, w := range words {
whereClauses = append(whereClauses, "name like ?") whereClauses = append(whereClauses, "name like ?")
args = append(args, w+"%") args = append(args, w+"%")

View File

@@ -760,8 +760,7 @@ func (qb *sceneQueryBuilder) getDefaultSceneSort() string {
} }
func (qb *sceneQueryBuilder) setSceneSort(query *queryBuilder, findFilter *models.FindFilterType) { func (qb *sceneQueryBuilder) setSceneSort(query *queryBuilder, findFilter *models.FindFilterType) {
if findFilter == nil { if findFilter == nil || findFilter.Sort == nil || *findFilter.Sort == "" {
query.sortAndPagination += qb.getDefaultSceneSort()
return return
} }
sort := findFilter.GetSort("title") sort := findFilter.GetSort("title")

View File

@@ -144,10 +144,6 @@ func (qb *studioQueryBuilder) QueryForAutoTag(words []string) ([]*models.Studio,
var whereClauses []string var whereClauses []string
var args []interface{} var args []interface{}
// always include names that begin with a single character
whereClauses = append(whereClauses, "studios.name regexp ? OR COALESCE(studio_aliases.alias, '') regexp ?")
args = append(args, singleFirstCharacterRegex, singleFirstCharacterRegex)
for _, w := range words { for _, w := range words {
ww := w + "%" ww := w + "%"
whereClauses = append(whereClauses, "studios.name like ?") whereClauses = append(whereClauses, "studios.name like ?")

View File

@@ -235,10 +235,6 @@ func (qb *tagQueryBuilder) QueryForAutoTag(words []string) ([]*models.Tag, error
var whereClauses []string var whereClauses []string
var args []interface{} var args []interface{}
// always include names that begin with a single character
whereClauses = append(whereClauses, "tags.name regexp ? OR COALESCE(tag_aliases.alias, '') regexp ?")
args = append(args, singleFirstCharacterRegex, singleFirstCharacterRegex)
for _, w := range words { for _, w := range words {
ww := w + "%" ww := w + "%"
whereClauses = append(whereClauses, "tags.name like ?") whereClauses = append(whereClauses, "tags.name like ?")

View File

@@ -16,6 +16,7 @@ import V0100 from "./versions/v0100.md";
import V0110 from "./versions/v0110.md"; import V0110 from "./versions/v0110.md";
import V0120 from "./versions/v0120.md"; import V0120 from "./versions/v0120.md";
import V0130 from "./versions/v0130.md"; import V0130 from "./versions/v0130.md";
import V0140 from "./versions/v0140.md";
import { MarkdownPage } from "../Shared/MarkdownPage"; import { MarkdownPage } from "../Shared/MarkdownPage";
// to avoid use of explicit any // to avoid use of explicit any
@@ -56,7 +57,7 @@ const Changelog: React.FC = () => {
// then update the current fields. // then update the current fields.
const currentVersion = stashVersion || "v0.13.0"; const currentVersion = stashVersion || "v0.13.0";
const currentDate = buildDate; const currentDate = buildDate;
const currentPage = V0130; const currentPage = V0140;
const releases: IStashRelease[] = [ const releases: IStashRelease[] = [
{ {
@@ -65,6 +66,11 @@ const Changelog: React.FC = () => {
page: currentPage, page: currentPage,
defaultOpen: true, defaultOpen: true,
}, },
{
version: "v0.13.0",
date: "2021-03-08",
page: V0130,
},
{ {
version: "v0.12.0", version: "v0.12.0",
date: "2021-12-29", date: "2021-12-29",

View File

@@ -0,0 +1,2 @@
### 🎨 Improvements
* Improved autotag performance. ([#2368](https://github.com/stashapp/stash/pull/2368))