String regex filter criteria and selective autotag (#1082)

* Add regex string filter criterion
* Use query interface for auto tagging
* Use Query interface for filename parser
* Remove query regex interfaces
* Add selective auto tag
* Use page size 0 as no limit

pkg/sqlite/gallery_test.go

@@ -166,6 +166,13 @@ func TestGalleryQueryPath(t *testing.T) {
 		pathCriterion.Modifier = models.CriterionModifierNotEquals
 		verifyGalleriesPath(t, r.Gallery(), pathCriterion)
 
+		pathCriterion.Modifier = models.CriterionModifierMatchesRegex
+		pathCriterion.Value = "gallery.*1_Path"
+		verifyGalleriesPath(t, r.Gallery(), pathCriterion)
+
+		pathCriterion.Modifier = models.CriterionModifierNotMatchesRegex
+		verifyGalleriesPath(t, r.Gallery(), pathCriterion)
+
 		return nil
 	})
 }

pkg/sqlite/image_test.go

@@ -114,13 +114,20 @@ func TestImageQueryPath(t *testing.T) {
 		Modifier: models.CriterionModifierEquals,
 	}
 
-	verifyImagePath(t, pathCriterion)
+	verifyImagePath(t, pathCriterion, 1)
 
 	pathCriterion.Modifier = models.CriterionModifierNotEquals
-	verifyImagePath(t, pathCriterion)
+	verifyImagePath(t, pathCriterion, totalImages-1)
+
+	pathCriterion.Modifier = models.CriterionModifierMatchesRegex
+	pathCriterion.Value = "image_.*1_Path"
+	verifyImagePath(t, pathCriterion, 1) // TODO - 2 if zip path is included
+
+	pathCriterion.Modifier = models.CriterionModifierNotMatchesRegex
+	verifyImagePath(t, pathCriterion, totalImages-1) // TODO - -2 if zip path is included
 }
 
-func verifyImagePath(t *testing.T, pathCriterion models.StringCriterionInput) {
+func verifyImagePath(t *testing.T, pathCriterion models.StringCriterionInput, expected int) {
 	withTxn(func(r models.Repository) error {
 		sqb := r.Image()
 		imageFilter := models.ImageFilterType{
@@ -132,6 +139,8 @@ func verifyImagePath(t *testing.T, pathCriterion models.StringCriterionInput) {
 			t.Errorf("Error querying image: %s", err.Error())
 		}
 
+		assert.Equal(t, expected, len(images), "number of returned images")
+
 		for _, image := range images {
 			verifyString(t, image.Path, pathCriterion)
 		}

pkg/sqlite/query.go

@@ -1,6 +1,10 @@
 package sqlite
 
-import "github.com/stashapp/stash/pkg/models"
+import (
+	"regexp"
+
+	"github.com/stashapp/stash/pkg/models"
+)
 
 type queryBuilder struct {
 	repository *repository
@@ -12,9 +16,15 @@ type queryBuilder struct {
 	args          []interface{}
 
 	sortAndPagination string
+
+	err error
 }
 
 func (qb queryBuilder) executeFind() ([]int, int, error) {
+	if qb.err != nil {
+		return nil, 0, qb.err
+	}
+
 	return qb.repository.executeFindQuery(qb.body, qb.args, qb.sortAndPagination, qb.whereClauses, qb.havingClauses)
 }
 
@@ -66,6 +76,20 @@ func (qb *queryBuilder) handleStringCriterionInput(c *models.StringCriterionInpu
 			case models.CriterionModifierNotEquals:
 				qb.addWhere(column + " NOT LIKE ?")
 				qb.addArg(c.Value)
+			case models.CriterionModifierMatchesRegex:
+				if _, err := regexp.Compile(c.Value); err != nil {
+					qb.err = err
+					return
+				}
+				qb.addWhere(column + " regexp ?")
+				qb.addArg(c.Value)
+			case models.CriterionModifierNotMatchesRegex:
+				if _, err := regexp.Compile(c.Value); err != nil {
+					qb.err = err
+					return
+				}
+				qb.addWhere(column + " NOT regexp ?")
+				qb.addArg(c.Value)
 			default:
 				clause, count := getSimpleCriterionClause(modifier, "?")
 				qb.addWhere(column + " " + clause)

pkg/sqlite/scene.go

@@ -3,6 +3,7 @@ package sqlite
 import (
 	"database/sql"
 	"fmt"
+	"path/filepath"
 	"strings"
 
 	"github.com/jmoiron/sqlx"
@@ -289,6 +290,53 @@ func (qb *sceneQueryBuilder) All() ([]*models.Scene, error) {
 	return qb.queryScenes(selectAll(sceneTable)+qb.getSceneSort(nil), nil)
 }
 
+// QueryForAutoTag queries for scenes whose paths match the provided regex and
+// are optionally within the provided path. Excludes organized scenes.
+// TODO - this should be replaced with Query once it can perform multiple
+// filters on the same field.
+func (qb *sceneQueryBuilder) QueryForAutoTag(regex string, pathPrefixes []string) ([]*models.Scene, error) {
+	var args []interface{}
+	body := selectDistinctIDs("scenes") + ` WHERE 
+	scenes.path regexp ? AND 
+	scenes.organized = 0`
+
+	args = append(args, "(?i)"+regex)
+
+	var pathClauses []string
+	for _, p := range pathPrefixes {
+		pathClauses = append(pathClauses, "scenes.path like ?")
+
+		sep := string(filepath.Separator)
+		if !strings.HasSuffix(p, sep) {
+			p = p + sep
+		}
+		args = append(args, p+"%")
+	}
+
+	if len(pathClauses) > 0 {
+		body += " AND (" + strings.Join(pathClauses, " OR ") + ")"
+	}
+
+	idsResult, err := qb.runIdsQuery(body, args)
+
+	if err != nil {
+		return nil, err
+	}
+
+	var scenes []*models.Scene
+	for _, id := range idsResult {
+		scene, err := qb.Find(id)
+
+		if err != nil {
+			return nil, err
+		}
+
+		scenes = append(scenes, scene)
+	}
+
+	return scenes, nil
+}
+
 func (qb *sceneQueryBuilder) Query(sceneFilter *models.SceneFilterType, findFilter *models.FindFilterType) ([]*models.Scene, int, error) {
 	if sceneFilter == nil {
 		sceneFilter = &models.SceneFilterType{}
@@ -448,6 +496,7 @@ func (qb *sceneQueryBuilder) Query(sceneFilter *models.SceneFilterType, findFilt
 	}
 
 	query.sortAndPagination = qb.getSceneSort(findFilter) + getPagination(findFilter)
+
 	idsResult, countResult, err := query.executeFind()
 	if err != nil {
 		return nil, 0, err
@@ -501,70 +550,6 @@ func getDurationWhereClause(durationFilter models.IntCriterionInput) (string, []
 	return clause, args
 }
 
-func (qb *sceneQueryBuilder) QueryAllByPathRegex(regex string, ignoreOrganized bool) ([]*models.Scene, error) {
-	var args []interface{}
-	body := selectDistinctIDs("scenes") + " WHERE scenes.path regexp ?"
-
-	if ignoreOrganized {
-		body += " AND scenes.organized = 0"
-	}
-
-	args = append(args, "(?i)"+regex)
-
-	idsResult, err := qb.runIdsQuery(body, args)
-
-	if err != nil {
-		return nil, err
-	}
-
-	var scenes []*models.Scene
-	for _, id := range idsResult {
-		scene, err := qb.Find(id)
-
-		if err != nil {
-			return nil, err
-		}
-
-		scenes = append(scenes, scene)
-	}
-
-	return scenes, nil
-}
-
-func (qb *sceneQueryBuilder) QueryByPathRegex(findFilter *models.FindFilterType) ([]*models.Scene, int, error) {
-	if findFilter == nil {
-		findFilter = &models.FindFilterType{}
-	}
-
-	var whereClauses []string
-	var havingClauses []string
-	var args []interface{}
-	body := selectDistinctIDs("scenes")
-
-	if q := findFilter.Q; q != nil && *q != "" {
-		whereClauses = append(whereClauses, "scenes.path regexp ?")
-		args = append(args, "(?i)"+*q)
-	}
-
-	sortAndPagination := qb.getSceneSort(findFilter) + getPagination(findFilter)
-	idsResult, countResult, err := qb.executeFindQuery(body, args, sortAndPagination, whereClauses, havingClauses)
-	if err != nil {
-		return nil, 0, err
-	}
-
-	var scenes []*models.Scene
-	for _, id := range idsResult {
-		scene, err := qb.Find(id)
-		if err != nil {
-			return nil, 0, err
-		}
-
-		scenes = append(scenes, scene)
-	}
-
-	return scenes, countResult, nil
-}
-
 func (qb *sceneQueryBuilder) getSceneSort(findFilter *models.FindFilterType) string {
 	if findFilter == nil {
 		return " ORDER BY scenes.path, scenes.date ASC "

pkg/sqlite/scene_test.go

@@ -5,6 +5,7 @@ package sqlite_test
 import (
 	"database/sql"
 	"fmt"
+	"regexp"
 	"strconv"
 	"testing"
 
@@ -176,6 +177,13 @@ func TestSceneQueryPath(t *testing.T) {
 
 	pathCriterion.Modifier = models.CriterionModifierNotEquals
 	verifyScenesPath(t, pathCriterion)
+
+	pathCriterion.Modifier = models.CriterionModifierMatchesRegex
+	pathCriterion.Value = "scene_.*1_Path"
+	verifyScenesPath(t, pathCriterion)
+
+	pathCriterion.Modifier = models.CriterionModifierNotMatchesRegex
+	verifyScenesPath(t, pathCriterion)
 }
 
 func verifyScenesPath(t *testing.T, pathCriterion models.StringCriterionInput) {
@@ -221,6 +229,12 @@ func verifyString(t *testing.T, value string, criterion models.StringCriterionIn
 	if criterion.Modifier == models.CriterionModifierNotEquals {
 		assert.NotEqual(criterion.Value, value)
 	}
+	if criterion.Modifier == models.CriterionModifierMatchesRegex {
+		assert.Regexp(regexp.MustCompile(criterion.Value), value)
+	}
+	if criterion.Modifier == models.CriterionModifierNotMatchesRegex {
+		assert.NotRegexp(regexp.MustCompile(criterion.Value), value)
+	}
 }
 
 func TestSceneQueryRating(t *testing.T) {

pkg/sqlite/setup_test.go

@@ -21,7 +21,7 @@ import (
 )
 
 const totalScenes = 12
-const totalImages = 6
+const totalImages = 6 // TODO - add one for zip file
 const performersNameCase = 6
 const performersNameNoCase = 2
 const moviesNameCase = 2
@@ -61,6 +61,7 @@ const imageIdxWithTwoPerformers = 2
 const imageIdxWithTag = 3
 const imageIdxWithTwoTags = 4
 const imageIdxWithStudio = 5
+const imageIdxInZip = 6
 
 const performerIdxWithScene = 0
 const performerIdx1WithScene = 1
@@ -110,6 +111,7 @@ const markerIdxWithScene = 0
 const pathField = "Path"
 const checksumField = "Checksum"
 const titleField = "Title"
+const zipPath = "zipPath.zip"
 
 func TestMain(m *testing.M) {
 	ret := runTests(m)
@@ -318,10 +320,19 @@ func getImageStringValue(index int, field string) string {
 	return fmt.Sprintf("image_%04d_%s", index, field)
 }
 
+func getImagePath(index int) string {
+	// TODO - currently not working
+	// if index == imageIdxInZip {
+	// 	return image.ZipFilename(zipPath, "image_0001_Path")
+	// }
+
+	return getImageStringValue(index, pathField)
+}
+
 func createImages(qb models.ImageReaderWriter, n int) error {
 	for i := 0; i < n; i++ {
 		image := models.Image{
-			Path:     getImageStringValue(i, pathField),
+			Path:     getImagePath(i),
 			Title:    sql.NullString{String: getImageStringValue(i, titleField), Valid: true},
 			Checksum: getImageStringValue(i, checksumField),
 			Rating:   getRating(i),

pkg/sqlite/sql.go

@@ -32,26 +32,14 @@ func getPagination(findFilter *models.FindFilterType) string {
 		panic("nil find filter for pagination")
 	}
 
-	var page int
-	if findFilter.Page == nil || *findFilter.Page < 1 {
-		page = 1
-	} else {
-		page = *findFilter.Page
+	if findFilter.IsGetAll() {
+		return " "
 	}
 
-	var perPage int
-	if findFilter.PerPage == nil {
-		perPage = 25
-	} else {
-		perPage = *findFilter.PerPage
-	}
-
-	if perPage > 1000 {
-		perPage = 1000
-	} else if perPage < 1 {
-		perPage = 1
-	}
+	return getPaginationSQL(findFilter.GetPage(), findFilter.GetPageSize())
+}
 
+func getPaginationSQL(page int, perPage int) string {
 	page = (page - 1) * perPage
 	return " LIMIT " + strconv.Itoa(perPage) + " OFFSET " + strconv.Itoa(page) + " "
 }