Add search string parsing (#1982)

* Add search string parsing
* Add manual page

pkg/models/search.go

@@ -0,0 +1,167 @@
+package models
+
+import "strings"
+
+const (
+	or         = "OR"
+	orSymbol   = "|"
+	notPrefix  = '-'
+	phraseChar = '"'
+)
+
+// SearchSpecs provides the specifications for text-based searches.
+type SearchSpecs struct {
+	// MustHave specifies all of the terms that must appear in the results.
+	MustHave []string
+
+	// AnySets specifies sets of terms where one of each set must appear in the results.
+	AnySets [][]string
+
+	// MustNot specifies all terms that must not appear in the results.
+	MustNot []string
+}
+
+// combinePhrases detects quote characters at the start and end of
+// words and combines the contents into a single word.
+func combinePhrases(words []string) []string {
+	var ret []string
+	startIndex := -1
+	for i, w := range words {
+		if startIndex == -1 {
+			// looking for start of phrase
+			// this could either be " or -"
+			ww := w
+			if len(w) > 0 && w[0] == notPrefix {
+				ww = w[1:]
+			}
+			if len(ww) > 0 && ww[0] == phraseChar && (len(ww) < 2 || ww[len(ww)-1] != phraseChar) {
+				startIndex = i
+				continue
+			}
+
+			ret = append(ret, w)
+		} else if len(w) > 0 && w[len(w)-1] == phraseChar { // looking for end of phrase
+			// combine words
+			phrase := strings.Join(words[startIndex:i+1], " ")
+
+			// add to return value
+			ret = append(ret, phrase)
+			startIndex = -1
+		}
+	}
+
+	if startIndex != -1 {
+		ret = append(ret, words[startIndex:]...)
+	}
+
+	return ret
+}
+
+func extractOrConditions(words []string, searchSpec *SearchSpecs) []string {
+	for foundOr := true; foundOr; {
+		foundOr = false
+		for i, w := range words {
+			if i > 0 && i < len(words)-1 && (strings.EqualFold(w, or) || w == orSymbol) {
+				// found an OR keyword
+				// first operand will be the last word
+				startIndex := i - 1
+
+				// find the last operand
+				// this will be the last word not preceded by OR
+				lastIndex := len(words) - 1
+				for ii := i + 2; ii < len(words); ii += 2 {
+					if !strings.EqualFold(words[ii], or) {
+						lastIndex = ii - 1
+						break
+					}
+				}
+
+				foundOr = true
+
+				// combine the words into an any set
+				var set []string
+				for ii := startIndex; ii <= lastIndex; ii += 2 {
+					word := extractPhrase(words[ii])
+					if word == "" {
+						continue
+					}
+					set = append(set, word)
+				}
+
+				searchSpec.AnySets = append(searchSpec.AnySets, set)
+
+				// take out the OR'd words
+				words = append(words[0:startIndex], words[lastIndex+1:]...)
+
+				// break and reparse
+				break
+			}
+		}
+	}
+
+	return words
+}
+
+func extractNotConditions(words []string, searchSpec *SearchSpecs) []string {
+	var ret []string
+
+	for _, w := range words {
+		if len(w) > 1 && w[0] == notPrefix {
+			word := extractPhrase(w[1:])
+			if word == "" {
+				continue
+			}
+			searchSpec.MustNot = append(searchSpec.MustNot, word)
+		} else {
+			ret = append(ret, w)
+		}
+	}
+
+	return ret
+}
+
+func extractPhrase(w string) string {
+	if len(w) > 1 && w[0] == phraseChar && w[len(w)-1] == phraseChar {
+		return w[1 : len(w)-1]
+	}
+
+	return w
+}
+
+// ParseSearchString parses the Q value and returns a SearchSpecs object.
+//
+// By default, any words in the search value must appear in the results.
+// Words encompassed by quotes (") as treated as a single term.
+// Where keyword "OR" (case-insensitive) appears (and is not part of a quoted phrase), one of the
+// OR'd terms must appear in the results.
+// Where a keyword is prefixed with "-", that keyword must not appear in the results.
+// Where OR appears as the first or last term, or where one of the OR operands has a
+// not prefix, then the OR is treated literally.
+func ParseSearchString(s string) SearchSpecs {
+	s = strings.TrimSpace(s)
+
+	if s == "" {
+		return SearchSpecs{}
+	}
+
+	// break into words
+	words := strings.Split(s, " ")
+
+	// combine phrases first, then extract OR conditions, then extract NOT conditions
+	// and the leftovers will be AND'd
+	ret := SearchSpecs{}
+	words = combinePhrases(words)
+	words = extractOrConditions(words, &ret)
+	words = extractNotConditions(words, &ret)
+
+	for _, w := range words {
+		// ignore empty quotes
+		word := extractPhrase(w)
+		if word == "" {
+			continue
+		}
+		ret.MustHave = append(ret.MustHave, word)
+	}
+
+	return ret
+}

pkg/models/search_test.go

@@ -0,0 +1,227 @@
+package models
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestParseSearchString(t *testing.T) {
+	tests := []struct {
+		name string
+		q    string
+		want SearchSpecs
+	}{
+		{
+			"basic",
+			"a b c",
+			SearchSpecs{
+				MustHave: []string{"a", "b", "c"},
+			},
+		},
+		{
+			"empty",
+			"",
+			SearchSpecs{},
+		},
+		{
+			"whitespace",
+			" ",
+			SearchSpecs{},
+		},
+		{
+			"single",
+			"a",
+			SearchSpecs{
+				MustHave: []string{"a"},
+			},
+		},
+		{
+			"quoted",
+			`"a b" c`,
+			SearchSpecs{
+				MustHave: []string{"a b", "c"},
+			},
+		},
+		{
+			"quoted double space",
+			`"a  b" c`,
+			SearchSpecs{
+				MustHave: []string{"a  b", "c"},
+			},
+		},
+		{
+			"quoted end space",
+			`"a  b " c`,
+			SearchSpecs{
+				MustHave: []string{"a  b ", "c"},
+			},
+		},
+		{
+			"no matching end quote",
+			`"a b c`,
+			SearchSpecs{
+				MustHave: []string{`"a`, "b", "c"},
+			},
+		},
+		{
+			"no matching start quote",
+			`a b c"`,
+			SearchSpecs{
+				MustHave: []string{"a", "b", `c"`},
+			},
+		},
+		{
+			"or",
+			"a OR b",
+			SearchSpecs{
+				AnySets: [][]string{
+					{"a", "b"},
+				},
+			},
+		},
+		{
+			"multi or",
+			"a OR b c OR d",
+			SearchSpecs{
+				AnySets: [][]string{
+					{"a", "b"},
+					{"c", "d"},
+				},
+			},
+		},
+		{
+			"lowercase or",
+			"a or b",
+			SearchSpecs{
+				AnySets: [][]string{
+					{"a", "b"},
+				},
+			},
+		},
+		{
+			"or symbol",
+			"a | b",
+			SearchSpecs{
+				AnySets: [][]string{
+					{"a", "b"},
+				},
+			},
+		},
+		{
+			"quoted or",
+			`a "OR" b`,
+			SearchSpecs{
+				MustHave: []string{"a", "OR", "b"},
+			},
+		},
+		{
+			"quoted or symbol",
+			`a "|" b`,
+			SearchSpecs{
+				MustHave: []string{"a", "|", "b"},
+			},
+		},
+		{
+			"or phrases",
+			`"a b" OR "c d"`,
+			SearchSpecs{
+				AnySets: [][]string{
+					{"a b", "c d"},
+				},
+			},
+		},
+		{
+			"or at start",
+			"OR a",
+			SearchSpecs{
+				MustHave: []string{"OR", "a"},
+			},
+		},
+		{
+			"or at end",
+			"a OR",
+			SearchSpecs{
+				MustHave: []string{"a", "OR"},
+			},
+		},
+		{
+			"or symbol at start",
+			"| a",
+			SearchSpecs{
+				MustHave: []string{"|", "a"},
+			},
+		},
+		{
+			"or symbol at end",
+			"a |",
+			SearchSpecs{
+				MustHave: []string{"a", "|"},
+			},
+		},
+		{
+			"nots",
+			"-a -b",
+			SearchSpecs{
+				MustNot: []string{"a", "b"},
+			},
+		},
+		{
+			"not or",
+			"-a OR b",
+			SearchSpecs{
+				AnySets: [][]string{
+					{"-a", "b"},
+				},
+			},
+		},
+		{
+			"not phrase",
+			`-"a b"`,
+			SearchSpecs{
+				MustNot: []string{"a b"},
+			},
+		},
+		{
+			"not in phrase",
+			`"-a b"`,
+			SearchSpecs{
+				MustHave: []string{"-a b"},
+			},
+		},
+		{
+			"double not",
+			"--a",
+			SearchSpecs{
+				MustNot: []string{"-a"},
+			},
+		},
+		{
+			"empty quote",
+			`"" a`,
+			SearchSpecs{
+				MustHave: []string{"a"},
+			},
+		},
+		{
+			"not empty quote",
+			`-"" a`,
+			SearchSpecs{
+				MustHave: []string{"a"},
+			},
+		},
+		{
+			"quote in word",
+			`ab"cd"`,
+			SearchSpecs{
+				MustHave: []string{`ab"cd"`},
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := ParseSearchString(tt.q); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("FindFilterType.ParseSearchString() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

pkg/sqlite/gallery.go

@@ -237,9 +237,7 @@ func (qb *galleryQueryBuilder) makeQuery(galleryFilter *models.GalleryFilterType
 
 	if q := findFilter.Q; q != nil && *q != "" {
 		searchColumns := []string{"galleries.title", "galleries.path", "galleries.checksum"}
-		clause, thisArgs := getSearchBinding(searchColumns, *q, false)
-		query.addWhere(clause)
-		query.addArg(thisArgs...)
+		query.parseQueryString(searchColumns, *q)
 	}
 
 	if err := qb.validateFilter(galleryFilter); err != nil {

pkg/sqlite/image.go

@@ -265,9 +265,7 @@ func (qb *imageQueryBuilder) makeQuery(imageFilter *models.ImageFilterType, find
 
 	if q := findFilter.Q; q != nil && *q != "" {
 		searchColumns := []string{"images.title", "images.path", "images.checksum"}
-		clause, thisArgs := getSearchBinding(searchColumns, *q, false)
-		query.addWhere(clause)
-		query.addArg(thisArgs...)
+		query.parseQueryString(searchColumns, *q)
 	}
 
 	if err := qb.validateFilter(imageFilter); err != nil {

pkg/sqlite/movies.go

@@ -145,9 +145,7 @@ func (qb *movieQueryBuilder) Query(movieFilter *models.MovieFilterType, findFilt
 
 	if q := findFilter.Q; q != nil && *q != "" {
 		searchColumns := []string{"movies.name"}
-		clause, thisArgs := getSearchBinding(searchColumns, *q, false)
-		query.addWhere(clause)
-		query.addArg(thisArgs...)
+		query.parseQueryString(searchColumns, *q)
 	}
 
 	filter := qb.makeFilter(movieFilter)

pkg/sqlite/performer.go

@@ -308,9 +308,7 @@ func (qb *performerQueryBuilder) Query(performerFilter *models.PerformerFilterTy
 
 	if q := findFilter.Q; q != nil && *q != "" {
 		searchColumns := []string{"performers.name", "performers.aliases"}
-		clause, thisArgs := getSearchBinding(searchColumns, *q, false)
-		query.addWhere(clause)
-		query.addArg(thisArgs...)
+		query.parseQueryString(searchColumns, *q)
 	}
 
 	if err := qb.validateFilter(performerFilter); err != nil {

pkg/sqlite/query.go

@@ -3,6 +3,9 @@ package sqlite
 import (
 	"fmt"
 	"strings"
+
+	"github.com/stashapp/stash/pkg/logger"
+	"github.com/stashapp/stash/pkg/models"
 )
 
 type queryBuilder struct {
@@ -53,7 +56,9 @@ func (qb queryBuilder) toSQL(includeSortPagination bool) string {
 
 func (qb queryBuilder) findIDs() ([]int, error) {
 	const includeSortPagination = true
-	return qb.repository.runIdsQuery(qb.toSQL(includeSortPagination), qb.args)
+	sql := qb.toSQL(includeSortPagination)
+	logger.Tracef("SQL: %s, args: %v", sql, qb.args)
+	return qb.repository.runIdsQuery(sql, qb.args)
 }
 
 func (qb queryBuilder) executeFind() ([]int, int, error) {
@@ -168,3 +173,38 @@ func (qb *queryBuilder) addFilter(f *filterBuilder) {
 
 	qb.addJoins(f.getAllJoins()...)
 }
+
+func (qb *queryBuilder) parseQueryString(columns []string, q string) {
+	specs := models.ParseSearchString(q)
+
+	for _, t := range specs.MustHave {
+		var clauses []string
+
+		for _, column := range columns {
+			clauses = append(clauses, column+" LIKE ?")
+			qb.addArg(like(t))
+		}
+
+		qb.addWhere("(" + strings.Join(clauses, " OR ") + ")")
+	}
+
+	for _, t := range specs.MustNot {
+		for _, column := range columns {
+			qb.addWhere(coalesce(column) + " NOT LIKE ?")
+			qb.addArg(like(t))
+		}
+	}
+
+	for _, set := range specs.AnySets {
+		var clauses []string
+
+		for _, column := range columns {
+			for _, v := range set {
+				clauses = append(clauses, column+" LIKE ?")
+				qb.addArg(like(v))
+			}
+		}
+
+		qb.addWhere("(" + strings.Join(clauses, " OR ") + ")")
+	}
+}

pkg/sqlite/scene.go

@@ -412,9 +412,7 @@ func (qb *sceneQueryBuilder) Query(options models.SceneQueryOptions) (*models.Sc
 	if q := findFilter.Q; q != nil && *q != "" {
 		query.join("scene_markers", "", "scene_markers.scene_id = scenes.id")
 		searchColumns := []string{"scenes.title", "scenes.details", "scenes.path", "scenes.oshash", "scenes.checksum", "scene_markers.title"}
-		clause, thisArgs := getSearchBinding(searchColumns, *q, false)
-		query.addWhere(clause)
-		query.addArg(thisArgs...)
+		query.parseQueryString(searchColumns, *q)
 	}
 
 	if err := qb.validateFilter(sceneFilter); err != nil {

pkg/sqlite/scene_marker.go

@@ -151,9 +151,7 @@ func (qb *sceneMarkerQueryBuilder) Query(sceneMarkerFilter *models.SceneMarkerFi
 
 	if q := findFilter.Q; q != nil && *q != "" {
 		searchColumns := []string{"scene_markers.title", "scenes.title"}
-		clause, thisArgs := getSearchBinding(searchColumns, *q, false)
-		query.addWhere(clause)
-		query.addArg(thisArgs...)
+		query.parseQueryString(searchColumns, *q)
 	}
 
 	filter := qb.makeFilter(sceneMarkerFilter)

pkg/sqlite/sql.go

@@ -249,3 +249,11 @@ func getImage(tx dbi, query string, args ...interface{}) ([]byte, error) {
 
 	return ret, nil
 }
+
+func coalesce(column string) string {
+	return fmt.Sprintf("COALESCE(%s, '')", column)
+}
+
+func like(v string) string {
+	return "%" + v + "%"
+}

pkg/sqlite/studio.go

@@ -244,9 +244,7 @@ func (qb *studioQueryBuilder) Query(studioFilter *models.StudioFilterType, findF
 		query.join(studioAliasesTable, "", "studio_aliases.studio_id = studios.id")
 		searchColumns := []string{"studios.name", "studio_aliases.alias"}
 
-		clause, thisArgs := getSearchBinding(searchColumns, *q, false)
-		query.addWhere(clause)
-		query.addArg(thisArgs...)
+		query.parseQueryString(searchColumns, *q)
 	}
 
 	if err := qb.validateFilter(studioFilter); err != nil {

pkg/sqlite/tag.go

@@ -329,9 +329,7 @@ func (qb *tagQueryBuilder) Query(tagFilter *models.TagFilterType, findFilter *mo
 	if q := findFilter.Q; q != nil && *q != "" {
 		query.join(tagAliasesTable, "", "tag_aliases.tag_id = tags.id")
 		searchColumns := []string{"tags.name", "tag_aliases.alias"}
-		clause, thisArgs := getSearchBinding(searchColumns, *q, false)
-		query.addWhere(clause)
-		query.addArg(thisArgs...)
+		query.parseQueryString(searchColumns, *q)
 	}
 
 	if err := qb.validateFilter(tagFilter); err != nil {

ui/v2.5/src/components/Changelog/versions/v0120.md

@@ -1,4 +1,5 @@
 ### ✨ New Features
+* Changed query string parsing behaviour to require all words by default, with the option to `or` keywords and exclude keywords. See the `Browsing` section of the manual for details. ([#1982](https://github.com/stashapp/stash/pull/1982))
 * Add forward jump 10 second button to video player. ([#1973](https://github.com/stashapp/stash/pull/1973))
 
 ### 🎨 Improvements

ui/v2.5/src/components/Help/Manual.tsx

@@ -20,6 +20,7 @@ import Help from "src/docs/en/Help.md";
 import Deduplication from "src/docs/en/Deduplication.md";
 import Interactive from "src/docs/en/Interactive.md";
 import Identify from "src/docs/en/Identify.md";
+import Browsing from "src/docs/en/Browsing.md";
 import { MarkdownPage } from "../Shared/MarkdownPage";
 
 interface IManualProps {
@@ -80,6 +81,11 @@ export const Manual: React.FC<IManualProps> = ({
       content: JSONSpec,
       className: "indent-1",
     },
+    {
+      key: "Browsing.md",
+      title: "Browsing",
+      content: Browsing,
+    },
     {
       key: "Galleries.md",
       title: "Image Galleries",

ui/v2.5/src/docs/en/Browsing.md

@@ -0,0 +1,45 @@
+# Browsing
+
+## Querying and Filtering
+
+### Keyword searching
+
+The text field allows you to search using keywords. Keyword searching matches on different fields depending on the object type:
+
+| Type | Fields searched |
+|------|-----------------|
+| Scene | Title, Details, Path, OSHash, Checksum, Marker titles |
+| Image | Title, Path, Checksum |
+| Movie | Title |
+| Marker | Title, Scene title |
+| Gallery | Title, Path, Checksum |
+| Performer | Name, Aliases |
+| Studio | Name, Aliases |
+| Tag | Name, Aliases |
+
+Keyword matching uses the following rules:
+* all words are required in the matching field. For example, `foo bar` matches scenes with both `foo` and `bar` in the title.
+* the `or` keyword or symbol (`|`) is used to match either fields. For example, `foo or bar` (or `foo | bar`) matches scenes with `foo` or `bar` in the title. Or sets can be combined. For example, `foo or bar or baz xyz or zyx` matches scenes with one of `foo`, `bar` and `baz`, *and* `xyz` or `zyx`.
+* the not symbol (`-`) is used to exclude terms. For example, `foo -bar` matches scenes with `foo` and excludes those with `bar`. The not symbol cannot be combined with an or operand. That is, `-foo or bar` will be interpreted to match `-foo` or `bar`. On the other hand, `foo or bar -baz` will match `foo` or `bar` and exclude `baz`.
+* surrounding a phrase in quotes (`"`) matches on that exact phrase. For example, `"foo bar"` matches scenes with `foo bar` in the title. Quotes may also be used to escape the keywords and symbols. For example, `foo "-bar"` will match scenes with `foo` and `-bar`.
+* quoted phrases may be used with the or and not operators. For example, `"foo bar" or baz -"xyz zyx"` will match scenes with `foo bar` *or* `baz`, and exclude those with `xyz zyx`.
+* `or` keywords or symbols at the start or end of a line will be treated literally. That is, `or foo` will match scenes with `or` and `foo`.
+* all matching is case-insensitive
+
+### Filters
+
+Filters can be accessed by clicking the filter button on the right side of the query text field. 
+
+Note that only one filter criterion per criterion type may be assigned.
+
+### Sorting and page size
+
+The current sorting field is shown next to the query text field, indicating the current sort field and order. The page size dropdown allows selecting from a standard set of objects per page, and allows setting a custom page size.
+
+### Saved filters
+
+Saved filters can be accessed with the bookmark button on the left of the query text field. The current filter can be saved by entering a filter name and clicking on the save button. Existing saved filters may be overwritten with the current filter by clicking on the save button next to the filter name. Saved filters may also be deleted by pressing the delete button next to the filter name.
+
+### Default filter
+
+The default filter for the top-level pages may be set to the current filter by clicking the `Set as default` button in the saved filter menu.