Fix joined hierarchical filtering (#3775)

* Fix joined hierarchical filtering
* Fix scene performer tag filter
* Generalise performer tag handler
* Add unit tests
* Add equals handling
* Make performer tags equals/not equals unsupported
* Make tags not equals unsupported
* Make not equals unsupported for performers criterion
* Support equals/not equals for studio criterion
* Fix marker scene tags equals filter
* Fix scene performer tag filter
* Make equals/not equals unsupported for hierarchical criterion
* Use existing studio handler in movie
* Hide unsupported tag modifier options
* Use existing performer tags logic where possible
* Restore old parent/child filter logic
* Disable sub-tags in equals modifier for tags criterion
This commit is contained in:
WithoutPants
2023-06-06 13:01:50 +10:00
committed by GitHub
parent 4acf843229
commit 256e0a11ea
19 changed files with 2153 additions and 938 deletions

View File

@@ -668,7 +668,8 @@ func Test_sceneQueryBuilder_UpdatePartial(t *testing.T) {
sceneIDs[sceneIdxWithSpacedName],
clearScenePartial(),
models.Scene{
ID: sceneIDs[sceneIdxWithSpacedName],
ID: sceneIDs[sceneIdxWithSpacedName],
OCounter: getOCounter(sceneIdxWithSpacedName),
Files: models.NewRelatedVideoFiles([]*file.VideoFile{
makeSceneFile(sceneIdxWithSpacedName),
}),
@@ -677,6 +678,10 @@ func Test_sceneQueryBuilder_UpdatePartial(t *testing.T) {
PerformerIDs: models.NewRelatedIDs([]int{}),
Movies: models.NewRelatedMovies([]models.MoviesScenes{}),
StashIDs: models.NewRelatedStashIDs([]models.StashID{}),
PlayCount: getScenePlayCount(sceneIdxWithSpacedName),
PlayDuration: getScenePlayDuration(sceneIdxWithSpacedName),
LastPlayedAt: getSceneLastPlayed(sceneIdxWithSpacedName),
ResumeTime: getSceneResumeTime(sceneIdxWithSpacedName),
},
false,
},
@@ -2101,6 +2106,8 @@ func sceneQueryQ(ctx context.Context, t *testing.T, sqb models.SceneReader, q st
// no Q should return all results
filter.Q = nil
pp := totalScenes
filter.PerPage = &pp
scenes = queryScene(ctx, t, sqb, nil, &filter)
assert.Len(t, scenes, totalScenes)
@@ -2230,8 +2237,8 @@ func TestSceneQuery(t *testing.T) {
return
}
include := indexesToIDs(performerIDs, tt.includeIdxs)
exclude := indexesToIDs(performerIDs, tt.excludeIdxs)
include := indexesToIDs(sceneIDs, tt.includeIdxs)
exclude := indexesToIDs(sceneIDs, tt.excludeIdxs)
for _, i := range include {
assert.Contains(results.IDs, i)
@@ -3057,7 +3064,13 @@ func queryScenes(ctx context.Context, t *testing.T, queryBuilder models.SceneRea
},
}
return queryScene(ctx, t, queryBuilder, &sceneFilter, nil)
// needed so that we don't hit the default limit of 25 scenes
pp := 1000
findFilter := &models.FindFilterType{
PerPage: &pp,
}
return queryScene(ctx, t, queryBuilder, &sceneFilter, findFilter)
}
func createScene(ctx context.Context, width int, height int) (*models.Scene, error) {
@@ -3329,192 +3342,473 @@ func TestSceneQueryIsMissingPhash(t *testing.T) {
}
func TestSceneQueryPerformers(t *testing.T) {
withTxn(func(ctx context.Context) error {
sqb := db.Scene
performerCriterion := models.MultiCriterionInput{
Value: []string{
strconv.Itoa(performerIDs[performerIdxWithScene]),
strconv.Itoa(performerIDs[performerIdx1WithScene]),
tests := []struct {
name string
filter models.MultiCriterionInput
includeIdxs []int
excludeIdxs []int
wantErr bool
}{
{
"includes",
models.MultiCriterionInput{
Value: []string{
strconv.Itoa(performerIDs[performerIdxWithScene]),
strconv.Itoa(performerIDs[performerIdx1WithScene]),
},
Modifier: models.CriterionModifierIncludes,
},
Modifier: models.CriterionModifierIncludes,
}
sceneFilter := models.SceneFilterType{
Performers: &performerCriterion,
}
scenes := queryScene(ctx, t, sqb, &sceneFilter, nil)
assert.Len(t, scenes, 2)
// ensure ids are correct
for _, scene := range scenes {
assert.True(t, scene.ID == sceneIDs[sceneIdxWithPerformer] || scene.ID == sceneIDs[sceneIdxWithTwoPerformers])
}
performerCriterion = models.MultiCriterionInput{
Value: []string{
strconv.Itoa(performerIDs[performerIdx1WithScene]),
strconv.Itoa(performerIDs[performerIdx2WithScene]),
[]int{
sceneIdxWithPerformer,
sceneIdxWithTwoPerformers,
},
Modifier: models.CriterionModifierIncludesAll,
}
scenes = queryScene(ctx, t, sqb, &sceneFilter, nil)
assert.Len(t, scenes, 1)
assert.Equal(t, sceneIDs[sceneIdxWithTwoPerformers], scenes[0].ID)
performerCriterion = models.MultiCriterionInput{
Value: []string{
strconv.Itoa(performerIDs[performerIdx1WithScene]),
[]int{
sceneIdxWithGallery,
},
Modifier: models.CriterionModifierExcludes,
}
false,
},
{
"includes all",
models.MultiCriterionInput{
Value: []string{
strconv.Itoa(performerIDs[performerIdx1WithScene]),
strconv.Itoa(performerIDs[performerIdx2WithScene]),
},
Modifier: models.CriterionModifierIncludesAll,
},
[]int{
sceneIdxWithTwoPerformers,
},
[]int{
sceneIdxWithPerformer,
},
false,
},
{
"excludes",
models.MultiCriterionInput{
Modifier: models.CriterionModifierExcludes,
Value: []string{strconv.Itoa(tagIDs[performerIdx1WithScene])},
},
nil,
[]int{sceneIdxWithTwoPerformers},
false,
},
{
"is null",
models.MultiCriterionInput{
Modifier: models.CriterionModifierIsNull,
},
[]int{sceneIdxWithTag},
[]int{
sceneIdxWithPerformer,
sceneIdxWithTwoPerformers,
sceneIdxWithPerformerTwoTags,
},
false,
},
{
"not null",
models.MultiCriterionInput{
Modifier: models.CriterionModifierNotNull,
},
[]int{
sceneIdxWithPerformer,
sceneIdxWithTwoPerformers,
sceneIdxWithPerformerTwoTags,
},
[]int{sceneIdxWithTag},
false,
},
{
"equals",
models.MultiCriterionInput{
Modifier: models.CriterionModifierEquals,
Value: []string{
strconv.Itoa(tagIDs[performerIdx1WithScene]),
strconv.Itoa(tagIDs[performerIdx2WithScene]),
},
},
[]int{sceneIdxWithTwoPerformers},
[]int{
sceneIdxWithThreePerformers,
},
false,
},
{
"not equals",
models.MultiCriterionInput{
Modifier: models.CriterionModifierNotEquals,
Value: []string{
strconv.Itoa(tagIDs[performerIdx1WithScene]),
strconv.Itoa(tagIDs[performerIdx2WithScene]),
},
},
nil,
nil,
true,
},
}
q := getSceneStringValue(sceneIdxWithTwoPerformers, titleField)
findFilter := models.FindFilterType{
Q: &q,
}
for _, tt := range tests {
runWithRollbackTxn(t, tt.name, func(t *testing.T, ctx context.Context) {
assert := assert.New(t)
scenes = queryScene(ctx, t, sqb, &sceneFilter, &findFilter)
assert.Len(t, scenes, 0)
results, err := db.Scene.Query(ctx, models.SceneQueryOptions{
SceneFilter: &models.SceneFilterType{
Performers: &tt.filter,
},
})
if (err != nil) != tt.wantErr {
t.Errorf("SceneStore.Query() error = %v, wantErr %v", err, tt.wantErr)
return
}
return nil
})
include := indexesToIDs(sceneIDs, tt.includeIdxs)
exclude := indexesToIDs(sceneIDs, tt.excludeIdxs)
for _, i := range include {
assert.Contains(results.IDs, i)
}
for _, e := range exclude {
assert.NotContains(results.IDs, e)
}
})
}
}
func TestSceneQueryTags(t *testing.T) {
withTxn(func(ctx context.Context) error {
sqb := db.Scene
tagCriterion := models.HierarchicalMultiCriterionInput{
Value: []string{
strconv.Itoa(tagIDs[tagIdxWithScene]),
strconv.Itoa(tagIDs[tagIdx1WithScene]),
tests := []struct {
name string
filter models.HierarchicalMultiCriterionInput
includeIdxs []int
excludeIdxs []int
wantErr bool
}{
{
"includes",
models.HierarchicalMultiCriterionInput{
Value: []string{
strconv.Itoa(tagIDs[tagIdxWithScene]),
strconv.Itoa(tagIDs[tagIdx1WithScene]),
},
Modifier: models.CriterionModifierIncludes,
},
Modifier: models.CriterionModifierIncludes,
}
sceneFilter := models.SceneFilterType{
Tags: &tagCriterion,
}
scenes := queryScene(ctx, t, sqb, &sceneFilter, nil)
assert.Len(t, scenes, 2)
// ensure ids are correct
for _, scene := range scenes {
assert.True(t, scene.ID == sceneIDs[sceneIdxWithTag] || scene.ID == sceneIDs[sceneIdxWithTwoTags])
}
tagCriterion = models.HierarchicalMultiCriterionInput{
Value: []string{
strconv.Itoa(tagIDs[tagIdx1WithScene]),
strconv.Itoa(tagIDs[tagIdx2WithScene]),
[]int{
sceneIdxWithTag,
sceneIdxWithTwoTags,
},
Modifier: models.CriterionModifierIncludesAll,
}
scenes = queryScene(ctx, t, sqb, &sceneFilter, nil)
assert.Len(t, scenes, 1)
assert.Equal(t, sceneIDs[sceneIdxWithTwoTags], scenes[0].ID)
tagCriterion = models.HierarchicalMultiCriterionInput{
Value: []string{
strconv.Itoa(tagIDs[tagIdx1WithScene]),
[]int{
sceneIdxWithGallery,
},
Modifier: models.CriterionModifierExcludes,
}
false,
},
{
"includes all",
models.HierarchicalMultiCriterionInput{
Value: []string{
strconv.Itoa(tagIDs[tagIdx1WithScene]),
strconv.Itoa(tagIDs[tagIdx2WithScene]),
},
Modifier: models.CriterionModifierIncludesAll,
},
[]int{
sceneIdxWithTwoTags,
},
[]int{
sceneIdxWithTag,
},
false,
},
{
"excludes",
models.HierarchicalMultiCriterionInput{
Modifier: models.CriterionModifierExcludes,
Value: []string{strconv.Itoa(tagIDs[tagIdx1WithScene])},
},
nil,
[]int{sceneIdxWithTwoTags},
false,
},
{
"is null",
models.HierarchicalMultiCriterionInput{
Modifier: models.CriterionModifierIsNull,
},
[]int{sceneIdx1WithPerformer},
[]int{
sceneIdxWithTag,
sceneIdxWithTwoTags,
sceneIdxWithMarkerAndTag,
},
false,
},
{
"not null",
models.HierarchicalMultiCriterionInput{
Modifier: models.CriterionModifierNotNull,
},
[]int{
sceneIdxWithTag,
sceneIdxWithTwoTags,
sceneIdxWithMarkerAndTag,
},
[]int{sceneIdx1WithPerformer},
false,
},
{
"equals",
models.HierarchicalMultiCriterionInput{
Modifier: models.CriterionModifierEquals,
Value: []string{
strconv.Itoa(tagIDs[tagIdx1WithScene]),
strconv.Itoa(tagIDs[tagIdx2WithScene]),
},
},
[]int{sceneIdxWithTwoTags},
[]int{
sceneIdxWithThreeTags,
},
false,
},
{
"not equals",
models.HierarchicalMultiCriterionInput{
Modifier: models.CriterionModifierNotEquals,
Value: []string{
strconv.Itoa(tagIDs[tagIdx1WithScene]),
strconv.Itoa(tagIDs[tagIdx2WithScene]),
},
},
nil,
nil,
true,
},
}
q := getSceneStringValue(sceneIdxWithTwoTags, titleField)
findFilter := models.FindFilterType{
Q: &q,
}
for _, tt := range tests {
runWithRollbackTxn(t, tt.name, func(t *testing.T, ctx context.Context) {
assert := assert.New(t)
scenes = queryScene(ctx, t, sqb, &sceneFilter, &findFilter)
assert.Len(t, scenes, 0)
results, err := db.Scene.Query(ctx, models.SceneQueryOptions{
SceneFilter: &models.SceneFilterType{
Tags: &tt.filter,
},
})
if (err != nil) != tt.wantErr {
t.Errorf("SceneStore.Query() error = %v, wantErr %v", err, tt.wantErr)
return
}
return nil
})
include := indexesToIDs(sceneIDs, tt.includeIdxs)
exclude := indexesToIDs(sceneIDs, tt.excludeIdxs)
for _, i := range include {
assert.Contains(results.IDs, i)
}
for _, e := range exclude {
assert.NotContains(results.IDs, e)
}
})
}
}
func TestSceneQueryPerformerTags(t *testing.T) {
withTxn(func(ctx context.Context) error {
sqb := db.Scene
tagCriterion := models.HierarchicalMultiCriterionInput{
Value: []string{
strconv.Itoa(tagIDs[tagIdxWithPerformer]),
strconv.Itoa(tagIDs[tagIdx1WithPerformer]),
allDepth := -1
tests := []struct {
name string
findFilter *models.FindFilterType
filter *models.SceneFilterType
includeIdxs []int
excludeIdxs []int
wantErr bool
}{
{
"includes",
nil,
&models.SceneFilterType{
PerformerTags: &models.HierarchicalMultiCriterionInput{
Value: []string{
strconv.Itoa(tagIDs[tagIdxWithPerformer]),
strconv.Itoa(tagIDs[tagIdx1WithPerformer]),
},
Modifier: models.CriterionModifierIncludes,
},
},
Modifier: models.CriterionModifierIncludes,
}
sceneFilter := models.SceneFilterType{
PerformerTags: &tagCriterion,
}
scenes := queryScene(ctx, t, sqb, &sceneFilter, nil)
assert.Len(t, scenes, 2)
// ensure ids are correct
for _, scene := range scenes {
assert.True(t, scene.ID == sceneIDs[sceneIdxWithPerformerTag] || scene.ID == sceneIDs[sceneIdxWithPerformerTwoTags])
}
tagCriterion = models.HierarchicalMultiCriterionInput{
Value: []string{
strconv.Itoa(tagIDs[tagIdx1WithPerformer]),
strconv.Itoa(tagIDs[tagIdx2WithPerformer]),
[]int{
sceneIdxWithPerformerTag,
sceneIdxWithPerformerTwoTags,
sceneIdxWithTwoPerformerTag,
},
Modifier: models.CriterionModifierIncludesAll,
}
scenes = queryScene(ctx, t, sqb, &sceneFilter, nil)
assert.Len(t, scenes, 1)
assert.Equal(t, sceneIDs[sceneIdxWithPerformerTwoTags], scenes[0].ID)
tagCriterion = models.HierarchicalMultiCriterionInput{
Value: []string{
strconv.Itoa(tagIDs[tagIdx1WithPerformer]),
[]int{
sceneIdxWithPerformer,
},
Modifier: models.CriterionModifierExcludes,
}
false,
},
{
"includes sub-tags",
nil,
&models.SceneFilterType{
PerformerTags: &models.HierarchicalMultiCriterionInput{
Value: []string{
strconv.Itoa(tagIDs[tagIdxWithParentAndChild]),
},
Depth: &allDepth,
Modifier: models.CriterionModifierIncludes,
},
},
[]int{
sceneIdxWithPerformerParentTag,
},
[]int{
sceneIdxWithPerformer,
sceneIdxWithPerformerTag,
sceneIdxWithPerformerTwoTags,
sceneIdxWithTwoPerformerTag,
},
false,
},
{
"includes all",
nil,
&models.SceneFilterType{
PerformerTags: &models.HierarchicalMultiCriterionInput{
Value: []string{
strconv.Itoa(tagIDs[tagIdx1WithPerformer]),
strconv.Itoa(tagIDs[tagIdx2WithPerformer]),
},
Modifier: models.CriterionModifierIncludesAll,
},
},
[]int{
sceneIdxWithPerformerTwoTags,
},
[]int{
sceneIdxWithPerformer,
sceneIdxWithPerformerTag,
sceneIdxWithTwoPerformerTag,
},
false,
},
{
"excludes performer tag tagIdx2WithPerformer",
nil,
&models.SceneFilterType{
PerformerTags: &models.HierarchicalMultiCriterionInput{
Modifier: models.CriterionModifierExcludes,
Value: []string{strconv.Itoa(tagIDs[tagIdx2WithPerformer])},
},
},
nil,
[]int{sceneIdxWithTwoPerformerTag},
false,
},
{
"excludes sub-tags",
nil,
&models.SceneFilterType{
PerformerTags: &models.HierarchicalMultiCriterionInput{
Value: []string{
strconv.Itoa(tagIDs[tagIdxWithParentAndChild]),
},
Depth: &allDepth,
Modifier: models.CriterionModifierExcludes,
},
},
[]int{
sceneIdxWithPerformer,
sceneIdxWithPerformerTag,
sceneIdxWithPerformerTwoTags,
sceneIdxWithTwoPerformerTag,
},
[]int{
sceneIdxWithPerformerParentTag,
},
false,
},
{
"is null",
nil,
&models.SceneFilterType{
PerformerTags: &models.HierarchicalMultiCriterionInput{
Modifier: models.CriterionModifierIsNull,
},
},
[]int{sceneIdx1WithPerformer},
[]int{sceneIdxWithPerformerTag},
false,
},
{
"not null",
nil,
&models.SceneFilterType{
PerformerTags: &models.HierarchicalMultiCriterionInput{
Modifier: models.CriterionModifierNotNull,
},
},
[]int{sceneIdxWithPerformerTag},
[]int{sceneIdx1WithPerformer},
false,
},
{
"equals",
nil,
&models.SceneFilterType{
PerformerTags: &models.HierarchicalMultiCriterionInput{
Modifier: models.CriterionModifierEquals,
Value: []string{
strconv.Itoa(tagIDs[tagIdx2WithPerformer]),
},
},
},
nil,
nil,
true,
},
{
"not equals",
nil,
&models.SceneFilterType{
PerformerTags: &models.HierarchicalMultiCriterionInput{
Modifier: models.CriterionModifierNotEquals,
Value: []string{
strconv.Itoa(tagIDs[tagIdx2WithPerformer]),
},
},
},
nil,
nil,
true,
},
}
q := getSceneStringValue(sceneIdxWithPerformerTwoTags, titleField)
findFilter := models.FindFilterType{
Q: &q,
}
for _, tt := range tests {
runWithRollbackTxn(t, tt.name, func(t *testing.T, ctx context.Context) {
assert := assert.New(t)
scenes = queryScene(ctx, t, sqb, &sceneFilter, &findFilter)
assert.Len(t, scenes, 0)
results, err := db.Scene.Query(ctx, models.SceneQueryOptions{
SceneFilter: tt.filter,
QueryOptions: models.QueryOptions{
FindFilter: tt.findFilter,
},
})
if (err != nil) != tt.wantErr {
t.Errorf("SceneStore.Query() error = %v, wantErr %v", err, tt.wantErr)
return
}
tagCriterion = models.HierarchicalMultiCriterionInput{
Modifier: models.CriterionModifierIsNull,
}
q = getSceneStringValue(sceneIdx1WithPerformer, titleField)
include := indexesToIDs(sceneIDs, tt.includeIdxs)
exclude := indexesToIDs(sceneIDs, tt.excludeIdxs)
scenes = queryScene(ctx, t, sqb, &sceneFilter, &findFilter)
assert.Len(t, scenes, 1)
assert.Equal(t, sceneIDs[sceneIdx1WithPerformer], scenes[0].ID)
q = getSceneStringValue(sceneIdxWithPerformerTag, titleField)
scenes = queryScene(ctx, t, sqb, &sceneFilter, &findFilter)
assert.Len(t, scenes, 0)
tagCriterion.Modifier = models.CriterionModifierNotNull
scenes = queryScene(ctx, t, sqb, &sceneFilter, &findFilter)
assert.Len(t, scenes, 1)
assert.Equal(t, sceneIDs[sceneIdxWithPerformerTag], scenes[0].ID)
q = getSceneStringValue(sceneIdx1WithPerformer, titleField)
scenes = queryScene(ctx, t, sqb, &sceneFilter, &findFilter)
assert.Len(t, scenes, 0)
return nil
})
for _, i := range include {
assert.Contains(results.IDs, i)
}
for _, e := range exclude {
assert.NotContains(results.IDs, e)
}
})
}
}
func TestSceneQueryStudio(t *testing.T) {
@@ -3561,6 +3855,30 @@ func TestSceneQueryStudio(t *testing.T) {
[]int{sceneIDs[sceneIdxWithGallery]},
false,
},
{
"equals",
"",
models.HierarchicalMultiCriterionInput{
Value: []string{
strconv.Itoa(studioIDs[studioIdxWithScene]),
},
Modifier: models.CriterionModifierEquals,
},
[]int{sceneIDs[sceneIdxWithStudio]},
false,
},
{
"not equals",
getSceneStringValue(sceneIdxWithStudio, titleField),
models.HierarchicalMultiCriterionInput{
Value: []string{
strconv.Itoa(studioIDs[studioIdxWithScene]),
},
Modifier: models.CriterionModifierNotEquals,
},
[]int{},
false,
},
}
qb := db.Scene