Fix tag query performance problems (#657)

* Fix sql tracing
* Disable query by marker count
* Disable unit test

pkg/manager/config/config.go

@@ -344,7 +344,7 @@ func GetLogLevel() string {
 	const defaultValue = "Info"
 
 	value := viper.GetString(LogLevel)
-	if value != "Debug" && value != "Info" && value != "Warning" && value != "Error" {
+	if value != "Debug" && value != "Info" && value != "Warning" && value != "Error" && value != "Trace" {
 		value = defaultValue
 	}
 

pkg/models/querybuilder_sql.go

@@ -311,9 +311,12 @@ func executeFindQuery(tableName string, body string, args []interface{}, sortAnd
 	}
 
 	countQuery := buildCountQuery(body)
-	countResult, countErr := runCountQuery(countQuery, args)
-
 	idsQuery := body + sortAndPagination
+
+	// Perform query and fetch result
+	logger.Tracef("SQL: %s, args: %v", idsQuery, args)
+
+	countResult, countErr := runCountQuery(countQuery, args)
 	idsResult, idsErr := runIdsQuery(idsQuery, args)
 
 	if countErr != nil {
@@ -325,9 +328,6 @@ func executeFindQuery(tableName string, body string, args []interface{}, sortAnd
 		panic(idsErr)
 	}
 
-	// Perform query and fetch result
-	logger.Tracef("SQL: %s, args: %v", idsQuery, args)
-
 	return idsResult, countResult
 }
 

pkg/models/querybuilder_tag.go

@@ -161,11 +161,23 @@ func (qb *TagQueryBuilder) Query(tagFilter *TagFilterType, findFilter *FindFilte
 	}
 
 	query.body = selectDistinctIDs(tagTable)
+
+	/*
+		query.body += `
+		left join tags_image on tags_image.tag_id = tags.id
+		left join scenes_tags on scenes_tags.tag_id = tags.id
+		left join scene_markers_tags on scene_markers_tags.tag_id = tags.id
+		left join scene_markers on scene_markers.primary_tag_id = tags.id OR scene_markers.id = scene_markers_tags.scene_marker_id
+		left join scenes on scenes_tags.scene_id = scenes.id`
+	*/
+
+	// the presence of joining on scene_markers.primary_tag_id and scene_markers_tags.tag_id
+	// appears to confuse sqlite and causes serious performance issues.
+	// Disabling querying/sorting on marker count for now.
+
 	query.body += ` 
 	left join tags_image on tags_image.tag_id = tags.id
 	left join scenes_tags on scenes_tags.tag_id = tags.id
-	left join scene_markers_tags on scene_markers_tags.tag_id = tags.id
-	left join scene_markers on scene_markers.primary_tag_id = tags.id OR scene_markers.id = scene_markers_tags.scene_marker_id
 	left join scenes on scenes_tags.scene_id = scenes.id`
 
 	if q := findFilter.Q; q != nil && *q != "" {
@@ -192,13 +204,13 @@ func (qb *TagQueryBuilder) Query(tagFilter *TagFilterType, findFilter *FindFilte
 		}
 	}
 
-	if markerCount := tagFilter.MarkerCount; markerCount != nil {
+	// if markerCount := tagFilter.MarkerCount; markerCount != nil {
-		clause, count := getIntCriterionWhereClause("count(distinct scene_markers.id)", *markerCount)
+	// 	clause, count := getIntCriterionWhereClause("count(distinct scene_markers.id)", *markerCount)
-		query.addHaving(clause)
+	// 	query.addHaving(clause)
-		if count == 1 {
+	// 	if count == 1 {
-			query.addArg(markerCount.Value)
+	// 		query.addArg(markerCount.Value)
-		}
+	// 	}
-	}
+	// }
 
 	query.sortAndPagination = qb.getTagSort(findFilter) + getPagination(findFilter)
 	idsResult, countResult := query.executeFind()

pkg/models/querybuilder_tag_test.go

@@ -169,24 +169,26 @@ func verifyTagSceneCount(t *testing.T, sceneCountCriterion models.IntCriterionIn
 	}
 }
 
-func TestTagQueryMarkerCount(t *testing.T) {
+// disabled due to performance issues
-	countCriterion := models.IntCriterionInput{
-		Value:    1,
-		Modifier: models.CriterionModifierEquals,
-	}
 
-	verifyTagMarkerCount(t, countCriterion)
+// func TestTagQueryMarkerCount(t *testing.T) {
+// 	countCriterion := models.IntCriterionInput{
+// 		Value:    1,
+// 		Modifier: models.CriterionModifierEquals,
+// 	}
 
-	countCriterion.Modifier = models.CriterionModifierNotEquals
+// 	verifyTagMarkerCount(t, countCriterion)
-	verifyTagMarkerCount(t, countCriterion)
 
-	countCriterion.Modifier = models.CriterionModifierLessThan
+// 	countCriterion.Modifier = models.CriterionModifierNotEquals
-	verifyTagMarkerCount(t, countCriterion)
+// 	verifyTagMarkerCount(t, countCriterion)
 
-	countCriterion.Value = 0
+// 	countCriterion.Modifier = models.CriterionModifierLessThan
-	countCriterion.Modifier = models.CriterionModifierGreaterThan
+// 	verifyTagMarkerCount(t, countCriterion)
-	verifyTagMarkerCount(t, countCriterion)
+
-}
+// 	countCriterion.Value = 0
+// 	countCriterion.Modifier = models.CriterionModifierGreaterThan
+// 	verifyTagMarkerCount(t, countCriterion)
+// }
 
 func verifyTagMarkerCount(t *testing.T, markerCountCriterion models.IntCriterionInput) {
 	qb := models.NewTagQueryBuilder()

ui/v2.5/src/models/list-filter/filter.ts

@@ -211,13 +211,19 @@ export class ListFilterModel {
         break;
       case FilterMode.Tags:
         this.sortBy = "name";
-        this.sortByOptions = ["name", "scenes_count", "scene_markers_count"];
+        // scene markers count has been disabled for now due to performance
+        // issues
+        this.sortByOptions = [
+          "name",
+          "scenes_count" /* , "scene_markers_count"*/,
+        ];
         this.displayModeOptions = [DisplayMode.Grid, DisplayMode.List];
         this.criterionOptions = [
           new NoneCriterionOption(),
           new TagIsMissingCriterionOption(),
           ListFilterModel.createCriterionOption("scene_count"),
-          ListFilterModel.createCriterionOption("marker_count"),
+          // marker count has been disabled for now due to performance issues
+          // ListFilterModel.createCriterionOption("marker_count"),
         ];
         break;
       default:
@@ -652,14 +658,15 @@ export class ListFilterModel {
           };
           break;
         }
-        case "marker_count": {
+        // disabled due to performance issues
-          const countCrit = criterion as NumberCriterion;
+        // case "marker_count": {
-          result.marker_count = {
+        //   const countCrit = criterion as NumberCriterion;
-            value: countCrit.value,
+        //   result.marker_count = {
-            modifier: countCrit.modifier,
+        //     value: countCrit.value,
-          };
+        //     modifier: countCrit.modifier,
-          break;
+        //   };
-        }
+        //   break;
+        // }
         // no default
       }
     });