diff --git a/graphql/documents/queries/scene.graphql b/graphql/documents/queries/scene.graphql index 83fb58cab..add95cca1 100644 --- a/graphql/documents/queries/scene.graphql +++ b/graphql/documents/queries/scene.graphql @@ -30,4 +30,24 @@ query FindScene($id: ID!, $checksum: String) { ...SceneMarkerData } } +} + +query ParseSceneFilenames($filter: FindFilterType!, $config: SceneParserInput!) { + parseSceneFilenames(filter: $filter, config: $config) { + count + results { + scene { + ...SlimSceneData + } + title + details + url + date + rating + studio_id + gallery_id + performer_ids + tag_ids + } + } } \ No newline at end of file diff --git a/graphql/schema/schema.graphql b/graphql/schema/schema.graphql index c03fc6df4..60f02cd6c 100644 --- a/graphql/schema/schema.graphql +++ b/graphql/schema/schema.graphql @@ -7,6 +7,8 @@ type Query { findScenesByPathRegex(filter: FindFilterType): FindScenesResultType! + parseSceneFilenames(filter: FindFilterType, config: SceneParserInput!): SceneParserResultType! + """A function which queries SceneMarker objects""" findSceneMarkers(scene_marker_filter: SceneMarkerFilterType filter: FindFilterType): FindSceneMarkersResultType! diff --git a/graphql/schema/types/scene.graphql b/graphql/schema/types/scene.graphql index 18bd19e32..e526dca14 100644 --- a/graphql/schema/types/scene.graphql +++ b/graphql/schema/types/scene.graphql @@ -76,4 +76,28 @@ input SceneDestroyInput { type FindScenesResultType { count: Int! scenes: [Scene!]! +} + +input SceneParserInput { + ignoreWords: [String!], + whitespaceCharacters: String, + capitalizeTitle: Boolean +} + +type SceneParserResult { + scene: Scene! + title: String + details: String + url: String + date: String + rating: Int + studio_id: ID + gallery_id: ID + performer_ids: [ID!] + tag_ids: [ID!] +} + +type SceneParserResultType { + count: Int! + results: [SceneParserResult!]! } \ No newline at end of file diff --git a/pkg/api/resolver_query_find_scene.go b/pkg/api/resolver_query_find_scene.go index 38feecbd8..c06d25148 100644 --- a/pkg/api/resolver_query_find_scene.go +++ b/pkg/api/resolver_query_find_scene.go @@ -2,8 +2,10 @@ package api import ( "context" - "github.com/stashapp/stash/pkg/models" "strconv" + + "github.com/stashapp/stash/pkg/manager" + "github.com/stashapp/stash/pkg/models" ) func (r *queryResolver) FindScene(ctx context.Context, id *string, checksum *string) (*models.Scene, error) { @@ -37,3 +39,18 @@ func (r *queryResolver) FindScenesByPathRegex(ctx context.Context, filter *model Scenes: scenes, }, nil } + +func (r *queryResolver) ParseSceneFilenames(ctx context.Context, filter *models.FindFilterType, config models.SceneParserInput) (*models.SceneParserResultType, error) { + parser := manager.NewSceneFilenameParser(filter, config) + + result, count, err := parser.Parse() + + if err != nil { + return nil, err + } + + return &models.SceneParserResultType{ + Count: count, + Results: result, + }, nil +} diff --git a/pkg/manager/filename_parser.go b/pkg/manager/filename_parser.go new file mode 100644 index 000000000..f586d8ff9 --- /dev/null +++ b/pkg/manager/filename_parser.go @@ -0,0 +1,616 @@ +package manager + +import ( + "database/sql" + "errors" + "path/filepath" + "regexp" + "strconv" + "strings" + + "github.com/stashapp/stash/pkg/models" + + "github.com/jmoiron/sqlx" +) + +type parserField struct { + field string + fieldRegex *regexp.Regexp + regex string + isFullDateField bool + isCaptured bool +} + +func newParserField(field string, regex string, captured bool) parserField { + ret := parserField{ + field: field, + isFullDateField: false, + isCaptured: captured, + } + + ret.fieldRegex, _ = regexp.Compile(`\{` + ret.field + `\}`) + + regexStr := regex + + if captured { + regexStr = "(" + regexStr + ")" + } + ret.regex = regexStr + + return ret +} + +func newFullDateParserField(field string, regex string) parserField { + ret := newParserField(field, regex, true) + ret.isFullDateField = true + return ret +} + +func (f parserField) replaceInPattern(pattern string) string { + return string(f.fieldRegex.ReplaceAllString(pattern, f.regex)) +} + +func (f parserField) getFieldPattern() string { + return "{" + f.field + "}" +} + +var validFields map[string]parserField +var escapeCharRE *regexp.Regexp +var capitalizeTitleRE *regexp.Regexp +var multiWSRE *regexp.Regexp +var delimiterRE *regexp.Regexp + +func compileREs() { + const escapeCharPattern = `([\-\.\(\)\[\]])` + escapeCharRE = regexp.MustCompile(escapeCharPattern) + + const capitaliseTitlePattern = `(?:^| )\w` + capitalizeTitleRE = regexp.MustCompile(capitaliseTitlePattern) + + const multiWSPattern = ` {2,}` + multiWSRE = regexp.MustCompile(multiWSPattern) + + const delimiterPattern = `(?:\.|-|_)` + delimiterRE = regexp.MustCompile(delimiterPattern) +} + +func initParserFields() { + if validFields != nil { + return + } + + ret := make(map[string]parserField) + + ret["title"] = newParserField("title", ".*", true) + ret["ext"] = newParserField("ext", ".*$", false) + + //I = new ParserField("i", undefined, "Matches any ignored word", false); + + ret["d"] = newParserField("d", `(?:\.|-|_)`, false) + ret["performer"] = newParserField("performer", ".*", true) + ret["studio"] = newParserField("studio", ".*", true) + ret["tag"] = newParserField("tag", ".*", true) + + // date fields + ret["date"] = newParserField("date", `\d{4}-\d{2}-\d{2}`, true) + ret["yyyy"] = newParserField("yyyy", `\d{4}`, true) + ret["yy"] = newParserField("yy", `\d{2}`, true) + ret["mm"] = newParserField("mm", `\d{2}`, true) + ret["dd"] = newParserField("dd", `\d{2}`, true) + ret["yyyymmdd"] = newFullDateParserField("yyyymmdd", `\d{8}`) + ret["yymmdd"] = newFullDateParserField("yymmdd", `\d{6}`) + ret["ddmmyyyy"] = newFullDateParserField("ddmmyyyy", `\d{8}`) + ret["ddmmyy"] = newFullDateParserField("ddmmyy", `\d{6}`) + ret["mmddyyyy"] = newFullDateParserField("mmddyyyy", `\d{8}`) + ret["mmddyy"] = newFullDateParserField("mmddyy", `\d{6}`) + + validFields = ret +} + +func replacePatternWithRegex(pattern string, ignoreWords []string) string { + initParserFields() + + for _, field := range validFields { + pattern = field.replaceInPattern(pattern) + } + + ignoreClause := getIgnoreClause(ignoreWords) + ignoreField := newParserField("i", ignoreClause, false) + pattern = ignoreField.replaceInPattern(pattern) + + return pattern +} + +type parseMapper struct { + fields []string + regexString string + regex *regexp.Regexp +} + +func getIgnoreClause(ignoreFields []string) string { + if len(ignoreFields) == 0 { + return "" + } + + var ignoreClauses []string + + for _, v := range ignoreFields { + newVal := string(escapeCharRE.ReplaceAllString(v, `\$1`)) + newVal = strings.TrimSpace(newVal) + newVal = "(?:" + newVal + ")" + ignoreClauses = append(ignoreClauses, newVal) + } + + return "(?:" + strings.Join(ignoreClauses, "|") + ")" +} + +func newParseMapper(pattern string, ignoreFields []string) (*parseMapper, error) { + ret := &parseMapper{} + + // escape control characters + regex := escapeCharRE.ReplaceAllString(pattern, `\$1`) + + // replace {} with wildcard + braceRE := regexp.MustCompile(`\{\}`) + regex = braceRE.ReplaceAllString(regex, ".*") + + // replace all known fields with applicable regexes + regex = replacePatternWithRegex(regex, ignoreFields) + + ret.regexString = regex + + // make case insensitive + regex = "(?i)" + regex + + var err error + + ret.regex, err = regexp.Compile(regex) + + if err != nil { + return nil, err + } + + // find invalid fields + invalidRE := regexp.MustCompile(`\{[A-Za-z]+\}`) + foundInvalid := invalidRE.FindAllString(regex, -1) + if len(foundInvalid) > 0 { + return nil, errors.New("Invalid fields: " + strings.Join(foundInvalid, ", ")) + } + + fieldExtractor := regexp.MustCompile(`\{([A-Za-z]+)\}`) + + result := fieldExtractor.FindAllStringSubmatch(pattern, -1) + + var fields []string + for _, v := range result { + field := v[1] + + // only add to fields if it is captured + parserField, found := validFields[field] + if found && parserField.isCaptured { + fields = append(fields, field) + } + } + + ret.fields = fields + + return ret, nil +} + +type sceneHolder struct { + scene *models.Scene + result *models.Scene + yyyy string + mm string + dd string + performers []string + studio string + tags []string +} + +func newSceneHolder(scene *models.Scene) *sceneHolder { + sceneCopy := models.Scene{ + ID: scene.ID, + Checksum: scene.Checksum, + Path: scene.Path, + } + ret := sceneHolder{ + scene: scene, + result: &sceneCopy, + } + + return &ret +} + +func validateDate(dateStr string) bool { + splits := strings.Split(dateStr, "-") + if len(splits) != 3 { + return false + } + + year, _ := strconv.Atoi(splits[0]) + month, _ := strconv.Atoi(splits[1]) + d, _ := strconv.Atoi(splits[2]) + + // assume year must be between 1900 and 2100 + if year < 1900 || year > 2100 { + return false + } + + if month < 1 || month > 12 { + return false + } + + // not checking individual months to ensure date is in the correct range + if d < 1 || d > 31 { + return false + } + + return true +} + +func (h *sceneHolder) setDate(field *parserField, value string) { + yearIndex := 0 + yearLength := len(strings.Split(field.field, "y")) - 1 + dateIndex := 0 + monthIndex := 0 + + switch field.field { + case "yyyymmdd", "yymmdd": + monthIndex = yearLength + dateIndex = monthIndex + 2 + case "ddmmyyyy", "ddmmyy": + monthIndex = 2 + yearIndex = monthIndex + 2 + case "mmddyyyy", "mmddyy": + dateIndex = monthIndex + 2 + yearIndex = dateIndex + 2 + } + + yearValue := value[yearIndex : yearIndex+yearLength] + monthValue := value[monthIndex : monthIndex+2] + dateValue := value[dateIndex : dateIndex+2] + + fullDate := yearValue + "-" + monthValue + "-" + dateValue + + // ensure the date is valid + // only set if new value is different from the old + if validateDate(fullDate) && h.scene.Date.String != fullDate { + h.result.Date = models.SQLiteDate{ + String: fullDate, + Valid: true, + } + } +} + +func (h *sceneHolder) setField(field parserField, value interface{}) { + if field.isFullDateField { + h.setDate(&field, value.(string)) + return + } + + switch field.field { + case "title": + h.result.Title = sql.NullString{ + String: value.(string), + Valid: true, + } + case "date": + if validateDate(value.(string)) { + h.result.Date = models.SQLiteDate{ + String: value.(string), + Valid: true, + } + } + case "performer": + // add performer to list + h.performers = append(h.performers, value.(string)) + case "studio": + h.studio = value.(string) + case "tag": + h.tags = append(h.tags, value.(string)) + case "yyyy": + h.yyyy = value.(string) + break + case "yy": + v := value.(string) + v = "20" + v + h.yyyy = v + break + case "mm": + h.mm = value.(string) + break + case "dd": + h.dd = value.(string) + break + } +} + +func (h *sceneHolder) postParse() { + // set the date if the components are set + if h.yyyy != "" && h.mm != "" && h.dd != "" { + fullDate := h.yyyy + "-" + h.mm + "-" + h.dd + h.setField(validFields["date"], fullDate) + } +} + +func (m parseMapper) parse(scene *models.Scene) *sceneHolder { + // perform matching on basename + // TODO - may want to handle full path at some point + filename := filepath.Base(scene.Path) + + result := m.regex.FindStringSubmatch(filename) + + if len(result) == 0 { + return nil + } + + initParserFields() + + sceneHolder := newSceneHolder(scene) + + for index, match := range result { + if index == 0 { + // skip entire match + continue + } + + field := m.fields[index-1] + parserField, found := validFields[field] + if found { + sceneHolder.setField(parserField, match) + } + } + + sceneHolder.postParse() + + return sceneHolder +} + +type performerQueryer interface { + FindByNames(names []string, tx *sqlx.Tx) ([]*models.Performer, error) +} + +type sceneQueryer interface { + QueryByPathRegex(findFilter *models.FindFilterType) ([]*models.Scene, int) +} + +type tagQueryer interface { + FindByName(name string, tx *sqlx.Tx) (*models.Tag, error) +} + +type studioQueryer interface { + FindByName(name string, tx *sqlx.Tx) (*models.Studio, error) +} + +type SceneFilenameParser struct { + Pattern string + ParserInput models.SceneParserInput + Filter *models.FindFilterType + whitespaceRE *regexp.Regexp + performerCache map[string]*models.Performer + studioCache map[string]*models.Studio + tagCache map[string]*models.Tag + + performerQuery performerQueryer + sceneQuery sceneQueryer + tagQuery tagQueryer + studioQuery studioQueryer +} + +func NewSceneFilenameParser(filter *models.FindFilterType, config models.SceneParserInput) *SceneFilenameParser { + p := &SceneFilenameParser{ + Pattern: *filter.Q, + ParserInput: config, + Filter: filter, + } + + p.performerCache = make(map[string]*models.Performer) + p.studioCache = make(map[string]*models.Studio) + p.tagCache = make(map[string]*models.Tag) + + p.initWhiteSpaceRegex() + + performerQuery := models.NewPerformerQueryBuilder() + p.performerQuery = &performerQuery + + sceneQuery := models.NewSceneQueryBuilder() + p.sceneQuery = &sceneQuery + + tagQuery := models.NewTagQueryBuilder() + p.tagQuery = &tagQuery + + studioQuery := models.NewStudioQueryBuilder() + p.studioQuery = &studioQuery + + return p +} + +func (p *SceneFilenameParser) initWhiteSpaceRegex() { + compileREs() + + wsChars := "" + if p.ParserInput.WhitespaceCharacters != nil { + wsChars = *p.ParserInput.WhitespaceCharacters + wsChars = strings.TrimSpace(wsChars) + } + + if len(wsChars) > 0 { + wsRegExp := escapeCharRE.ReplaceAllString(wsChars, `\$1`) + wsRegExp = "[" + wsRegExp + "]" + p.whitespaceRE = regexp.MustCompile(wsRegExp) + } +} + +func (p *SceneFilenameParser) Parse() ([]*models.SceneParserResult, int, error) { + // perform the query to find the scenes + mapper, err := newParseMapper(p.Pattern, p.ParserInput.IgnoreWords) + + if err != nil { + return nil, 0, err + } + + p.Filter.Q = &mapper.regexString + + scenes, total := p.sceneQuery.QueryByPathRegex(p.Filter) + + ret := p.parseScenes(scenes, mapper) + + return ret, total, nil +} + +func (p *SceneFilenameParser) parseScenes(scenes []*models.Scene, mapper *parseMapper) []*models.SceneParserResult { + var ret []*models.SceneParserResult + for _, scene := range scenes { + sceneHolder := mapper.parse(scene) + + if sceneHolder != nil { + r := &models.SceneParserResult{ + Scene: scene, + } + p.setParserResult(*sceneHolder, r) + + if r != nil { + ret = append(ret, r) + } + } + } + + return ret +} + +func (p SceneFilenameParser) replaceWhitespaceCharacters(value string) string { + if p.whitespaceRE != nil { + value = p.whitespaceRE.ReplaceAllString(value, " ") + // remove consecutive spaces + value = multiWSRE.ReplaceAllString(value, " ") + } + + return value +} + +func (p *SceneFilenameParser) queryPerformer(performerName string) *models.Performer { + // massage the performer name + performerName = delimiterRE.ReplaceAllString(performerName, " ") + + // check cache first + if ret, found := p.performerCache[performerName]; found { + return ret + } + + // perform an exact match and grab the first + performers, _ := p.performerQuery.FindByNames([]string{performerName}, nil) + + var ret *models.Performer + if len(performers) > 0 { + ret = performers[0] + } + + // add result to cache + p.performerCache[performerName] = ret + + return ret +} + +func (p *SceneFilenameParser) queryStudio(studioName string) *models.Studio { + // massage the performer name + studioName = delimiterRE.ReplaceAllString(studioName, " ") + + // check cache first + if ret, found := p.studioCache[studioName]; found { + return ret + } + + ret, _ := p.studioQuery.FindByName(studioName, nil) + + // add result to cache + p.studioCache[studioName] = ret + + return ret +} + +func (p *SceneFilenameParser) queryTag(tagName string) *models.Tag { + // massage the performer name + tagName = delimiterRE.ReplaceAllString(tagName, " ") + + // check cache first + if ret, found := p.tagCache[tagName]; found { + return ret + } + + // match tag name exactly + ret, _ := p.tagQuery.FindByName(tagName, nil) + + // add result to cache + p.tagCache[tagName] = ret + + return ret +} + +func (p *SceneFilenameParser) setPerformers(h sceneHolder, result *models.SceneParserResult) { + // query for each performer + performersSet := make(map[int]bool) + for _, performerName := range h.performers { + if performerName != "" { + performer := p.queryPerformer(performerName) + if performer != nil { + if _, found := performersSet[performer.ID]; !found { + result.PerformerIds = append(result.PerformerIds, strconv.Itoa(performer.ID)) + performersSet[performer.ID] = true + } + } + } + } +} + +func (p *SceneFilenameParser) setTags(h sceneHolder, result *models.SceneParserResult) { + // query for each performer + tagsSet := make(map[int]bool) + for _, tagName := range h.tags { + if tagName != "" { + tag := p.queryTag(tagName) + if tag != nil { + if _, found := tagsSet[tag.ID]; !found { + result.TagIds = append(result.TagIds, strconv.Itoa(tag.ID)) + tagsSet[tag.ID] = true + } + } + } + } +} + +func (p *SceneFilenameParser) setStudio(h sceneHolder, result *models.SceneParserResult) { + // query for each performer + if h.studio != "" { + studio := p.queryStudio(h.studio) + if studio != nil { + studioId := strconv.Itoa(studio.ID) + result.StudioID = &studioId + } + } +} + +func (p *SceneFilenameParser) setParserResult(h sceneHolder, result *models.SceneParserResult) { + if h.result.Title.Valid { + title := h.result.Title.String + title = p.replaceWhitespaceCharacters(title) + + if p.ParserInput.CapitalizeTitle != nil && *p.ParserInput.CapitalizeTitle { + title = capitalizeTitleRE.ReplaceAllStringFunc(title, strings.ToUpper) + } + + result.Title = &title + } + + if h.result.Date.Valid { + result.Date = &h.result.Date.String + } + + if len(h.performers) > 0 { + p.setPerformers(h, result) + } + if len(h.tags) > 0 { + p.setTags(h, result) + } + p.setStudio(h, result) +} diff --git a/pkg/models/querybuilder_scene.go b/pkg/models/querybuilder_scene.go index bb9f7f97b..34ea9f5e5 100644 --- a/pkg/models/querybuilder_scene.go +++ b/pkg/models/querybuilder_scene.go @@ -242,7 +242,7 @@ func (qb *SceneQueryBuilder) Query(sceneFilter *SceneFilterType, findFilter *Fin for _, studioID := range studiosFilter.Value { args = append(args, studioID) } - + whereClause, havingClause := getMultiCriterionClause("studio", "", "studio_id", studiosFilter) whereClauses = appendClause(whereClauses, whereClause) havingClauses = appendClause(havingClauses, havingClause) @@ -274,14 +274,14 @@ func getMultiCriterionClause(table string, joinTable string, joinTableField stri havingClause := "" if criterion.Modifier == CriterionModifierIncludes { // includes any of the provided ids - whereClause = table + ".id IN "+ getInBinding(len(criterion.Value)) + whereClause = table + ".id IN " + getInBinding(len(criterion.Value)) } else if criterion.Modifier == CriterionModifierIncludesAll { // includes all of the provided ids - whereClause = table + ".id IN "+ getInBinding(len(criterion.Value)) + whereClause = table + ".id IN " + getInBinding(len(criterion.Value)) havingClause = "count(distinct " + table + ".id) IS " + strconv.Itoa(len(criterion.Value)) } else if criterion.Modifier == CriterionModifierExcludes { // excludes all of the provided ids - if (joinTable != "") { + if joinTable != "" { whereClause = "not exists (select " + joinTable + ".scene_id from " + joinTable + " where " + joinTable + ".scene_id = scenes.id and " + joinTable + "." + joinTableField + " in " + getInBinding(len(criterion.Value)) + ")" } else { whereClause = "not exists (select s.id from scenes as s where s.id = scenes.id and s." + joinTableField + " in " + getInBinding(len(criterion.Value)) + ")" @@ -302,7 +302,7 @@ func (qb *SceneQueryBuilder) QueryByPathRegex(findFilter *FindFilterType) ([]*Sc body := selectDistinctIDs("scenes") if q := findFilter.Q; q != nil && *q != "" { - whereClauses = append(whereClauses, "scenes.path regexp '" + *q + "'") + whereClauses = append(whereClauses, "scenes.path regexp '(?i)"+*q+"'") } sortAndPagination := qb.getSceneSort(findFilter) + getPagination(findFilter) @@ -363,4 +363,3 @@ func (qb *SceneQueryBuilder) queryScenes(query string, args []interface{}, tx *s return scenes, nil } - diff --git a/pkg/models/querybuilder_tag.go b/pkg/models/querybuilder_tag.go index 16a8bcb82..91c376d33 100644 --- a/pkg/models/querybuilder_tag.go +++ b/pkg/models/querybuilder_tag.go @@ -1,8 +1,9 @@ package models import ( - "errors" "database/sql" + "errors" + "github.com/jmoiron/sqlx" "github.com/stashapp/stash/pkg/database" ) @@ -134,6 +135,33 @@ func (qb *TagQueryBuilder) All() ([]*Tag, error) { return qb.queryTags(selectAll("tags")+qb.getTagSort(nil), nil, nil) } +func (qb *TagQueryBuilder) Query(findFilter *FindFilterType) ([]*Tag, int) { + if findFilter == nil { + findFilter = &FindFilterType{} + } + + var whereClauses []string + var havingClauses []string + var args []interface{} + body := selectDistinctIDs("tags") + + if q := findFilter.Q; q != nil && *q != "" { + searchColumns := []string{"tags.name"} + whereClauses = append(whereClauses, getSearch(searchColumns, *q)) + } + + sortAndPagination := qb.getTagSort(findFilter) + getPagination(findFilter) + idsResult, countResult := executeFindQuery("tags", body, args, sortAndPagination, whereClauses, havingClauses) + + var tags []*Tag + for _, id := range idsResult { + tag, _ := qb.Find(id, nil) + tags = append(tags, tag) + } + + return tags, countResult +} + func (qb *TagQueryBuilder) getTagSort(findFilter *FindFilterType) string { var sort string var direction string diff --git a/ui/v2/src/components/scenes/SceneFilenameParser.tsx b/ui/v2/src/components/scenes/SceneFilenameParser.tsx index 6f27db6b7..0a845e212 100644 --- a/ui/v2/src/components/scenes/SceneFilenameParser.tsx +++ b/ui/v2/src/components/scenes/SceneFilenameParser.tsx @@ -10,8 +10,11 @@ import { H5, MenuItem, HTMLSelect, + TagInput, + Tree, + ITreeNode, } from "@blueprintjs/core"; -import React, { FunctionComponent, useEffect, useState } from "react"; +import React, { FunctionComponent, useEffect, useState, useRef } from "react"; import { IBaseProps } from "../../models"; import { StashService } from "../../core/StashService"; import * as GQL from "../../core/generated-graphql"; @@ -22,6 +25,8 @@ import { ToastUtils } from "../../utils/toasts"; import { ErrorUtils } from "../../utils/errors"; import { Pagination } from "../list/Pagination"; import { Select, ItemRenderer, ItemPredicate } from "@blueprintjs/select"; +import { FilterMultiSelect } from "../select/FilterMultiSelect"; +import { FilterSelect } from "../select/FilterSelect"; interface IProps extends IBaseProps {} @@ -34,37 +39,22 @@ class ParserResult { this.originalValue = v; this.value = v; } + + public setValue(v : Maybe) { + if (!!v) { + this.value = v; + this.set = !_.isEqual(this.value, this.originalValue); + } + } } class ParserField { public field : string; - public fieldRegex: RegExp; - public regex : string; public helperText? : string; - constructor(field: string, regex?: string, helperText?: string, captured?: boolean) { - if (regex === undefined) { - regex = ".*"; - } - - if (captured === undefined) { - captured = true; - } - + constructor(field: string, helperText?: string) { this.field = field; this.helperText = helperText; - - this.fieldRegex = new RegExp("\\{" + this.field + "\\}", "g"); - - var regexStr = regex; - if (captured) { - regexStr = "(" + regexStr + ")"; - } - this.regex = regexStr; - } - - public replaceInPattern(pattern : string) { - return pattern.replace(this.fieldRegex, this.regex); } public getFieldPattern() { @@ -72,29 +62,36 @@ class ParserField { } static Title = new ParserField("title"); - static Ext = new ParserField("ext", ".*$", "File extension", false); + static Ext = new ParserField("ext", "File extension"); - static I = new ParserField("i", undefined, "Matches any ignored word", false); - static D = new ParserField("d", "(?:\\.|-|_)", "Matches any delimiter (.-_)", false); + static I = new ParserField("i", "Matches any ignored word"); + static D = new ParserField("d", "Matches any delimiter (.-_)"); + + static Performer = new ParserField("performer"); + static Studio = new ParserField("studio"); + static Tag = new ParserField("tag"); // date fields - static Date = new ParserField("date", "\\d{4}-\\d{2}-\\d{2}", "YYYY-MM-DD"); - static YYYY = new ParserField("yyyy", "\\d{4}", "Year"); - static YY = new ParserField("yy", "\\d{2}", "Year (20YY)"); - static MM = new ParserField("mm", "\\d{2}", "Two digit month"); - static DD = new ParserField("dd", "\\d{2}", "Two digit date"); - static YYYYMMDD = new ParserField("yyyymmdd", "\\d{8}"); - static YYMMDD = new ParserField("yymmdd", "\\d{6}"); - static DDMMYYYY = new ParserField("ddmmyyyy", "\\d{8}"); - static DDMMYY = new ParserField("ddmmyy", "\\d{6}"); - static MMDDYYYY = new ParserField("mmddyyyy", "\\d{8}"); - static MMDDYY = new ParserField("mmddyy", "\\d{6}"); + static Date = new ParserField("date", "YYYY-MM-DD"); + static YYYY = new ParserField("yyyy", "Year"); + static YY = new ParserField("yy", "Year (20YY)"); + static MM = new ParserField("mm", "Two digit month"); + static DD = new ParserField("dd", "Two digit date"); + static YYYYMMDD = new ParserField("yyyymmdd"); + static YYMMDD = new ParserField("yymmdd"); + static DDMMYYYY = new ParserField("ddmmyyyy"); + static DDMMYY = new ParserField("ddmmyy"); + static MMDDYYYY = new ParserField("mmddyyyy"); + static MMDDYY = new ParserField("mmddyy"); static validFields = [ ParserField.Title, ParserField.Ext, ParserField.D, ParserField.I, + ParserField.Performer, + ParserField.Studio, + ParserField.Tag, ParserField.Date, ParserField.YYYY, ParserField.YY, @@ -116,158 +113,68 @@ class ParserField { ParserField.MMDDYYYY, ParserField.MMDDYY ]; - - public static getParserField(field: string) { - return ParserField.validFields.find((f) => { - return f.field === field; - }); - } - - public static isValidField(field : string) { - return !!ParserField.getParserField(field); - } - - public static isFullDateField(field : ParserField) { - return ParserField.fullDateFields.includes(field); - } - - public static replacePatternWithRegex(pattern: string) { - ParserField.validFields.forEach((field) => { - pattern = field.replaceInPattern(pattern); - }); - return pattern; - } } - class SceneParserResult { public id: string; public filename: string; public title: ParserResult = new ParserResult(); public date: ParserResult = new ParserResult(); - public yyyy : ParserResult = new ParserResult(); - public mm : ParserResult = new ParserResult(); - public dd : ParserResult = new ParserResult(); - + public studio: ParserResult = new ParserResult(); public studioId: ParserResult = new ParserResult(); - public tags: ParserResult = new ParserResult(); + public tags: ParserResult = new ParserResult(); + public tagIds: ParserResult = new ParserResult(); + public performers: ParserResult = new ParserResult(); public performerIds: ParserResult = new ParserResult(); public scene : SlimSceneDataFragment; - constructor(scene : SlimSceneDataFragment) { - this.id = scene.id; - this.filename = TextUtils.fileNameFromPath(scene.path); - this.title.setOriginalValue(scene.title); - this.date.setOriginalValue(scene.date); + constructor(result : GQL.ParseSceneFilenamesResults) { + this.scene = result.scene; - this.scene = scene; - } + this.id = this.scene.id; + this.filename = TextUtils.fileNameFromPath(this.scene.path); + this.title.setOriginalValue(this.scene.title); + this.date.setOriginalValue(this.scene.date); + this.performerIds.setOriginalValue(this.scene.performers.map((p) => p.id)); + this.performers.setOriginalValue(this.scene.performers); + this.tagIds.setOriginalValue(this.scene.tags.map((t) => t.id)); + this.tags.setOriginalValue(this.scene.tags); + this.studioId.setOriginalValue(this.scene.studio ? this.scene.studio.id : undefined); + this.studio.setOriginalValue(this.scene.studio); - public static validateDate(dateStr: string) { - var splits = dateStr.split("-"); - if (splits.length != 3) { - return false; - } - - var year = parseInt(splits[0]); - var month = parseInt(splits[1]); - var d = parseInt(splits[2]); + this.title.setValue(result.title); + this.date.setValue(result.date); + this.performerIds.setValue(result.performer_ids); + this.tagIds.setValue(result.tag_ids); + this.studioId.setValue(result.studio_id); - var date = new Date(); - date.setMonth(month - 1); - date.setDate(d); - - // assume year must be between 1900 and 2100 - if (year < 1900 || year > 2100) { - return false; + if (result.performer_ids) { + this.performers.setValue(result.performer_ids.map((p) => { + return { + id: p, + name: "", + favorite: false, + image_path: "" + }; + })); } - if (month < 1 || month > 12) { - return false; + if (result.tag_ids) { + this.tags.setValue(result.tag_ids.map((t) => { + return { + id: t, + name: "", + }; + })); } - // not checking individual months to ensure date is in the correct range - if (d < 1 || d > 31) { - return false; - } - - return true; - } - - private setDate(field: ParserField, value: string) { - var yearIndex = 0; - var yearLength = field.field.split("y").length - 1; - var dateIndex = 0; - var monthIndex = 0; - - switch (field) { - case ParserField.YYYYMMDD: - case ParserField.YYMMDD: - monthIndex = yearLength; - dateIndex = monthIndex + 2; - break; - case ParserField.DDMMYYYY: - case ParserField.DDMMYY: - monthIndex = 2; - yearIndex = monthIndex + 2; - break; - case ParserField.MMDDYYYY: - case ParserField.MMDDYY: - dateIndex = monthIndex + 2; - yearIndex = dateIndex + 2; - break; - } - - var yearValue = value.substring(yearIndex, yearIndex + yearLength); - var monthValue = value.substring(monthIndex, monthIndex + 2); - var dateValue = value.substring(dateIndex, dateIndex + 2); - - var fullDate = yearValue + "-" + monthValue + "-" + dateValue; - - // ensure the date is valid - // only set if new value is different from the old - if (SceneParserResult.validateDate(fullDate) && this.date.originalValue !== fullDate) { - this.date.set = true; - this.date.value = fullDate - } - } - - public setField(field: ParserField, value: any) { - var parserResult : ParserResult | undefined = undefined; - - if (ParserField.isFullDateField(field)) { - this.setDate(field, value); - return; - } - - switch (field) { - case ParserField.Title: - parserResult = this.title; - break; - case ParserField.Date: - parserResult = this.date; - break; - case ParserField.YYYY: - parserResult = this.yyyy; - break; - case ParserField.YY: - parserResult = this.yyyy; - value = "20" + value; - break; - case ParserField.MM: - parserResult = this.mm; - break; - case ParserField.DD: - parserResult = this.dd; - break; - } - // TODO - other fields - - // only set if different from original value - if (!!parserResult && parserResult.originalValue !== value) { - parserResult.set = true; - parserResult.value = value; + if (result.studio_id) { + this.studio.setValue({ + id: result.studio_id, + name: "", + image_path: "" + }); } } @@ -304,97 +211,21 @@ class SceneParserResult { } }; -class ParseMapper { - public fields : string[] = []; - public regex : string = ""; - public matched : boolean = true; - - constructor(pattern : string, ignoreFields : string[]) { - // escape control characters - this.regex = pattern.replace(/([\-\.\(\)\[\]])/g, "\\$1"); - - // replace {} with wildcard - this.regex = this.regex.replace(/\{\}/g, ".*"); - - // set ignore fields - ignoreFields = ignoreFields.map((s) => s.replace(/([\-\.\(\)\[\]])/g, "\\$1").trim()); - var ignoreClause = ignoreFields.map((s) => "(?:" + s + ")").join("|"); - ignoreClause = "(?:" + ignoreClause + ")"; - - ParserField.I.regex = ignoreClause; - - // replace all known fields with applicable regexes - this.regex = ParserField.replacePatternWithRegex(this.regex); - - var ignoreField = new ParserField("i", ignoreClause, undefined, false); - this.regex = ignoreField.replaceInPattern(this.regex); - - // find invalid fields - var foundInvalid = this.regex.match(/\{[A-Za-z]+\}/g); - if (foundInvalid) { - throw new Error("Invalid fields: " + foundInvalid.join(", ")); - } - - var fieldExtractor = new RegExp(/\{([A-Za-z]+)\}/); - var result = pattern.match(fieldExtractor); - - while(!!result && result.index !== undefined) { - var field = result[1]; - - this.fields.push(field); - pattern = pattern.substring(result.index + result[0].length); - result = pattern.match(fieldExtractor); - } - } - - private postParse(scene: SceneParserResult) { - // set the date if the components are set - if (scene.yyyy.set && scene.mm.set && scene.dd.set) { - var fullDate = scene.yyyy.value + "-" + scene.mm.value + "-" + scene.dd.value; - if (SceneParserResult.validateDate(fullDate)) { - scene.setField(ParserField.Date, scene.yyyy.value + "-" + scene.mm.value + "-" + scene.dd.value); - } - } - } - - public parse(scene : SceneParserResult) { - var regex = new RegExp(this.regex, "i"); - - var result = scene.filename.match(regex); - - if(!result) { - return false; - } - - var mapper = this; - - result.forEach((match, index) => { - if (index === 0) { - // skip entire match - return; - } - - var field = mapper.fields[index - 1]; - var parserField = ParserField.getParserField(field); - if (!!parserField) { - scene.setField(parserField, match); - } - }); - - this.postParse(scene); - - return true; - } -} - interface IParserInput { pattern: string, ignoreWords: string[], whitespaceCharacters: string, - capitalizeTitle: boolean + capitalizeTitle: boolean, + page: number, + pageSize: number, + findClicked: boolean } -interface IParserRecipe extends IParserInput { +interface IParserRecipe { + pattern: string, + ignoreWords: string[], + whitespaceCharacters: string, + capitalizeTitle: boolean, description: string } @@ -447,15 +278,17 @@ const builtInRecipes = [ // Add mappings for tags, performers, studio export const SceneFilenameParser: FunctionComponent = (props: IProps) => { - const [parser, setParser] = useState(); const [parserResult, setParserResult] = useState([]); const [parserInput, setParserInput] = useState(initialParserInput()); const [allTitleSet, setAllTitleSet] = useState(false); const [allDateSet, setAllDateSet] = useState(false); + const [allPerformerSet, setAllPerformerSet] = useState(false); + const [allTagSet, setAllTagSet] = useState(false); + const [allStudioSet, setAllStudioSet] = useState(false); + + const [showFields, setShowFields] = useState>(initialShowFieldsState()); - const [page, setPage] = useState(1); - const [pageSize, setPageSize] = useState(20); const [totalItems, setTotalItems] = useState(0); // Network state @@ -468,33 +301,52 @@ export const SceneFilenameParser: FunctionComponent = (props: IProps) => pattern: "{title}.{ext}", ignoreWords: [], whitespaceCharacters: "._", - capitalizeTitle: true + capitalizeTitle: true, + page: 1, + pageSize: 20, + findClicked: false }; } - function getQueryFilter(regex : string, page: number, perPage: number) : GQL.FindFilterType { + function initialShowFieldsState() { + return new Map([ + ["Title", true], + ["Date", true], + ["Performers", true], + ["Tags", true], + ["Studio", true] + ]); + } + + function getParserFilter() { return { - q: regex, - page: page, - per_page: perPage + q: parserInput.pattern, + page: parserInput.page, + per_page: parserInput.pageSize, + sort: "path", + direction: GQL.SortDirectionEnum.Asc, + }; + } + + function getParserInput() { + return { + ignoreWords: parserInput.ignoreWords, + whitespaceCharacters: parserInput.whitespaceCharacters, + capitalizeTitle: parserInput.capitalizeTitle }; } async function onFind() { setParserResult([]); - if (!parser) { - return; - } - setIsLoading(true); try { - const response = await StashService.querySceneByPathRegex(getQueryFilter(parser.regex, page, pageSize)); + const response = await StashService.queryParseSceneFilenames(getParserFilter(), getParserInput()); - let result = response.data.findScenesByPathRegex; + let result = response.data.parseSceneFilenames; if (!!result) { - parseResults(result.scenes); + parseResults(result.results); setTotalItems(result.count); } } catch (err) { @@ -505,26 +357,30 @@ export const SceneFilenameParser: FunctionComponent = (props: IProps) => } useEffect(() => { - onFind(); - }, [page, parser, parserInput]); + if(parserInput.findClicked) { + onFind(); + } + }, [parserInput]); - useEffect(() => { - setPage(1); - onFind(); - }, [pageSize]) + function onPageSizeChanged(newSize : number) { + var newInput = _.clone(parserInput); + newInput.page = 1; + newInput.pageSize = newSize; + setParserInput(newInput); + } + + function onPageChanged(newPage : number) { + if (newPage !== parserInput.page) { + var newInput = _.clone(parserInput); + newInput.page = newPage; + setParserInput(newInput); + } + } function onFindClicked(input : IParserInput) { - var parser; - try { - parser = new ParseMapper(input.pattern, input.ignoreWords); - } catch(err) { - ErrorUtils.handle(err); - return; - } - - setParser(parser); + input.page = 1; + input.findClicked = true; setParserInput(input); - setPage(1); setTotalItems(0); } @@ -545,36 +401,38 @@ export const SceneFilenameParser: FunctionComponent = (props: IProps) => setIsLoading(false); } - function parseResults(scenes : GQL.SlimSceneDataFragment[]) { - if (scenes && parser) { - var result = scenes.map((scene) => { - var parserResult = new SceneParserResult(scene); - if(!parser.parse(parserResult)) { - return undefined; - } - - // post-process - if (parserResult.title && !!parserResult.title.value) { - if (parserInput.whitespaceCharacters) { - var wsRegExp = parserInput.whitespaceCharacters.replace(/([\-\.\(\)\[\]])/g, "\\$1"); - wsRegExp = "[" + wsRegExp + "]"; - parserResult.title.value = parserResult.title.value.replace(new RegExp(wsRegExp, "g"), " "); - } - - if (parserInput.capitalizeTitle) { - parserResult.title.value = parserResult.title.value.replace(/(?:^| )\w/g, function (chr) { - return chr.toUpperCase(); - }); - } - } - - return parserResult; + function parseResults(results : GQL.ParseSceneFilenamesResults[]) { + if (results) { + var result = results.map((r) => { + return new SceneParserResult(r); }).filter((r) => !!r) as SceneParserResult[]; setParserResult(result); + determineFieldsToHide(); } } + function determineFieldsToHide() { + var pattern = parserInput.pattern; + var titleSet = pattern.includes("{title}"); + var dateSet = pattern.includes("{date}") || + pattern.includes("{dd}") || // don't worry about other partial date fields since this should be implied + ParserField.fullDateFields.some((f) => { + return pattern.includes("{" + f.field + "}"); + }); + var performerSet = pattern.includes("{performer}"); + var tagSet = pattern.includes("{tag}"); + var studioSet = pattern.includes("{studio}"); + + var showFieldsCopy = _.clone(showFields); + showFieldsCopy.set("Title", titleSet); + showFieldsCopy.set("Date", dateSet); + showFieldsCopy.set("Performers", performerSet); + showFieldsCopy.set("Tags", tagSet); + showFieldsCopy.set("Studio", studioSet); + setShowFields(showFieldsCopy); + } + useEffect(() => { var newAllTitleSet = !parserResult.some((r) => { return !r.title.set; @@ -582,6 +440,15 @@ export const SceneFilenameParser: FunctionComponent = (props: IProps) => var newAllDateSet = !parserResult.some((r) => { return !r.date.set; }); + var newAllPerformerSet = !parserResult.some((r) => { + return !r.performerIds.set; + }); + var newAllTagSet = !parserResult.some((r) => { + return !r.tagIds.set; + }); + var newAllStudioSet = !parserResult.some((r) => { + return !r.studioId.set; + }); if (newAllTitleSet != allTitleSet) { setAllTitleSet(newAllTitleSet); @@ -589,6 +456,15 @@ export const SceneFilenameParser: FunctionComponent = (props: IProps) => if (newAllDateSet != allDateSet) { setAllDateSet(newAllDateSet); } + if (newAllPerformerSet != allPerformerSet) { + setAllTagSet(newAllPerformerSet); + } + if (newAllTagSet != allTagSet) { + setAllTagSet(newAllTagSet); + } + if (newAllStudioSet != allStudioSet) { + setAllStudioSet(newAllStudioSet); + } }, [parserResult]); function onSelectAllTitleSet(selected : boolean) { @@ -613,6 +489,114 @@ export const SceneFilenameParser: FunctionComponent = (props: IProps) => setAllDateSet(selected); } + function onSelectAllPerformerSet(selected : boolean) { + var newResult = [...parserResult]; + + newResult.forEach((r) => { + r.performerIds.set = selected; + }); + + setParserResult(newResult); + setAllPerformerSet(selected); + } + + function onSelectAllTagSet(selected : boolean) { + var newResult = [...parserResult]; + + newResult.forEach((r) => { + r.tagIds.set = selected; + }); + + setParserResult(newResult); + setAllTagSet(selected); + } + + function onSelectAllStudioSet(selected : boolean) { + var newResult = [...parserResult]; + + newResult.forEach((r) => { + r.studioId.set = selected; + }); + + setParserResult(newResult); + setAllStudioSet(selected); + } + + interface IShowFieldsTreeProps { + showFields: Map + onShowFieldsChanged: (fields : Map) => void + } + + function ShowFieldsTree(props : IShowFieldsTreeProps) { + const [displayFieldsExpanded, setDisplayFieldsExpanded] = useState(); + + const treeState: ITreeNode[] = [ + { + id: 0, + hasCaret: true, + label: "Display fields", + childNodes: [ + { + id: 1, + label: "Title", + }, + { + id: 2, + label: "Date", + }, + { + id: 3, + label: "Performers", + }, + { + id: 4, + label: "Tags", + }, + { + id: 5, + label: "Studio", + } + ] + } + ]; + + function setNodeState() { + if (!!treeState[0].childNodes) { + treeState[0].childNodes.forEach((n) => { + n.icon = props.showFields.get(n.label as string) ? "tick" : "cross"; + }); + } + + treeState[0].isExpanded = displayFieldsExpanded; + } + + setNodeState(); + + function expandNode() { + setDisplayFieldsExpanded(true); + } + + function collapseNode() { + setDisplayFieldsExpanded(false); + } + + function handleClick(nodeData: ITreeNode) { + var field = nodeData.label as string; + var fieldsCopy = _.clone(props.showFields); + fieldsCopy.set(field, !fieldsCopy.get(field)); + props.onShowFieldsChanged(fieldsCopy); + } + + return ( + + ); + } + interface IParserInputProps { input: IParserInput, onFind: (input : IParserInput) => void @@ -629,7 +613,10 @@ export const SceneFilenameParser: FunctionComponent = (props: IProps) => pattern: pattern, ignoreWords: ignoreWords.split(" "), whitespaceCharacters: whitespaceCharacters, - capitalizeTitle: capitalizeTitle + capitalizeTitle: capitalizeTitle, + page: 1, + pageSize: props.input.pageSize, + findClicked: props.input.findClicked }); } @@ -653,7 +640,7 @@ export const SceneFilenameParser: FunctionComponent = (props: IProps) => return item.pattern.includes(query); }; - function setParserRecipe(recipe: IParserInput) { + function setParserRecipe(recipe: IParserRecipe) { setPattern(recipe.pattern); setIgnoreWords(recipe.ignoreWords.join(" ")); setWhitespaceCharacters(recipe.whitespaceCharacters); @@ -680,7 +667,7 @@ export const SceneFilenameParser: FunctionComponent = (props: IProps) => return item.field.includes(query); }; - const validFields = [new ParserField("", undefined, "Wildcard")].concat(ParserField.validFields); + const validFields = [new ParserField("", "Wildcard")].concat(ParserField.validFields); function addParserField(field: ParserField) { setPattern(pattern + field.getFieldPattern()); @@ -760,13 +747,21 @@ export const SceneFilenameParser: FunctionComponent = (props: IProps) => + + setShowFields(fields)} + /> + + - + ) } diff --git a/ui/v2/src/components/select/FilterSelect.tsx b/ui/v2/src/components/select/FilterSelect.tsx index 6736fa880..b906667db 100644 --- a/ui/v2/src/components/select/FilterSelect.tsx +++ b/ui/v2/src/components/select/FilterSelect.tsx @@ -18,6 +18,7 @@ type ValidTypes = interface IProps extends HTMLInputProps { type: "performers" | "studios" | "tags"; initialId?: string; + noSelectionString?: string; onSelectItem: (item: ValidTypes | undefined) => void; } @@ -98,7 +99,8 @@ export const FilterSelect: React.FunctionComponent = (props: IProps) => setSelectedItem(item); } - const buttonText = selectedItem ? selectedItem.name : "(No selection)"; + const noSelection = props.noSelectionString !== undefined ? props.noSelectionString : "(No selection)" + const buttonText = selectedItem ? selectedItem.name : noSelection; return ( ({ + query: GQL.ParseSceneFilenamesDocument, + variables: {filter: filter, config: config}, + fetchPolicy: "network-only", + }); + } + public static nullToUndefined(value: any): any { if (_.isPlainObject(value)) { return _.mapValues(value, StashService.nullToUndefined); diff --git a/ui/v2/src/index.scss b/ui/v2/src/index.scss index 411a5a408..4905a813e 100755 --- a/ui/v2/src/index.scss +++ b/ui/v2/src/index.scss @@ -264,32 +264,53 @@ span.block { #parser-container { margin: 10px auto; width: 75%; -} -#parser-container .inputs label { - width: 12em; -} + & .inputs label { + width: 12em; + } + + & .inputs .bp3-input-group { + width: 80ch; + } -#parser-container .inputs .bp3-input-group { - width: 80ch; -} + & .scene-parser-results { + overflow-x: auto; + } + + & .scene-parser-row .bp3-checkbox { + margin: 0px -20px 0px 0px; + } + + & .scene-parser-row .parser-field-title input { + width: 50ch; + } -.scene-parser-row .bp3-checkbox { - margin: 0px -20px 0px 0px; -} + & .scene-parser-row .parser-field-date input { + width: 13ch; + } -.scene-parser-row .title input { - width: 50ch; -} + & .scene-parser-row .parser-field-performers input { + width: 20ch; + } -.scene-parser-row input { - min-width: 10ch; -} + & .scene-parser-row .parser-field-tags input { + width: 20ch; + } -.scene-parser-row .bp3-form-group { - margin-bottom: 0px; -} + & .scene-parser-row .parser-field-studio input { + width: 15ch; + } + + & .scene-parser-row input { + min-width: 10ch; + } + + & .scene-parser-row .bp3-form-group { + margin-bottom: 0px; + } + + & .scene-parser-row div:first-child > input { + margin-bottom: 5px; + } -.scene-parser-row div:first-child > input { - margin-bottom: 5px; } \ No newline at end of file