Scene filename to metadata parser (#164)

* Initial UI prototype

* Add backend support to update multiple scenes

* Fix title editing issues

* Add query regex support. UI improvements

* Rewrite parser. Add fields button and page size

* Add helper text for escaping {} characters

* Validate date

* Only set values if different from original

* Only update scenes that have something changed

* Add built in parser input recipes

* Make pattern matching case-insensistive
This commit is contained in:
WithoutPants
2019-10-31 00:37:21 +11:00
committed by Leopere
parent e59fd147cf
commit 7cb9cd8a38
12 changed files with 1140 additions and 14 deletions

View File

@@ -54,6 +54,12 @@ mutation BulkSceneUpdate(
} }
} }
mutation ScenesUpdate($input : [SceneUpdateInput!]!) {
scenesUpdate(input: $input) {
...SceneData
}
}
mutation SceneDestroy($id: ID!, $delete_file: Boolean, $delete_generated : Boolean) { mutation SceneDestroy($id: ID!, $delete_file: Boolean, $delete_generated : Boolean) {
sceneDestroy(input: {id: $id, delete_file: $delete_file, delete_generated: $delete_generated}) sceneDestroy(input: {id: $id, delete_file: $delete_file, delete_generated: $delete_generated})
} }

View File

@@ -7,6 +7,15 @@ query FindScenes($filter: FindFilterType, $scene_filter: SceneFilterType, $scene
} }
} }
query FindScenesByPathRegex($filter: FindFilterType) {
findScenesByPathRegex(filter: $filter) {
count
scenes {
...SlimSceneData
}
}
}
query FindScene($id: ID!, $checksum: String) { query FindScene($id: ID!, $checksum: String) {
findScene(id: $id, checksum: $checksum) { findScene(id: $id, checksum: $checksum) {
...SceneData ...SceneData

View File

@@ -5,6 +5,8 @@ type Query {
"""A function which queries Scene objects""" """A function which queries Scene objects"""
findScenes(scene_filter: SceneFilterType, scene_ids: [Int!], filter: FindFilterType): FindScenesResultType! findScenes(scene_filter: SceneFilterType, scene_ids: [Int!], filter: FindFilterType): FindScenesResultType!
findScenesByPathRegex(filter: FindFilterType): FindScenesResultType!
"""A function which queries SceneMarker objects""" """A function which queries SceneMarker objects"""
findSceneMarkers(scene_marker_filter: SceneMarkerFilterType filter: FindFilterType): FindSceneMarkersResultType! findSceneMarkers(scene_marker_filter: SceneMarkerFilterType filter: FindFilterType): FindSceneMarkersResultType!
@@ -79,6 +81,7 @@ type Mutation {
sceneUpdate(input: SceneUpdateInput!): Scene sceneUpdate(input: SceneUpdateInput!): Scene
bulkSceneUpdate(input: BulkSceneUpdateInput!): [Scene!] bulkSceneUpdate(input: BulkSceneUpdateInput!): [Scene!]
sceneDestroy(input: SceneDestroyInput!): Boolean! sceneDestroy(input: SceneDestroyInput!): Boolean!
scenesUpdate(input: [SceneUpdateInput!]!): [Scene]
sceneMarkerCreate(input: SceneMarkerCreateInput!): SceneMarker sceneMarkerCreate(input: SceneMarkerCreateInput!): SceneMarker
sceneMarkerUpdate(input: SceneMarkerUpdateInput!): SceneMarker sceneMarkerUpdate(input: SceneMarkerUpdateInput!): SceneMarker

View File

@@ -6,12 +6,57 @@ import (
"strconv" "strconv"
"time" "time"
"github.com/jmoiron/sqlx"
"github.com/stashapp/stash/pkg/database" "github.com/stashapp/stash/pkg/database"
"github.com/stashapp/stash/pkg/manager" "github.com/stashapp/stash/pkg/manager"
"github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/models"
) )
func (r *mutationResolver) SceneUpdate(ctx context.Context, input models.SceneUpdateInput) (*models.Scene, error) { func (r *mutationResolver) SceneUpdate(ctx context.Context, input models.SceneUpdateInput) (*models.Scene, error) {
// Start the transaction and save the scene
tx := database.DB.MustBeginTx(ctx, nil)
ret, err := r.sceneUpdate(input, tx)
if err != nil {
_ = tx.Rollback()
return nil, err
}
// Commit
if err := tx.Commit(); err != nil {
return nil, err
}
return ret, nil
}
func (r *mutationResolver) ScenesUpdate(ctx context.Context, input []*models.SceneUpdateInput) ([]*models.Scene, error) {
// Start the transaction and save the scene
tx := database.DB.MustBeginTx(ctx, nil)
var ret []*models.Scene
for _, scene := range input {
thisScene, err := r.sceneUpdate(*scene, tx)
ret = append(ret, thisScene)
if err != nil {
_ = tx.Rollback()
return nil, err
}
}
// Commit
if err := tx.Commit(); err != nil {
return nil, err
}
return ret, nil
}
func (r *mutationResolver) sceneUpdate(input models.SceneUpdateInput, tx *sqlx.Tx) (*models.Scene, error) {
// Populate scene from the input // Populate scene from the input
sceneID, _ := strconv.Atoi(input.ID) sceneID, _ := strconv.Atoi(input.ID)
updatedTime := time.Now() updatedTime := time.Now()
@@ -47,13 +92,10 @@ func (r *mutationResolver) SceneUpdate(ctx context.Context, input models.SceneUp
updatedScene.StudioID = &sql.NullInt64{Valid: false} updatedScene.StudioID = &sql.NullInt64{Valid: false}
} }
// Start the transaction and save the scene marker
tx := database.DB.MustBeginTx(ctx, nil)
qb := models.NewSceneQueryBuilder() qb := models.NewSceneQueryBuilder()
jqb := models.NewJoinsQueryBuilder() jqb := models.NewJoinsQueryBuilder()
scene, err := qb.Update(updatedScene, tx) scene, err := qb.Update(updatedScene, tx)
if err != nil { if err != nil {
_ = tx.Rollback()
return nil, err return nil, err
} }
@@ -61,7 +103,6 @@ func (r *mutationResolver) SceneUpdate(ctx context.Context, input models.SceneUp
gqb := models.NewGalleryQueryBuilder() gqb := models.NewGalleryQueryBuilder()
err = gqb.ClearGalleryId(sceneID, tx) err = gqb.ClearGalleryId(sceneID, tx)
if err != nil { if err != nil {
_ = tx.Rollback()
return nil, err return nil, err
} }
@@ -76,7 +117,6 @@ func (r *mutationResolver) SceneUpdate(ctx context.Context, input models.SceneUp
gqb := models.NewGalleryQueryBuilder() gqb := models.NewGalleryQueryBuilder()
_, err := gqb.Update(updatedGallery, tx) _, err := gqb.Update(updatedGallery, tx)
if err != nil { if err != nil {
_ = tx.Rollback()
return nil, err return nil, err
} }
} }
@@ -92,7 +132,6 @@ func (r *mutationResolver) SceneUpdate(ctx context.Context, input models.SceneUp
performerJoins = append(performerJoins, performerJoin) performerJoins = append(performerJoins, performerJoin)
} }
if err := jqb.UpdatePerformersScenes(sceneID, performerJoins, tx); err != nil { if err := jqb.UpdatePerformersScenes(sceneID, performerJoins, tx); err != nil {
_ = tx.Rollback()
return nil, err return nil, err
} }
@@ -107,12 +146,6 @@ func (r *mutationResolver) SceneUpdate(ctx context.Context, input models.SceneUp
tagJoins = append(tagJoins, tagJoin) tagJoins = append(tagJoins, tagJoin)
} }
if err := jqb.UpdateScenesTags(sceneID, tagJoins, tx); err != nil { if err := jqb.UpdateScenesTags(sceneID, tagJoins, tx); err != nil {
_ = tx.Rollback()
return nil, err
}
// Commit
if err := tx.Commit(); err != nil {
return nil, err return nil, err
} }

View File

@@ -27,3 +27,13 @@ func (r *queryResolver) FindScenes(ctx context.Context, sceneFilter *models.Scen
Scenes: scenes, Scenes: scenes,
}, nil }, nil
} }
func (r *queryResolver) FindScenesByPathRegex(ctx context.Context, filter *models.FindFilterType) (*models.FindScenesResultType, error) {
qb := models.NewSceneQueryBuilder()
scenes, total := qb.QueryByPathRegex(filter)
return &models.FindScenesResultType{
Count: total,
Scenes: scenes,
}, nil
}

View File

@@ -1,12 +1,14 @@
package database package database
import ( import (
"database/sql"
"fmt" "fmt"
"regexp"
"github.com/gobuffalo/packr/v2" "github.com/gobuffalo/packr/v2"
"github.com/golang-migrate/migrate/v4" "github.com/golang-migrate/migrate/v4"
"github.com/golang-migrate/migrate/v4/source" "github.com/golang-migrate/migrate/v4/source"
"github.com/jmoiron/sqlx" "github.com/jmoiron/sqlx"
_ "github.com/mattn/go-sqlite3" sqlite3 "github.com/mattn/go-sqlite3"
"github.com/stashapp/stash/pkg/logger" "github.com/stashapp/stash/pkg/logger"
"github.com/stashapp/stash/pkg/utils" "github.com/stashapp/stash/pkg/utils"
"os" "os"
@@ -15,11 +17,16 @@ import (
var DB *sqlx.DB var DB *sqlx.DB
var appSchemaVersion uint = 1 var appSchemaVersion uint = 1
const sqlite3Driver = "sqlite3_regexp"
func Initialize(databasePath string) { func Initialize(databasePath string) {
runMigrations(databasePath) runMigrations(databasePath)
// register custom driver with regexp function
registerRegexpFunc()
// https://github.com/mattn/go-sqlite3 // https://github.com/mattn/go-sqlite3
conn, err := sqlx.Open("sqlite3", "file:"+databasePath+"?_fk=true") conn, err := sqlx.Open(sqlite3Driver, "file:"+databasePath+"?_fk=true")
conn.SetMaxOpenConns(25) conn.SetMaxOpenConns(25)
conn.SetMaxIdleConns(4) conn.SetMaxIdleConns(4)
if err != nil { if err != nil {
@@ -62,3 +69,16 @@ func runMigrations(databasePath string) {
} }
} }
} }
func registerRegexpFunc() {
regexFn := func(re, s string) (bool, error) {
return regexp.MatchString(re, s)
}
sql.Register(sqlite3Driver,
&sqlite3.SQLiteDriver{
ConnectHook: func(conn *sqlite3.SQLiteConn) error {
return conn.RegisterFunc("regexp", regexFn, true)
},
})
}

View File

@@ -291,6 +291,32 @@ func getMultiCriterionClause(table string, joinTable string, joinTableField stri
return whereClause, havingClause return whereClause, havingClause
} }
func (qb *SceneQueryBuilder) QueryByPathRegex(findFilter *FindFilterType) ([]*Scene, int) {
if findFilter == nil {
findFilter = &FindFilterType{}
}
var whereClauses []string
var havingClauses []string
var args []interface{}
body := selectDistinctIDs("scenes")
if q := findFilter.Q; q != nil && *q != "" {
whereClauses = append(whereClauses, "scenes.path regexp '" + *q + "'")
}
sortAndPagination := qb.getSceneSort(findFilter) + getPagination(findFilter)
idsResult, countResult := executeFindQuery("scenes", body, args, sortAndPagination, whereClauses, havingClauses)
var scenes []*Scene
for _, id := range idsResult {
scene, _ := qb.Find(id)
scenes = append(scenes, scene)
}
return scenes, countResult
}
func (qb *SceneQueryBuilder) getSceneSort(findFilter *FindFilterType) string { func (qb *SceneQueryBuilder) getSceneSort(findFilter *FindFilterType) string {
if findFilter == nil { if findFilter == nil {
return " ORDER BY scenes.path, scenes.date ASC " return " ORDER BY scenes.path, scenes.date ASC "

View File

@@ -10,6 +10,7 @@ import { Settings } from "./components/Settings/Settings";
import { Stats } from "./components/Stats"; import { Stats } from "./components/Stats";
import Studios from "./components/Studios/Studios"; import Studios from "./components/Studios/Studios";
import Tags from "./components/Tags/Tags"; import Tags from "./components/Tags/Tags";
import { SceneFilenameParser } from "./components/scenes/SceneFilenameParser";
interface IProps {} interface IProps {}
@@ -27,6 +28,7 @@ export const App: FunctionComponent<IProps> = (props: IProps) => {
<Route path="/tags" component={Tags} /> <Route path="/tags" component={Tags} />
<Route path="/studios" component={Studios} /> <Route path="/studios" component={Studios} />
<Route path="/settings" component={Settings} /> <Route path="/settings" component={Settings} />
<Route path="/sceneFilenameParser" component={SceneFilenameParser} />
<Route component={PageNotFound} /> <Route component={PageNotFound} />
</Switch> </Switch>
</ErrorBoundary> </ErrorBoundary>

View File

@@ -5,12 +5,14 @@ import {
Divider, Divider,
FormGroup, FormGroup,
H4, H4,
AnchorButton,
} from "@blueprintjs/core"; } from "@blueprintjs/core";
import React, { FunctionComponent, useState } from "react"; import React, { FunctionComponent, useState } from "react";
import { StashService } from "../../../core/StashService"; import { StashService } from "../../../core/StashService";
import { ErrorUtils } from "../../../utils/errors"; import { ErrorUtils } from "../../../utils/errors";
import { ToastUtils } from "../../../utils/toasts"; import { ToastUtils } from "../../../utils/toasts";
import { GenerateButton } from "./GenerateButton"; import { GenerateButton } from "./GenerateButton";
import { Link } from "react-router-dom";
interface IProps {} interface IProps {}
@@ -94,6 +96,12 @@ export const SettingsTasksPanel: FunctionComponent<IProps> = (props: IProps) =>
onChange={() => setNameFromMetadata(!nameFromMetadata)} onChange={() => setNameFromMetadata(!nameFromMetadata)}
/> />
<Button id="scan" text="Scan" onClick={() => onScan()} /> <Button id="scan" text="Scan" onClick={() => onScan()} />
</FormGroup>
<Link className="bp3-button" to={"/sceneFilenameParser"}>
Scene Filename Parser
</Link>
<FormGroup>
</FormGroup> </FormGroup>
<Divider /> <Divider />

View File

@@ -0,0 +1,965 @@
import {
Card,
FormGroup,
InputGroup,
Button,
H4,
Spinner,
HTMLTable,
Checkbox,
H5,
MenuItem,
HTMLSelect,
} from "@blueprintjs/core";
import React, { FunctionComponent, useEffect, useState } from "react";
import { IBaseProps } from "../../models";
import { StashService } from "../../core/StashService";
import * as GQL from "../../core/generated-graphql";
import { SlimSceneDataFragment, Maybe } from "../../core/generated-graphql";
import { TextUtils } from "../../utils/text";
import _ from "lodash";
import { ToastUtils } from "../../utils/toasts";
import { ErrorUtils } from "../../utils/errors";
import { Pagination } from "../list/Pagination";
import { Select, ItemRenderer, ItemPredicate } from "@blueprintjs/select";
interface IProps extends IBaseProps {}
class ParserResult<T> {
public value: Maybe<T>;
public originalValue: Maybe<T>;
public set: boolean = false;
public setOriginalValue(v : Maybe<T>) {
this.originalValue = v;
this.value = v;
}
}
class ParserField {
public field : string;
public fieldRegex: RegExp;
public regex : string;
public helperText? : string;
constructor(field: string, regex?: string, helperText?: string, captured?: boolean) {
if (regex === undefined) {
regex = ".*";
}
if (captured === undefined) {
captured = true;
}
this.field = field;
this.helperText = helperText;
this.fieldRegex = new RegExp("\\{" + this.field + "\\}", "g");
var regexStr = regex;
if (captured) {
regexStr = "(" + regexStr + ")";
}
this.regex = regexStr;
}
public replaceInPattern(pattern : string) {
return pattern.replace(this.fieldRegex, this.regex);
}
public getFieldPattern() {
return "{" + this.field + "}";
}
static Title = new ParserField("title");
static Ext = new ParserField("ext", ".*$", "File extension", false);
static I = new ParserField("i", undefined, "Matches any ignored word", false);
static D = new ParserField("d", "(?:\\.|-|_)", "Matches any delimiter (.-_)", false);
// date fields
static Date = new ParserField("date", "\\d{4}-\\d{2}-\\d{2}", "YYYY-MM-DD");
static YYYY = new ParserField("yyyy", "\\d{4}", "Year");
static YY = new ParserField("yy", "\\d{2}", "Year (20YY)");
static MM = new ParserField("mm", "\\d{2}", "Two digit month");
static DD = new ParserField("dd", "\\d{2}", "Two digit date");
static YYYYMMDD = new ParserField("yyyymmdd", "\\d{8}");
static YYMMDD = new ParserField("yymmdd", "\\d{6}");
static DDMMYYYY = new ParserField("ddmmyyyy", "\\d{8}");
static DDMMYY = new ParserField("ddmmyy", "\\d{6}");
static MMDDYYYY = new ParserField("mmddyyyy", "\\d{8}");
static MMDDYY = new ParserField("mmddyy", "\\d{6}");
static validFields = [
ParserField.Title,
ParserField.Ext,
ParserField.D,
ParserField.I,
ParserField.Date,
ParserField.YYYY,
ParserField.YY,
ParserField.MM,
ParserField.DD,
ParserField.YYYYMMDD,
ParserField.YYMMDD,
ParserField.DDMMYYYY,
ParserField.DDMMYY,
ParserField.MMDDYYYY,
ParserField.MMDDYY
]
static fullDateFields = [
ParserField.YYYYMMDD,
ParserField.YYMMDD,
ParserField.DDMMYYYY,
ParserField.DDMMYY,
ParserField.MMDDYYYY,
ParserField.MMDDYY
];
public static getParserField(field: string) {
return ParserField.validFields.find((f) => {
return f.field === field;
});
}
public static isValidField(field : string) {
return !!ParserField.getParserField(field);
}
public static isFullDateField(field : ParserField) {
return ParserField.fullDateFields.includes(field);
}
public static replacePatternWithRegex(pattern: string) {
ParserField.validFields.forEach((field) => {
pattern = field.replaceInPattern(pattern);
});
return pattern;
}
}
class SceneParserResult {
public id: string;
public filename: string;
public title: ParserResult<string> = new ParserResult();
public date: ParserResult<string> = new ParserResult();
public yyyy : ParserResult<string> = new ParserResult();
public mm : ParserResult<string> = new ParserResult();
public dd : ParserResult<string> = new ParserResult();
public studioId: ParserResult<string> = new ParserResult();
public tags: ParserResult<string[]> = new ParserResult();
public performerIds: ParserResult<string[]> = new ParserResult();
public scene : SlimSceneDataFragment;
constructor(scene : SlimSceneDataFragment) {
this.id = scene.id;
this.filename = TextUtils.fileNameFromPath(scene.path);
this.title.setOriginalValue(scene.title);
this.date.setOriginalValue(scene.date);
this.scene = scene;
}
public static validateDate(dateStr: string) {
var splits = dateStr.split("-");
if (splits.length != 3) {
return false;
}
var year = parseInt(splits[0]);
var month = parseInt(splits[1]);
var d = parseInt(splits[2]);
var date = new Date();
date.setMonth(month - 1);
date.setDate(d);
// assume year must be between 1900 and 2100
if (year < 1900 || year > 2100) {
return false;
}
if (month < 1 || month > 12) {
return false;
}
// not checking individual months to ensure date is in the correct range
if (d < 1 || d > 31) {
return false;
}
return true;
}
private setDate(field: ParserField, value: string) {
var yearIndex = 0;
var yearLength = field.field.split("y").length - 1;
var dateIndex = 0;
var monthIndex = 0;
switch (field) {
case ParserField.YYYYMMDD:
case ParserField.YYMMDD:
monthIndex = yearLength;
dateIndex = monthIndex + 2;
break;
case ParserField.DDMMYYYY:
case ParserField.DDMMYY:
monthIndex = 2;
yearIndex = monthIndex + 2;
break;
case ParserField.MMDDYYYY:
case ParserField.MMDDYY:
dateIndex = monthIndex + 2;
yearIndex = dateIndex + 2;
break;
}
var yearValue = value.substring(yearIndex, yearIndex + yearLength);
var monthValue = value.substring(monthIndex, monthIndex + 2);
var dateValue = value.substring(dateIndex, dateIndex + 2);
var fullDate = yearValue + "-" + monthValue + "-" + dateValue;
// ensure the date is valid
// only set if new value is different from the old
if (SceneParserResult.validateDate(fullDate) && this.date.originalValue !== fullDate) {
this.date.set = true;
this.date.value = fullDate
}
}
public setField(field: ParserField, value: any) {
var parserResult : ParserResult<any> | undefined = undefined;
if (ParserField.isFullDateField(field)) {
this.setDate(field, value);
return;
}
switch (field) {
case ParserField.Title:
parserResult = this.title;
break;
case ParserField.Date:
parserResult = this.date;
break;
case ParserField.YYYY:
parserResult = this.yyyy;
break;
case ParserField.YY:
parserResult = this.yyyy;
value = "20" + value;
break;
case ParserField.MM:
parserResult = this.mm;
break;
case ParserField.DD:
parserResult = this.dd;
break;
}
// TODO - other fields
// only set if different from original value
if (!!parserResult && parserResult.originalValue !== value) {
parserResult.set = true;
parserResult.value = value;
}
}
private static setInput(object: any, key: string, parserResult : ParserResult<any>) {
if (parserResult.set) {
object[key] = parserResult.value;
}
}
// returns true if any of its fields have set == true
public isChanged() {
return this.title.set || this.date.set;
}
public toSceneUpdateInput() {
var ret = {
id: this.id,
title: this.scene.title,
details: this.scene.details,
url: this.scene.url,
date: this.scene.date,
rating: this.scene.rating,
gallery_id: this.scene.gallery ? this.scene.gallery.id : undefined,
studio_id: this.scene.studio ? this.scene.studio.id : undefined,
performer_ids: this.scene.performers.map((performer) => performer.id),
tag_ids: this.scene.tags.map((tag) => tag.id)
};
SceneParserResult.setInput(ret, "title", this.title);
SceneParserResult.setInput(ret, "date", this.date);
// TODO - other fields as added
return ret;
}
};
class ParseMapper {
public fields : string[] = [];
public regex : string = "";
public matched : boolean = true;
constructor(pattern : string, ignoreFields : string[]) {
// escape control characters
this.regex = pattern.replace(/([\-\.\(\)\[\]])/g, "\\$1");
// replace {} with wildcard
this.regex = this.regex.replace(/\{\}/g, ".*");
// set ignore fields
ignoreFields = ignoreFields.map((s) => s.replace(/([\-\.\(\)\[\]])/g, "\\$1").trim());
var ignoreClause = ignoreFields.map((s) => "(?:" + s + ")").join("|");
ignoreClause = "(?:" + ignoreClause + ")";
ParserField.I.regex = ignoreClause;
// replace all known fields with applicable regexes
this.regex = ParserField.replacePatternWithRegex(this.regex);
var ignoreField = new ParserField("i", ignoreClause, undefined, false);
this.regex = ignoreField.replaceInPattern(this.regex);
// find invalid fields
var foundInvalid = this.regex.match(/\{[A-Za-z]+\}/g);
if (foundInvalid) {
throw new Error("Invalid fields: " + foundInvalid.join(", "));
}
var fieldExtractor = new RegExp(/\{([A-Za-z]+)\}/);
var result = pattern.match(fieldExtractor);
while(!!result && result.index !== undefined) {
var field = result[1];
this.fields.push(field);
pattern = pattern.substring(result.index + result[0].length);
result = pattern.match(fieldExtractor);
}
}
private postParse(scene: SceneParserResult) {
// set the date if the components are set
if (scene.yyyy.set && scene.mm.set && scene.dd.set) {
var fullDate = scene.yyyy.value + "-" + scene.mm.value + "-" + scene.dd.value;
if (SceneParserResult.validateDate(fullDate)) {
scene.setField(ParserField.Date, scene.yyyy.value + "-" + scene.mm.value + "-" + scene.dd.value);
}
}
}
public parse(scene : SceneParserResult) {
var regex = new RegExp(this.regex, "i");
var result = scene.filename.match(regex);
if(!result) {
return false;
}
var mapper = this;
result.forEach((match, index) => {
if (index === 0) {
// skip entire match
return;
}
var field = mapper.fields[index - 1];
var parserField = ParserField.getParserField(field);
if (!!parserField) {
scene.setField(parserField, match);
}
});
this.postParse(scene);
return true;
}
}
interface IParserInput {
pattern: string,
ignoreWords: string[],
whitespaceCharacters: string,
capitalizeTitle: boolean
}
interface IParserRecipe extends IParserInput {
description: string
}
const builtInRecipes = [
{
pattern: "{title}",
ignoreWords: [],
whitespaceCharacters: "",
capitalizeTitle: false,
description: "Filename"
},
{
pattern: "{title}.{ext}",
ignoreWords: [],
whitespaceCharacters: "",
capitalizeTitle: false,
description: "Without extension"
},
{
pattern: "{}.{yy}.{mm}.{dd}.{title}.XXX.{}.{ext}",
ignoreWords: [],
whitespaceCharacters: ".",
capitalizeTitle: true,
description: ""
},
{
pattern: "{}.{yy}.{mm}.{dd}.{title}.{ext}",
ignoreWords: [],
whitespaceCharacters: ".",
capitalizeTitle: true,
description: ""
},
{
pattern: "{title}.XXX.{}.{ext}",
ignoreWords: [],
whitespaceCharacters: ".",
capitalizeTitle: true,
description: ""
},
{
pattern: "{}.{yy}.{mm}.{dd}.{title}.{i}.{ext}",
ignoreWords: ["cz", "fr"],
whitespaceCharacters: ".",
capitalizeTitle: true,
description: "Foreign language"
}
];
// TODO:
// Add mappings for tags, performers, studio
export const SceneFilenameParser: FunctionComponent<IProps> = (props: IProps) => {
const [parser, setParser] = useState<ParseMapper | undefined>();
const [parserResult, setParserResult] = useState<SceneParserResult[]>([]);
const [parserInput, setParserInput] = useState<IParserInput>(initialParserInput());
const [allTitleSet, setAllTitleSet] = useState<boolean>(false);
const [allDateSet, setAllDateSet] = useState<boolean>(false);
const [page, setPage] = useState<number>(1);
const [pageSize, setPageSize] = useState<number>(20);
const [totalItems, setTotalItems] = useState<number>(0);
// Network state
const [isLoading, setIsLoading] = useState(false);
const updateScenes = StashService.useScenesUpdate(getScenesUpdateData());
function initialParserInput() {
return {
pattern: "{title}.{ext}",
ignoreWords: [],
whitespaceCharacters: "._",
capitalizeTitle: true
};
}
function getQueryFilter(regex : string, page: number, perPage: number) : GQL.FindFilterType {
return {
q: regex,
page: page,
per_page: perPage
};
}
async function onFind() {
setParserResult([]);
if (!parser) {
return;
}
setIsLoading(true);
try {
const response = await StashService.querySceneByPathRegex(getQueryFilter(parser.regex, page, pageSize));
let result = response.data.findScenesByPathRegex;
if (!!result) {
parseResults(result.scenes);
setTotalItems(result.count);
}
} catch (err) {
ErrorUtils.handle(err);
}
setIsLoading(false);
}
useEffect(() => {
onFind();
}, [page, parser, parserInput]);
useEffect(() => {
setPage(1);
onFind();
}, [pageSize])
function onFindClicked(input : IParserInput) {
var parser;
try {
parser = new ParseMapper(input.pattern, input.ignoreWords);
} catch(err) {
ErrorUtils.handle(err);
return;
}
setParser(parser);
setParserInput(input);
setPage(1);
setTotalItems(0);
}
function getScenesUpdateData() {
return parserResult.filter((result) => result.isChanged()).map((result) => result.toSceneUpdateInput());
}
async function onApply() {
setIsLoading(true);
try {
await updateScenes();
ToastUtils.success("Updated scenes");
} catch (e) {
ErrorUtils.handle(e);
}
setIsLoading(false);
}
function parseResults(scenes : GQL.SlimSceneDataFragment[]) {
if (scenes && parser) {
var result = scenes.map((scene) => {
var parserResult = new SceneParserResult(scene);
if(!parser.parse(parserResult)) {
return undefined;
}
// post-process
if (parserResult.title && !!parserResult.title.value) {
if (parserInput.whitespaceCharacters) {
var wsRegExp = parserInput.whitespaceCharacters.replace(/([\-\.\(\)\[\]])/g, "\\$1");
wsRegExp = "[" + wsRegExp + "]";
parserResult.title.value = parserResult.title.value.replace(new RegExp(wsRegExp, "g"), " ");
}
if (parserInput.capitalizeTitle) {
parserResult.title.value = parserResult.title.value.replace(/(?:^| )\w/g, function (chr) {
return chr.toUpperCase();
});
}
}
return parserResult;
}).filter((r) => !!r) as SceneParserResult[];
setParserResult(result);
}
}
useEffect(() => {
var newAllTitleSet = !parserResult.some((r) => {
return !r.title.set;
});
var newAllDateSet = !parserResult.some((r) => {
return !r.date.set;
});
if (newAllTitleSet != allTitleSet) {
setAllTitleSet(newAllTitleSet);
}
if (newAllDateSet != allDateSet) {
setAllDateSet(newAllDateSet);
}
}, [parserResult]);
function onSelectAllTitleSet(selected : boolean) {
var newResult = [...parserResult];
newResult.forEach((r) => {
r.title.set = selected;
});
setParserResult(newResult);
setAllTitleSet(selected);
}
function onSelectAllDateSet(selected : boolean) {
var newResult = [...parserResult];
newResult.forEach((r) => {
r.date.set = selected;
});
setParserResult(newResult);
setAllDateSet(selected);
}
interface IParserInputProps {
input: IParserInput,
onFind: (input : IParserInput) => void
}
function ParserInput(props : IParserInputProps) {
const [pattern, setPattern] = useState<string>(props.input.pattern);
const [ignoreWords, setIgnoreWords] = useState<string>(props.input.ignoreWords.join(" "));
const [whitespaceCharacters, setWhitespaceCharacters] = useState<string>(props.input.whitespaceCharacters);
const [capitalizeTitle, setCapitalizeTitle] = useState<boolean>(props.input.capitalizeTitle);
function onFind() {
props.onFind({
pattern: pattern,
ignoreWords: ignoreWords.split(" "),
whitespaceCharacters: whitespaceCharacters,
capitalizeTitle: capitalizeTitle
});
}
const ParserRecipeSelect = Select.ofType<IParserRecipe>();
const renderParserRecipe: ItemRenderer<IParserRecipe> = (input, { handleClick, modifiers }) => {
if (!modifiers.matchesPredicate) {
return null;
}
return (
<MenuItem
key={input.pattern}
onClick={handleClick}
text={input.pattern || "{}"}
label={input.description}
/>
);
};
const parserRecipePredicate: ItemPredicate<IParserRecipe> = (query, item) => {
return item.pattern.includes(query);
};
function setParserRecipe(recipe: IParserInput) {
setPattern(recipe.pattern);
setIgnoreWords(recipe.ignoreWords.join(" "));
setWhitespaceCharacters(recipe.whitespaceCharacters);
setCapitalizeTitle(recipe.capitalizeTitle);
}
const ParserFieldSelect = Select.ofType<ParserField>();
const renderParserField: ItemRenderer<ParserField> = (field, { handleClick, modifiers }) => {
if (!modifiers.matchesPredicate) {
return null;
}
return (
<MenuItem
key={field.field}
onClick={handleClick}
text={field.field || "{}"}
label={field.helperText}
/>
);
};
const parserFieldPredicate: ItemPredicate<ParserField> = (query, item) => {
return item.field.includes(query);
};
const validFields = [new ParserField("", undefined, "Wildcard")].concat(ParserField.validFields);
function addParserField(field: ParserField) {
setPattern(pattern + field.getFieldPattern());
}
const parserFieldSelect = (
<ParserFieldSelect
items={validFields}
onItemSelect={(item) => addParserField(item)}
itemRenderer={renderParserField}
itemPredicate={parserFieldPredicate}
>
<Button
text="Add field"
rightIcon="caret-down"
/>
</ParserFieldSelect>
);
const PAGE_SIZE_OPTIONS = ["20", "40", "60", "120"];
return (
<>
<FormGroup className="inputs">
<FormGroup
label="Filename pattern:"
inline={true}
helperText="Use '\\' to escape literal {} characters"
>
<InputGroup
onChange={(newValue: any) => setPattern(newValue.target.value)}
value={pattern}
rightElement={parserFieldSelect}
/>
</FormGroup>
<FormGroup>
<FormGroup label="Ignored words:" inline={true} helperText="Matches with {i}">
<InputGroup
onChange={(newValue: any) => setIgnoreWords(newValue.target.value)}
value={ignoreWords}
/>
</FormGroup>
</FormGroup>
<FormGroup>
<H5>Title</H5>
<FormGroup label="Whitespace characters:"
inline={true}
helperText="These characters will be replaced with whitespace in the title">
<InputGroup
onChange={(newValue: any) => setWhitespaceCharacters(newValue.target.value)}
value={whitespaceCharacters}
/>
</FormGroup>
<Checkbox
label="Capitalize title"
checked={capitalizeTitle}
onChange={() => setCapitalizeTitle(!capitalizeTitle)}
inline={true}
/>
</FormGroup>
{/* TODO - mapping stuff will go here */}
<FormGroup>
<ParserRecipeSelect
items={builtInRecipes}
onItemSelect={(item) => setParserRecipe(item)}
itemRenderer={renderParserRecipe}
itemPredicate={parserRecipePredicate}
>
<Button
text="Select Parser Recipe"
rightIcon="caret-down"
/>
</ParserRecipeSelect>
</FormGroup>
<FormGroup>
<Button text="Find" onClick={() => onFind()} />
<HTMLSelect
style={{flexBasis: "min-content"}}
options={PAGE_SIZE_OPTIONS}
onChange={(event) => setPageSize(parseInt(event.target.value))}
value={pageSize}
className="filter-item"
/>
</FormGroup>
</FormGroup>
</>
);
}
interface ISceneParserFieldProps {
parserResult : ParserResult<any>
className? : string
onSetChanged : (set : boolean) => void
onValueChanged : (value : any) => void
}
function SceneParserField(props : ISceneParserFieldProps) {
const [value, setValue] = useState<string>(props.parserResult.value);
function maybeValueChanged() {
if (value !== props.parserResult.value) {
props.onValueChanged(value);
}
}
useEffect(() => {
setValue(props.parserResult.value);
}, [props.parserResult.value]);
return (
<>
<td>
<Checkbox
checked={props.parserResult.set}
inline={true}
onChange={() => {props.onSetChanged(!props.parserResult.set)}}
/>
</td>
<td>
<FormGroup>
<InputGroup
key="originalValue"
className={props.className}
small={true}
disabled={true}
value={props.parserResult.originalValue || ""}
/>
<InputGroup
key="newValue"
className={props.className}
small={true}
onChange={(event : any) => {setValue(event.target.value)}}
onBlur={() => maybeValueChanged()}
disabled={!props.parserResult.set}
value={value || ""}
autoComplete={"new-password" /* required to prevent Chrome autofilling */}
/>
</FormGroup>
</td>
</>
);
}
interface ISceneParserRowProps {
scene : SceneParserResult,
onChange: (changedScene : SceneParserResult) => void
}
function SceneParserRow(props : ISceneParserRowProps) {
function changeParser(result : ParserResult<any>, set : boolean, value : any) {
var newParser = _.clone(result);
newParser.set = set;
newParser.value = value;
return newParser;
}
function onTitleChanged(set : boolean, value: string | undefined) {
var newResult = _.clone(props.scene);
newResult.title = changeParser(newResult.title, set, value);
props.onChange(newResult);
}
function onDateChanged(set : boolean, value: string | undefined) {
var newResult = _.clone(props.scene);
newResult.date = changeParser(newResult.date, set, value);
props.onChange(newResult);
}
return (
<>
<tr className="scene-parser-row">
<td style={{textAlign: "left"}}>
{props.scene.filename}
</td>
<SceneParserField
key="title"
className="title"
parserResult={props.scene.title}
onSetChanged={(set) => onTitleChanged(set, props.scene.title.value)}
onValueChanged={(value) => onTitleChanged(props.scene.title.set, value)}
/>
<SceneParserField
key="date"
parserResult={props.scene.date}
onSetChanged={(set) => onDateChanged(set, props.scene.date.value)}
onValueChanged={(value) => onDateChanged(props.scene.date.set, value)}
/>
{/*<td>
</td>
<td>
</td>
<td>
</td>*/}
</tr>
</>
)
}
function onChange(scene : SceneParserResult, changedScene : SceneParserResult) {
var newResult = [...parserResult];
var index = newResult.indexOf(scene);
newResult[index] = changedScene;
setParserResult(newResult);
}
function renderTable() {
if (parserResult.length == 0) { return undefined; }
return (
<>
<form autoComplete="off">
<div className="grid">
<HTMLTable condensed={true}>
<thead>
<tr className="scene-parser-row">
<th>Filename</th>
<td>
<Checkbox
checked={allTitleSet}
inline={true}
onChange={() => {onSelectAllTitleSet(!allTitleSet)}}
/>
</td>
<th>Title</th>
<td>
<Checkbox
checked={allDateSet}
inline={true}
onChange={() => {onSelectAllDateSet(!allDateSet)}}
/>
</td>
<th>Date</th>
{/* TODO <th>Tags</th>
<th>Performers</th>
<th>Studio</th>*/}
</tr>
</thead>
<tbody>
{parserResult.map((scene) =>
<SceneParserRow
scene={scene}
key={scene.id}
onChange={(changedScene) => onChange(scene, changedScene)}/>
)}
</tbody>
</HTMLTable>
</div>
<Pagination
currentPage={page}
itemsPerPage={pageSize}
totalItems={totalItems}
onChangePage={(page) => setPage(page)}
/>
<Button intent="primary" text="Apply" onClick={() => onApply()}></Button>
</form>
</>
)
}
return (
<Card id="parser-container">
<H4>Scene Filename Parser</H4>
<ParserInput
input={parserInput}
onFind={(input) => onFindClicked(input)}
/>
{isLoading ? <Spinner size={Spinner.SIZE_LARGE} /> : undefined}
{renderTable()}
</Card>
);
};

View File

@@ -183,6 +183,10 @@ export class StashService {
return GQL.useBulkSceneUpdate({ variables: input, refetchQueries: ["FindScenes"] }); return GQL.useBulkSceneUpdate({ variables: input, refetchQueries: ["FindScenes"] });
} }
public static useScenesUpdate(input: GQL.SceneUpdateInput[]) {
return GQL.useScenesUpdate({ variables: { input : input }});
}
public static useSceneDestroy(input: GQL.SceneDestroyInput) { public static useSceneDestroy(input: GQL.SceneDestroyInput) {
return GQL.useSceneDestroy({ variables: input }); return GQL.useSceneDestroy({ variables: input });
} }
@@ -275,6 +279,13 @@ export class StashService {
}); });
} }
public static querySceneByPathRegex(filter: GQL.FindFilterType) {
return StashService.client.query<GQL.FindScenesByPathRegexQuery>({
query: GQL.FindScenesByPathRegexDocument,
variables: {filter: filter},
});
}
public static nullToUndefined(value: any): any { public static nullToUndefined(value: any): any {
if (_.isPlainObject(value)) { if (_.isPlainObject(value)) {
return _.mapValues(value, StashService.nullToUndefined); return _.mapValues(value, StashService.nullToUndefined);

View File

@@ -260,3 +260,36 @@ span.block {
text-decoration: underline; text-decoration: underline;
} }
} }
#parser-container {
margin: 10px auto;
width: 75%;
}
#parser-container .inputs label {
width: 12em;
}
#parser-container .inputs .bp3-input-group {
width: 80ch;
}
.scene-parser-row .bp3-checkbox {
margin: 0px -20px 0px 0px;
}
.scene-parser-row .title input {
width: 50ch;
}
.scene-parser-row input {
min-width: 10ch;
}
.scene-parser-row .bp3-form-group {
margin-bottom: 0px;
}
.scene-parser-row div:first-child > input {
margin-bottom: 5px;
}