Mirrors/stash
1
0
Fork 0
mirror of https://github.com/stashapp/stash.git synced 2025-12-17 20:34:37 +03:00
Code Issues Packages Projects Releases Wiki Activity

Multiple scene URLs (#3852)

Browse Source 
* Add URLs scene relationship
* Update unit tests
* Update scene edit and details pages
* Update scrapers to use urls
* Post-process scenes during query scrape
* Update UI for URLs
* Change urls label
This commit is contained in:
WithoutPants
2023-07-12 11:51:52 +10:00
committed by GitHub
parent 76a4bfa49a
commit 67d4f9729a
50 changed files with 978 additions and 205 deletions
Download Patch File Download Diff File Expand all files Collapse all files

70
pkg/sqlite/anonymise.go
View File

@@ -230,7 +230,6 @@ func (db *Anonymiser) anonymiseScenes(ctx context.Context) error {
table.Col(idColumn),
table.Col("title"),
table.Col("details"),
table.Col("url"),
table.Col("code"),
table.Col("director"),
).Where(table.Col(idColumn).Gt(lastID)).Limit(1000)
@@ -243,7 +242,6 @@ func (db *Anonymiser) anonymiseScenes(ctx context.Context) error {
id int
title sql.NullString
details sql.NullString
url sql.NullString
code sql.NullString
director sql.NullString
)
@@ -252,7 +250,6 @@ func (db *Anonymiser) anonymiseScenes(ctx context.Context) error {
&id,
&title,
&details,
&url,
&code,
&director,
); err != nil {
@@ -264,7 +261,6 @@ func (db *Anonymiser) anonymiseScenes(ctx context.Context) error {
// if title set set new title
db.obfuscateNullString(set, "title", title)
db.obfuscateNullString(set, "details", details)
db.obfuscateNullString(set, "url", url)
if len(set) > 0 {
stmt := dialect.Update(table).Set(set).Where(table.Col(idColumn).Eq(id))
@@ -301,6 +297,10 @@ func (db *Anonymiser) anonymiseScenes(ctx context.Context) error {
}
}
if err := db.anonymiseURLs(ctx, goqu.T(scenesURLsTable), "scene_id"); err != nil {
return err
}
return nil
}
@@ -704,6 +704,68 @@ func (db *Anonymiser) anonymiseAliases(ctx context.Context, table exp.Identifier
return nil
}
func (db *Anonymiser) anonymiseURLs(ctx context.Context, table exp.IdentifierExpression, idColumn string) error {
lastID := 0
lastURL := ""
total := 0
const logEvery = 10000
for gotSome := true; gotSome; {
if err := txn.WithTxn(ctx, db, func(ctx context.Context) error {
query := dialect.From(table).Select(
table.Col(idColumn),
table.Col("url"),
).Where(goqu.L("(" + idColumn + ", url)").Gt(goqu.L("(?, ?)", lastID, lastURL))).Limit(1000)
gotSome = false
const single = false
return queryFunc(ctx, query, single, func(rows *sqlx.Rows) error {
var (
id int
url sql.NullString
)
if err := rows.Scan(
&id,
&url,
); err != nil {
return err
}
set := goqu.Record{}
db.obfuscateNullString(set, "url", url)
if len(set) > 0 {
stmt := dialect.Update(table).Set(set).Where(
table.Col(idColumn).Eq(id),
table.Col("url").Eq(url),
)
if _, err := exec(ctx, stmt); err != nil {
return fmt.Errorf("anonymising %s: %w", table.GetTable(), err)
}
}
lastID = id
lastURL = url.String
gotSome = true
total++
if total%logEvery == 0 {
logger.Infof("Anonymised %d %s URLs", total, table.GetTable())
}
return nil
})
}); err != nil {
return err
}
}
return nil
}
func (db *Anonymiser) anonymiseTags(ctx context.Context) error {
logger.Infof("Anonymising tags")
table := tagTableMgr.table

2
pkg/sqlite/database.go
View File

@@ -32,7 +32,7 @@ const (
dbConnTimeout = 30
)
var appSchemaVersion uint = 46
var appSchemaVersion uint = 47
//go:embed migrations/*.sql
var migrationsBox embed.FS

94
pkg/sqlite/migrations/47_scene_urls.up.sql Normal file
View File

@@ -0,0 +1,94 @@
PRAGMA foreign_keys=OFF;
CREATE TABLE `scene_urls` (
`scene_id` integer NOT NULL,
`position` integer NOT NULL,
`url` varchar(255) NOT NULL,
foreign key(`scene_id`) references `scenes`(`id`) on delete CASCADE,
PRIMARY KEY(`scene_id`, `position`, `url`)
);
CREATE INDEX `scene_urls_url` on `scene_urls` (`url`);
-- drop url
CREATE TABLE "scenes_new" (
`id` integer not null primary key autoincrement,
`title` varchar(255),
`details` text,
`date` date,
`rating` tinyint,
`studio_id` integer,
`o_counter` tinyint not null default 0,
`organized` boolean not null default '0',
`created_at` datetime not null,
`updated_at` datetime not null,
`code` text,
`director` text,
`resume_time` float not null default 0,
`last_played_at` datetime default null,
`play_count` tinyint not null default 0,
`play_duration` float not null default 0,
`cover_blob` varchar(255) REFERENCES `blobs`(`checksum`),
foreign key(`studio_id`) references `studios`(`id`) on delete SET NULL
);
INSERT INTO `scenes_new`
(
`id`,
`title`,
`details`,
`date`,
`rating`,
`studio_id`,
`o_counter`,
`organized`,
`created_at`,
`updated_at`,
`code`,
`director`,
`resume_time`,
`last_played_at`,
`play_count`,
`play_duration`,
`cover_blob`
)
SELECT
`id`,
`title`,
`details`,
`date`,
`rating`,
`studio_id`,
`o_counter`,
`organized`,
`created_at`,
`updated_at`,
`code`,
`director`,
`resume_time`,
`last_played_at`,
`play_count`,
`play_duration`,
`cover_blob`
FROM `scenes`;
INSERT INTO `scene_urls`
(
`scene_id`,
`position`,
`url`
)
SELECT
`id`,
'0',
`url`
FROM `scenes`
WHERE `scenes`.`url` IS NOT NULL AND `scenes`.`url` != '';
DROP INDEX `index_scenes_on_studio_id`;
DROP TABLE `scenes`;
ALTER TABLE `scenes_new` rename to `scenes`;
CREATE INDEX `index_scenes_on_studio_id` on `scenes` (`studio_id`);
PRAGMA foreign_keys=ON;

42
pkg/sqlite/scene.go
View File

@@ -31,6 +31,8 @@ const (
scenesTagsTable = "scenes_tags"
scenesGalleriesTable = "scenes_galleries"
moviesScenesTable = "movies_scenes"
scenesURLsTable = "scene_urls"
sceneURLColumn = "url"
sceneCoverBlobColumn = "cover_blob"
)
@@ -76,7 +78,6 @@ type sceneRow struct {
Code zero.String `db:"code"`
Details zero.String `db:"details"`
Director zero.String `db:"director"`
URL zero.String `db:"url"`
Date NullDate `db:"date"`
// expressed as 1-100
Rating null.Int `db:"rating"`
@@ -100,7 +101,6 @@ func (r *sceneRow) fromScene(o models.Scene) {
r.Code = zero.StringFrom(o.Code)
r.Details = zero.StringFrom(o.Details)
r.Director = zero.StringFrom(o.Director)
r.URL = zero.StringFrom(o.URL)
r.Date = NullDateFromDatePtr(o.Date)
r.Rating = intFromPtr(o.Rating)
r.Organized = o.Organized
@@ -130,7 +130,6 @@ func (r *sceneQueryRow) resolve() *models.Scene {
Code: r.Code.String,
Details: r.Details.String,
Director: r.Director.String,
URL: r.URL.String,
Date: r.Date.DatePtr(),
Rating: nullIntPtr(r.Rating),
Organized: r.Organized,
@@ -166,7 +165,6 @@ func (r *sceneRowRecord) fromPartial(o models.ScenePartial) {
r.setNullString("code", o.Code)
r.setNullString("details", o.Details)
r.setNullString("director", o.Director)
r.setNullString("url", o.URL)
r.setNullDate("date", o.Date)
r.setNullInt("rating", o.Rating)
r.setBool("organized", o.Organized)
@@ -268,6 +266,13 @@ func (qb *SceneStore) Create(ctx context.Context, newObject *models.Scene, fileI
}
}
if newObject.URLs.Loaded() {
const startPos = 0
if err := scenesURLsTableMgr.insertJoins(ctx, id, startPos, newObject.URLs.List()); err != nil {
return err
}
}
if newObject.PerformerIDs.Loaded() {
if err := scenesPerformersTableMgr.insertJoins(ctx, id, newObject.PerformerIDs.List()); err != nil {
return err
@@ -322,6 +327,11 @@ func (qb *SceneStore) UpdatePartial(ctx context.Context, id int, partial models.
}
}
if partial.URLs != nil {
if err := scenesURLsTableMgr.modifyJoins(ctx, id, partial.URLs.Values, partial.URLs.Mode); err != nil {
return nil, err
}
}
if partial.PerformerIDs != nil {
if err := scenesPerformersTableMgr.modifyJoins(ctx, id, partial.PerformerIDs.IDs, partial.PerformerIDs.Mode); err != nil {
return nil, err
@@ -364,6 +374,12 @@ func (qb *SceneStore) Update(ctx context.Context, updatedObject *models.Scene) e
return err
}
if updatedObject.URLs.Loaded() {
if err := scenesURLsTableMgr.replaceJoins(ctx, updatedObject.ID, updatedObject.URLs.List()); err != nil {
return err
}
}
if updatedObject.PerformerIDs.Loaded() {
if err := scenesPerformersTableMgr.replaceJoins(ctx, updatedObject.ID, updatedObject.PerformerIDs.List()); err != nil {
return err
@@ -974,7 +990,7 @@ func (qb *SceneStore) makeFilter(ctx context.Context, sceneFilter *models.SceneF
query.handleCriterion(ctx, hasMarkersCriterionHandler(sceneFilter.HasMarkers))
query.handleCriterion(ctx, sceneIsMissingCriterionHandler(qb, sceneFilter.IsMissing))
query.handleCriterion(ctx, stringCriterionHandler(sceneFilter.URL, "scenes.url"))
query.handleCriterion(ctx, sceneURLsCriterionHandler(sceneFilter.URL))
query.handleCriterion(ctx, criterionHandlerFunc(func(ctx context.Context, f *filterBuilder) {
if sceneFilter.StashID != nil {
@@ -1308,6 +1324,18 @@ func sceneIsMissingCriterionHandler(qb *SceneStore, isMissing *string) criterion
}
}
func sceneURLsCriterionHandler(url *models.StringCriterionInput) criterionHandlerFunc {
h := stringListCriterionHandlerBuilder{
joinTable: scenesURLsTable,
stringColumn: sceneURLColumn,
addJoinTable: func(f *filterBuilder) {
scenesURLsTableMgr.join(f, "", "scenes.id")
},
}
return h.handler(url)
}
func (qb *SceneStore) getMultiCriterionHandlerBuilder(foreignTable, joinTable, foreignFK string, addJoinsFunc func(f *filterBuilder)) multiCriterionHandlerBuilder {
return multiCriterionHandlerBuilder{
primaryTable: sceneTable,
@@ -1637,6 +1665,10 @@ func (qb *SceneStore) IncrementWatchCount(ctx context.Context, id int) (int, err
return qb.getPlayCount(ctx, id)
}
func (qb *SceneStore) GetURLs(ctx context.Context, sceneID int) ([]string, error) {
return scenesURLsTableMgr.get(ctx, sceneID)
}
func (qb *SceneStore) GetCover(ctx context.Context, sceneID int) ([]byte, error) {
return qb.GetImage(ctx, sceneID, sceneCoverBlobColumn)
}

44
pkg/sqlite/scene_test.go
View File

@@ -21,6 +21,12 @@ import (
)
func loadSceneRelationships(ctx context.Context, expected models.Scene, actual *models.Scene) error {
if expected.URLs.Loaded() {
if err := actual.LoadURLs(ctx, db.Scene); err != nil {
return err
}
}
if expected.GalleryIDs.Loaded() {
if err := actual.LoadGalleryIDs(ctx, db.Scene); err != nil {
return err
@@ -108,7 +114,7 @@ func Test_sceneQueryBuilder_Create(t *testing.T) {
Code: code,
Details: details,
Director: director,
URL: url,
URLs: models.NewRelatedStrings([]string{url}),
Date: &date,
Rating: &rating,
Organized: true,
@@ -153,7 +159,7 @@ func Test_sceneQueryBuilder_Create(t *testing.T) {
Code: code,
Details: details,
Director: director,
URL: url,
URLs: models.NewRelatedStrings([]string{url}),
Date: &date,
Rating: &rating,
Organized: true,
@@ -346,7 +352,7 @@ func Test_sceneQueryBuilder_Update(t *testing.T) {
Code: code,
Details: details,
Director: director,
URL: url,
URLs: models.NewRelatedStrings([]string{url}),
Date: &date,
Rating: &rating,
Organized: true,
@@ -513,7 +519,7 @@ func clearScenePartial() models.ScenePartial {
Code: models.OptionalString{Set: true, Null: true},
Details: models.OptionalString{Set: true, Null: true},
Director: models.OptionalString{Set: true, Null: true},
URL: models.OptionalString{Set: true, Null: true},
URLs: &models.UpdateStrings{Mode: models.RelationshipUpdateModeSet},
Date: models.OptionalDate{Set: true, Null: true},
Rating: models.OptionalInt{Set: true, Null: true},
StudioID: models.OptionalInt{Set: true, Null: true},
@@ -560,11 +566,14 @@ func Test_sceneQueryBuilder_UpdatePartial(t *testing.T) {
"full",
sceneIDs[sceneIdxWithSpacedName],
models.ScenePartial{
Title: models.NewOptionalString(title),
Code: models.NewOptionalString(code),
Details: models.NewOptionalString(details),
Director: models.NewOptionalString(director),
URL: models.NewOptionalString(url),
Title: models.NewOptionalString(title),
Code: models.NewOptionalString(code),
Details: models.NewOptionalString(details),
Director: models.NewOptionalString(director),
URLs: &models.UpdateStrings{
Values: []string{url},
Mode: models.RelationshipUpdateModeSet,
},
Date: models.NewOptionalDate(date),
Rating: models.NewOptionalInt(rating),
Organized: models.NewOptionalBool(true),
@@ -624,7 +633,7 @@ func Test_sceneQueryBuilder_UpdatePartial(t *testing.T) {
Code: code,
Details: details,
Director: director,
URL: url,
URLs: models.NewRelatedStrings([]string{url}),
Date: &date,
Rating: &rating,
Organized: true,
@@ -2400,7 +2409,14 @@ func TestSceneQueryURL(t *testing.T) {
verifyFn := func(s *models.Scene) {
t.Helper()
verifyString(t, s.URL, urlCriterion)
urls := s.URLs.List()
var url string
if len(urls) > 0 {
url = urls[0]
}
verifyString(t, url, urlCriterion)
}
verifySceneQuery(t, filter, verifyFn)
@@ -2576,6 +2592,12 @@ func verifySceneQuery(t *testing.T, filter models.SceneFilterType, verifyFn func
scenes := queryScene(ctx, t, sqb, &filter, nil)
for _, scene := range scenes {
if err := scene.LoadRelationships(ctx, sqb); err != nil {
t.Errorf("Error loading scene relationships: %v", err)
}
}
// assume it should find at least one
assert.Greater(t, len(scenes), 0)

8
pkg/sqlite/setup_test.go
View File

@@ -1065,9 +1065,11 @@ func makeScene(i int) *models.Scene {
rating := getRating(i)
return &models.Scene{
Title: title,
Details: details,
URL: getSceneEmptyString(i, urlField),
Title: title,
Details: details,
URLs: models.NewRelatedStrings([]string{
getSceneEmptyString(i, urlField),
}),
Rating: getIntPtr(rating),
OCounter: getOCounter(i),
Date: getObjectDate(i),

108
pkg/sqlite/table.go
View File

@@ -14,6 +14,7 @@ import (
"github.com/stashapp/stash/pkg/file"
"github.com/stashapp/stash/pkg/logger"
"github.com/stashapp/stash/pkg/models"
"github.com/stashapp/stash/pkg/sliceutil"
"github.com/stashapp/stash/pkg/sliceutil/intslice"
"github.com/stashapp/stash/pkg/sliceutil/stringslice"
)
@@ -472,6 +473,113 @@ func (t *stringTable) modifyJoins(ctx context.Context, id int, v []string, mode
return nil
}
type orderedValueTable[T comparable] struct {
table
valueColumn exp.IdentifierExpression
}
func (t *orderedValueTable[T]) positionColumn() exp.IdentifierExpression {
const positionColumn = "position"
return t.table.table.Col(positionColumn)
}
func (t *orderedValueTable[T]) get(ctx context.Context, id int) ([]T, error) {
q := dialect.Select(t.valueColumn).From(t.table.table).Where(t.idColumn.Eq(id)).Order(t.positionColumn().Asc())
const single = false
var ret []T
if err := queryFunc(ctx, q, single, func(rows *sqlx.Rows) error {
var v T
if err := rows.Scan(&v); err != nil {
return err
}
ret = append(ret, v)
return nil
}); err != nil {
return nil, fmt.Errorf("getting stash ids from %s: %w", t.table.table.GetTable(), err)
}
return ret, nil
}
func (t *orderedValueTable[T]) insertJoin(ctx context.Context, id int, position int, v T) (sql.Result, error) {
q := dialect.Insert(t.table.table).Cols(t.idColumn.GetCol(), t.positionColumn().GetCol(), t.valueColumn.GetCol()).Vals(
goqu.Vals{id, position, v},
)
ret, err := exec(ctx, q)
if err != nil {
return nil, fmt.Errorf("inserting into %s: %w", t.table.table.GetTable(), err)
}
return ret, nil
}
func (t *orderedValueTable[T]) insertJoins(ctx context.Context, id int, startPos int, v []T) error {
for i, fk := range v {
if _, err := t.insertJoin(ctx, id, i+startPos, fk); err != nil {
return err
}
}
return nil
}
func (t *orderedValueTable[T]) replaceJoins(ctx context.Context, id int, v []T) error {
if err := t.destroy(ctx, []int{id}); err != nil {
return err
}
const startPos = 0
return t.insertJoins(ctx, id, startPos, v)
}
func (t *orderedValueTable[T]) addJoins(ctx context.Context, id int, v []T) error {
// get existing foreign keys
existing, err := t.get(ctx, id)
if err != nil {
return err
}
// only add values that are not already present
filtered := sliceutil.Exclude(v, existing)
if len(filtered) == 0 {
return nil
}
startPos := len(existing)
return t.insertJoins(ctx, id, startPos, filtered)
}
func (t *orderedValueTable[T]) destroyJoins(ctx context.Context, id int, v []T) error {
existing, err := t.get(ctx, id)
if err != nil {
return fmt.Errorf("getting existing %s: %w", t.table.table.GetTable(), err)
}
newValue := sliceutil.Exclude(existing, v)
if len(newValue) == len(existing) {
return nil
}
return t.replaceJoins(ctx, id, newValue)
}
func (t *orderedValueTable[T]) modifyJoins(ctx context.Context, id int, v []T, mode models.RelationshipUpdateMode) error {
switch mode {
case models.RelationshipUpdateModeSet:
return t.replaceJoins(ctx, id, v)
case models.RelationshipUpdateModeAdd:
return t.addJoins(ctx, id, v)
case models.RelationshipUpdateModeRemove:
return t.destroyJoins(ctx, id, v)
}
return nil
}
type scenesMoviesTable struct {
table
}

9
pkg/sqlite/tables.go
View File

@@ -24,6 +24,7 @@ var (
scenesPerformersJoinTable = goqu.T(performersScenesTable)
scenesStashIDsJoinTable = goqu.T("scene_stash_ids")
scenesMoviesJoinTable = goqu.T(moviesScenesTable)
scenesURLsJoinTable = goqu.T(scenesURLsTable)
performersAliasesJoinTable = goqu.T(performersAliasesTable)
performersTagsJoinTable = goqu.T(performersTagsTable)
@@ -160,6 +161,14 @@ var (
idColumn: scenesMoviesJoinTable.Col(sceneIDColumn),
},
}
scenesURLsTableMgr = &orderedValueTable[string]{
table: table{
table: scenesURLsJoinTable,
idColumn: scenesURLsJoinTable.Col(sceneIDColumn),
},
valueColumn: scenesURLsJoinTable.Col(sceneURLColumn),
}
)
var (