Movie URLs (#4900)

* Fix exclude behaviour for stringListCriterionHandlerBuilder

pkg/sqlite/anonymise.go

@@ -847,7 +847,6 @@ func (db *Anonymiser) anonymiseMovies(ctx context.Context) error {
 				table.Col("name"),
 				table.Col("aliases"),
 				table.Col("synopsis"),
-				table.Col("url"),
 				table.Col("director"),
 			).Where(table.Col(idColumn).Gt(lastID)).Limit(1000)
 
@@ -860,7 +859,6 @@ func (db *Anonymiser) anonymiseMovies(ctx context.Context) error {
 					name     sql.NullString
 					aliases  sql.NullString
 					synopsis sql.NullString
-					url      sql.NullString
 					director sql.NullString
 				)
 
@@ -869,7 +867,6 @@ func (db *Anonymiser) anonymiseMovies(ctx context.Context) error {
 					&name,
 					&aliases,
 					&synopsis,
-					&url,
 					&director,
 				); err != nil {
 					return err
@@ -879,7 +876,6 @@ func (db *Anonymiser) anonymiseMovies(ctx context.Context) error {
 				db.obfuscateNullString(set, "name", name)
 				db.obfuscateNullString(set, "aliases", aliases)
 				db.obfuscateNullString(set, "synopsis", synopsis)
-				db.obfuscateNullString(set, "url", url)
 				db.obfuscateNullString(set, "director", director)
 
 				if len(set) > 0 {
@@ -905,6 +901,10 @@ func (db *Anonymiser) anonymiseMovies(ctx context.Context) error {
 		}
 	}
 
+	if err := db.anonymiseURLs(ctx, goqu.T(movieURLsTable), "movie_id"); err != nil {
+		return err
+	}
+
 	return nil
 }
 

pkg/sqlite/criterion_handlers.go

@@ -517,20 +517,51 @@ func (m *countCriterionHandlerBuilder) handler(criterion *models.IntCriterionInp
 
 // handler for StringCriterion for string list fields
 type stringListCriterionHandlerBuilder struct {
+	primaryTable string
+	// foreign key of the primary object on the join table
+	primaryFK string
 	// table joining primary and foreign objects
 	joinTable string
 	// string field on the join table
 	stringColumn string
 
-	addJoinTable func(f *filterBuilder)
+	addJoinTable   func(f *filterBuilder)
+	excludeHandler func(f *filterBuilder, criterion *models.StringCriterionInput)
 }
 
 func (m *stringListCriterionHandlerBuilder) handler(criterion *models.StringCriterionInput) criterionHandlerFunc {
 	return func(ctx context.Context, f *filterBuilder) {
 		if criterion != nil {
-			m.addJoinTable(f)
+			if criterion.Modifier == models.CriterionModifierExcludes {
+				// special handling for excludes
+				if m.excludeHandler != nil {
+					m.excludeHandler(f, criterion)
+					return
+				}
 
-			stringCriterionHandler(criterion, m.joinTable+"."+m.stringColumn)(ctx, f)
+				// excludes all of the provided values
+				// need to use actual join table name for this
+				// <primaryTable>.id NOT IN (select <joinTable>.<primaryFK> from <joinTable> where <joinTable>.<foreignFK> in <values>)
+				whereClause := utils.StrFormat("{primaryTable}.id NOT IN (SELECT {joinTable}.{primaryFK} from {joinTable} where {joinTable}.{stringColumn} LIKE ?)",
+					utils.StrFormatMap{
+						"primaryTable": m.primaryTable,
+						"joinTable":    m.joinTable,
+						"primaryFK":    m.primaryFK,
+						"stringColumn": m.stringColumn,
+					},
+				)
+
+				f.addWhere(whereClause, "%"+criterion.Value+"%")
+
+				// TODO - should we also exclude null values?
+				// m.addJoinTable(f)
+				// stringCriterionHandler(&models.StringCriterionInput{
+				// 	Modifier: models.CriterionModifierNotNull,
+				// }, m.joinTable+"."+m.stringColumn)(ctx, f)
+			} else {
+				m.addJoinTable(f)
+				stringCriterionHandler(criterion, m.joinTable+"."+m.stringColumn)(ctx, f)
+			}
 		}
 	}
 }

pkg/sqlite/database.go

@@ -30,7 +30,7 @@ const (
 	dbConnTimeout = 30
 )
 
-var appSchemaVersion uint = 58
+var appSchemaVersion uint = 59
 
 //go:embed migrations/*.sql
 var migrationsBox embed.FS

pkg/sqlite/gallery_filter.go

@@ -151,6 +151,8 @@ func (qb *galleryFilterHandler) criterionHandler() criterionHandler {
 
 func (qb *galleryFilterHandler) urlsCriterionHandler(url *models.StringCriterionInput) criterionHandlerFunc {
 	h := stringListCriterionHandlerBuilder{
+		primaryTable: galleryTable,
+		primaryFK:    galleryIDColumn,
 		joinTable:    galleriesURLsTable,
 		stringColumn: galleriesURLColumn,
 		addJoinTable: func(f *filterBuilder) {

pkg/sqlite/image_filter.go

@@ -160,6 +160,8 @@ func (qb *imageFilterHandler) missingCriterionHandler(isMissing *string) criteri
 
 func (qb *imageFilterHandler) urlsCriterionHandler(url *models.StringCriterionInput) criterionHandlerFunc {
 	h := stringListCriterionHandlerBuilder{
+		primaryTable: imageTable,
+		primaryFK:    imageIDColumn,
 		joinTable:    imagesURLsTable,
 		stringColumn: imageURLColumn,
 		addJoinTable: func(f *filterBuilder) {

pkg/sqlite/migrations/59_movie_urls.up.sql

@@ -0,0 +1,83 @@
+PRAGMA foreign_keys=OFF;
+
+CREATE TABLE `movie_urls` (
+  `movie_id` integer NOT NULL,
+  `position` integer NOT NULL,
+  `url` varchar(255) NOT NULL,
+  foreign key(`movie_id`) references `movies`(`id`) on delete CASCADE,
+  PRIMARY KEY(`movie_id`, `position`, `url`)
+);
+
+CREATE INDEX `movie_urls_url` on `movie_urls` (`url`);
+
+-- drop url
+CREATE TABLE `movies_new` (
+  `id` integer not null primary key autoincrement,
+  `name` varchar(255) not null,
+  `aliases` varchar(255),
+  `duration` integer,
+  `date` date,
+  `rating` tinyint,
+  `studio_id` integer REFERENCES `studios`(`id`) ON DELETE SET NULL,
+  `director` varchar(255),
+  `synopsis` text,
+  `created_at` datetime not null,
+  `updated_at` datetime not null, 
+  `front_image_blob` varchar(255) REFERENCES `blobs`(`checksum`), 
+  `back_image_blob` varchar(255) REFERENCES `blobs`(`checksum`)
+);
+
+INSERT INTO `movies_new`
+  (
+    `id`,
+    `name`,
+    `aliases`,
+    `duration`,
+    `date`,
+    `rating`,
+    `studio_id`,
+    `director`,
+    `synopsis`,
+    `created_at`,
+    `updated_at`,
+    `front_image_blob`,
+    `back_image_blob`
+  )
+  SELECT 
+    `id`,
+    `name`,
+    `aliases`,
+    `duration`,
+    `date`,
+    `rating`,
+    `studio_id`,
+    `director`,
+    `synopsis`,
+    `created_at`,
+    `updated_at`,
+    `front_image_blob`,
+    `back_image_blob`
+  FROM `movies`;
+
+INSERT INTO `movie_urls`
+  (
+    `movie_id`,
+    `position`,
+    `url`
+  )
+  SELECT 
+    `id`,
+    '0',
+    `url`
+  FROM `movies`
+  WHERE `movies`.`url` IS NOT NULL AND `movies`.`url` != '';
+
+DROP INDEX `index_movies_on_name_unique`;
+DROP INDEX `index_movies_on_studio_id`;
+DROP TABLE `movies`;
+ALTER TABLE `movies_new` rename to `movies`;
+
+CREATE INDEX `index_movies_on_name` ON `movies`(`name`);
+CREATE INDEX `index_movies_on_studio_id` on `movies` (`studio_id`);
+
+PRAGMA foreign_keys=ON;

pkg/sqlite/movies.go

@@ -22,6 +22,9 @@ const (
 
 	movieFrontImageBlobColumn = "front_image_blob"
 	movieBackImageBlobColumn  = "back_image_blob"
+
+	movieURLsTable = "movie_urls"
+	movieURLColumn = "url"
 )
 
 type movieRow struct {
@@ -35,7 +38,6 @@ type movieRow struct {
 	StudioID  null.Int    `db:"studio_id,omitempty"`
 	Director  zero.String `db:"director"`
 	Synopsis  zero.String `db:"synopsis"`
-	URL       zero.String `db:"url"`
 	CreatedAt Timestamp   `db:"created_at"`
 	UpdatedAt Timestamp   `db:"updated_at"`
 
@@ -54,7 +56,6 @@ func (r *movieRow) fromMovie(o models.Movie) {
 	r.StudioID = intFromPtr(o.StudioID)
 	r.Director = zero.StringFrom(o.Director)
 	r.Synopsis = zero.StringFrom(o.Synopsis)
-	r.URL = zero.StringFrom(o.URL)
 	r.CreatedAt = Timestamp{Timestamp: o.CreatedAt}
 	r.UpdatedAt = Timestamp{Timestamp: o.UpdatedAt}
 }
@@ -70,7 +71,6 @@ func (r *movieRow) resolve() *models.Movie {
 		StudioID:  nullIntPtr(r.StudioID),
 		Director:  r.Director.String,
 		Synopsis:  r.Synopsis.String,
-		URL:       r.URL.String,
 		CreatedAt: r.CreatedAt.Timestamp,
 		UpdatedAt: r.UpdatedAt.Timestamp,
 	}
@@ -91,7 +91,6 @@ func (r *movieRowRecord) fromPartial(o models.MoviePartial) {
 	r.setNullInt("studio_id", o.StudioID)
 	r.setNullString("director", o.Director)
 	r.setNullString("synopsis", o.Synopsis)
-	r.setNullString("url", o.URL)
 	r.setTimestamp("created_at", o.CreatedAt)
 	r.setTimestamp("updated_at", o.UpdatedAt)
 }
@@ -148,6 +147,13 @@ func (qb *MovieStore) Create(ctx context.Context, newObject *models.Movie) error
 		return err
 	}
 
+	if newObject.URLs.Loaded() {
+		const startPos = 0
+		if err := moviesURLsTableMgr.insertJoins(ctx, id, startPos, newObject.URLs.List()); err != nil {
+			return err
+		}
+	}
+
 	updated, err := qb.find(ctx, id)
 	if err != nil {
 		return fmt.Errorf("finding after create: %w", err)
@@ -173,6 +179,12 @@ func (qb *MovieStore) UpdatePartial(ctx context.Context, id int, partial models.
 		}
 	}
 
+	if partial.URLs != nil {
+		if err := moviesURLsTableMgr.modifyJoins(ctx, id, partial.URLs.Values, partial.URLs.Mode); err != nil {
+			return nil, err
+		}
+	}
+
 	return qb.find(ctx, id)
 }
 
@@ -184,6 +196,12 @@ func (qb *MovieStore) Update(ctx context.Context, updatedObject *models.Movie) e
 		return err
 	}
 
+	if updatedObject.URLs.Loaded() {
+		if err := moviesURLsTableMgr.replaceJoins(ctx, updatedObject.ID, updatedObject.URLs.List()); err != nil {
+			return err
+		}
+	}
+
 	return nil
 }
 
@@ -537,3 +555,7 @@ WHERE movies.studio_id = ?
 	args := []interface{}{studioID}
 	return movieRepository.runCountQuery(ctx, query, args)
 }
+
+func (qb *MovieStore) GetURLs(ctx context.Context, movieID int) ([]string, error) {
+	return moviesURLsTableMgr.get(ctx, movieID)
+}

pkg/sqlite/movies_filter.go

@@ -60,7 +60,7 @@ func (qb *movieFilterHandler) criterionHandler() criterionHandler {
 		intCriterionHandler(movieFilter.Rating100, "movies.rating", nil),
 		floatIntCriterionHandler(movieFilter.Duration, "movies.duration", nil),
 		qb.missingCriterionHandler(movieFilter.IsMissing),
-		stringCriterionHandler(movieFilter.URL, "movies.url"),
+		qb.urlsCriterionHandler(movieFilter.URL),
 		studioCriterionHandler(movieTable, movieFilter.Studios),
 		qb.performersCriterionHandler(movieFilter.Performers),
 		&dateCriterionHandler{movieFilter.Date, "movies.date", nil},
@@ -102,6 +102,20 @@ func (qb *movieFilterHandler) missingCriterionHandler(isMissing *string) criteri
 	}
 }
 
+func (qb *movieFilterHandler) urlsCriterionHandler(url *models.StringCriterionInput) criterionHandlerFunc {
+	h := stringListCriterionHandlerBuilder{
+		primaryTable: movieTable,
+		primaryFK:    movieIDColumn,
+		joinTable:    movieURLsTable,
+		stringColumn: movieURLColumn,
+		addJoinTable: func(f *filterBuilder) {
+			moviesURLsTableMgr.join(f, "", "movies.id")
+		},
+	}
+
+	return h.handler(url)
+}
+
 func (qb *movieFilterHandler) performersCriterionHandler(performers *models.MultiCriterionInput) criterionHandlerFunc {
 	return func(ctx context.Context, f *filterBuilder) {
 		if performers != nil {

pkg/sqlite/movies_test.go

@@ -15,6 +15,16 @@ import (
 	"github.com/stashapp/stash/pkg/models"
 )
 
+func loadMovieRelationships(ctx context.Context, expected models.Movie, actual *models.Movie) error {
+	if expected.URLs.Loaded() {
+		if err := actual.LoadURLs(ctx, db.Gallery); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
 func TestMovieFindByName(t *testing.T) {
 	withTxn(func(ctx context.Context) error {
 		mqb := db.Movie
@@ -205,7 +215,14 @@ func TestMovieQueryURL(t *testing.T) {
 
 	verifyFn := func(n *models.Movie) {
 		t.Helper()
-		verifyString(t, n.URL, urlCriterion)
+
+		urls := n.URLs.List()
+		var url string
+		if len(urls) > 0 {
+			url = urls[0]
+		}
+
+		verifyString(t, url, urlCriterion)
 	}
 
 	verifyMovieQuery(t, filter, verifyFn)
@@ -228,6 +245,56 @@ func TestMovieQueryURL(t *testing.T) {
 	verifyMovieQuery(t, filter, verifyFn)
 }
 
+func TestMovieQueryURLExcludes(t *testing.T) {
+	withRollbackTxn(func(ctx context.Context) error {
+		mqb := db.Movie
+
+		// create movie with two URLs
+		movie := models.Movie{
+			Name: "TestMovieQueryURLExcludes",
+			URLs: models.NewRelatedStrings([]string{
+				"aaa",
+				"bbb",
+			}),
+		}
+
+		err := mqb.Create(ctx, &movie)
+
+		if err != nil {
+			return fmt.Errorf("Error creating movie: %w", err)
+		}
+
+		// query for movies that exclude the URL "aaa"
+		urlCriterion := models.StringCriterionInput{
+			Value:    "aaa",
+			Modifier: models.CriterionModifierExcludes,
+		}
+
+		nameCriterion := models.StringCriterionInput{
+			Value:    movie.Name,
+			Modifier: models.CriterionModifierEquals,
+		}
+
+		filter := models.MovieFilterType{
+			URL:  &urlCriterion,
+			Name: &nameCriterion,
+		}
+
+		movies := queryMovie(ctx, t, mqb, &filter, nil)
+		assert.Len(t, movies, 0, "Expected no movies to be found")
+
+		// query for movies that exclude the URL "ccc"
+		urlCriterion.Value = "ccc"
+		movies = queryMovie(ctx, t, mqb, &filter, nil)
+
+		if assert.Len(t, movies, 1, "Expected one movie to be found") {
+			assert.Equal(t, movie.Name, movies[0].Name)
+		}
+
+		return nil
+	})
+}
+
 func verifyMovieQuery(t *testing.T, filter models.MovieFilterType, verifyFn func(s *models.Movie)) {
 	withTxn(func(ctx context.Context) error {
 		t.Helper()
@@ -235,6 +302,12 @@ func verifyMovieQuery(t *testing.T, filter models.MovieFilterType, verifyFn func
 
 		movies := queryMovie(ctx, t, sqb, &filter, nil)
 
+		for _, movie := range movies {
+			if err := movie.LoadURLs(ctx, sqb); err != nil {
+				t.Errorf("Error loading movie relationships: %v", err)
+			}
+		}
+
 		// assume it should find at least one
 		assert.Greater(t, len(movies), 0)
 

pkg/sqlite/performer_filter.go

@@ -243,6 +243,8 @@ func (qb *performerFilterHandler) performerAgeFilterCriterionHandler(age *models
 
 func (qb *performerFilterHandler) aliasCriterionHandler(alias *models.StringCriterionInput) criterionHandlerFunc {
 	h := stringListCriterionHandlerBuilder{
+		primaryTable: performerTable,
+		primaryFK:    performerIDColumn,
 		joinTable:    performersAliasesTable,
 		stringColumn: performerAliasColumn,
 		addJoinTable: func(f *filterBuilder) {

pkg/sqlite/scene_filter.go

@@ -345,6 +345,8 @@ func (qb *sceneFilterHandler) isMissingCriterionHandler(isMissing *string) crite
 
 func (qb *sceneFilterHandler) urlsCriterionHandler(url *models.StringCriterionInput) criterionHandlerFunc {
 	h := stringListCriterionHandlerBuilder{
+		primaryTable: sceneTable,
+		primaryFK:    sceneIDColumn,
 		joinTable:    scenesURLsTable,
 		stringColumn: sceneURLColumn,
 		addJoinTable: func(f *filterBuilder) {
@@ -368,12 +370,24 @@ func (qb *sceneFilterHandler) getMultiCriterionHandlerBuilder(foreignTable, join
 
 func (qb *sceneFilterHandler) captionCriterionHandler(captions *models.StringCriterionInput) criterionHandlerFunc {
 	h := stringListCriterionHandlerBuilder{
+		primaryTable: sceneTable,
+		primaryFK:    sceneIDColumn,
 		joinTable:    videoCaptionsTable,
 		stringColumn: captionCodeColumn,
 		addJoinTable: func(f *filterBuilder) {
 			qb.addSceneFilesTable(f)
 			f.addLeftJoin(videoCaptionsTable, "", "video_captions.file_id = scenes_files.file_id")
 		},
+		excludeHandler: func(f *filterBuilder, criterion *models.StringCriterionInput) {
+			excludeClause := `scenes.id NOT IN (
+				SELECT scenes_files.scene_id from scenes_files 
+				INNER JOIN video_captions on video_captions.file_id = scenes_files.file_id 
+				WHERE video_captions.language_code LIKE ?
+			)`
+			f.addWhere(excludeClause, criterion.Value)
+
+			// TODO - should we also exclude null values?
+		},
 	}
 
 	return h.handler(captions)

pkg/sqlite/setup_test.go

@@ -1303,6 +1303,15 @@ func getMovieNullStringValue(index int, field string) string {
 	return ret.String
 }
 
+func getMovieEmptyString(index int, field string) string {
+	v := getPrefixedNullStringValue("movie", index, field)
+	if !v.Valid {
+		return ""
+	}
+
+	return v.String
+}
+
 // createMoviees creates n movies with plain Name and o movies with camel cased NaMe included
 func createMovies(ctx context.Context, mqb models.MovieReaderWriter, n int, o int) error {
 	const namePlain = "Name"
@@ -1321,7 +1330,9 @@ func createMovies(ctx context.Context, mqb models.MovieReaderWriter, n int, o in
 		name = getMovieStringValue(index, name)
 		movie := models.Movie{
 			Name: name,
-			URL:  getMovieNullStringValue(index, urlField),
+			URLs: models.NewRelatedStrings([]string{
+				getMovieEmptyString(i, urlField),
+			}),
 		}
 
 		err := mqb.Create(ctx, &movie)

pkg/sqlite/studio_filter.go

@@ -178,6 +178,8 @@ func (qb *studioFilterHandler) parentCriterionHandler(parents *models.MultiCrite
 
 func (qb *studioFilterHandler) aliasCriterionHandler(alias *models.StringCriterionInput) criterionHandlerFunc {
 	h := stringListCriterionHandlerBuilder{
+		primaryTable: studioTable,
+		primaryFK:    studioIDColumn,
 		joinTable:    studioAliasesTable,
 		stringColumn: studioAliasColumn,
 		addJoinTable: func(f *filterBuilder) {

pkg/sqlite/tables.go

@@ -34,6 +34,8 @@ var (
 
 	studiosAliasesJoinTable  = goqu.T(studioAliasesTable)
 	studiosStashIDsJoinTable = goqu.T("studio_stash_ids")
+
+	moviesURLsJoinTable = goqu.T(movieURLsTable)
 )
 
 var (
@@ -299,6 +301,14 @@ var (
 		table:    goqu.T(movieTable),
 		idColumn: goqu.T(movieTable).Col(idColumn),
 	}
+
+	moviesURLsTableMgr = &orderedValueTable[string]{
+		table: table{
+			table:    moviesURLsJoinTable,
+			idColumn: moviesURLsJoinTable.Col(movieIDColumn),
+		},
+		valueColumn: moviesURLsJoinTable.Col(movieURLColumn),
+	}
 )
 
 var (

pkg/sqlite/tag_filter.go

@@ -105,6 +105,8 @@ func (qb *tagFilterHandler) criterionHandler() criterionHandler {
 
 func (qb *tagFilterHandler) aliasCriterionHandler(alias *models.StringCriterionInput) criterionHandlerFunc {
 	h := stringListCriterionHandlerBuilder{
+		primaryTable: tagTable,
+		primaryFK:    tagIDColumn,
 		joinTable:    tagAliasesTable,
 		stringColumn: tagAliasColumn,
 		addJoinTable: func(f *filterBuilder) {