Gallery URLs (#4114)

* Initial backend changes
* Fix unit tests
* UI changes
* Fix missing URL filters
This commit is contained in:
WithoutPants
2023-09-25 12:27:20 +10:00
committed by GitHub
parent a369e395e7
commit 9577600804
29 changed files with 361 additions and 117 deletions

View File

@@ -14,7 +14,7 @@ import (
func ToBasicJSON(gallery *models.Gallery) (*jsonschema.Gallery, error) {
newGalleryJSON := jsonschema.Gallery{
Title: gallery.Title,
URL: gallery.URL,
URLs: gallery.URLs.List(),
Details: gallery.Details,
CreatedAt: json.JSONTime{Time: gallery.CreatedAt},
UpdatedAt: json.JSONTime{Time: gallery.UpdatedAt},

View File

@@ -59,7 +59,7 @@ func createFullGallery(id int) models.Gallery {
Details: details,
Rating: &rating,
Organized: organized,
URL: url,
URLs: models.NewRelatedStrings([]string{url}),
CreatedAt: createTime,
UpdatedAt: updateTime,
}
@@ -85,7 +85,7 @@ func createFullJSONGallery() *jsonschema.Gallery {
Details: details,
Rating: rating,
Organized: organized,
URL: url,
URLs: []string{url},
ZipFiles: []string{path},
CreatedAt: json.JSONTime{
Time: createTime,

View File

@@ -65,8 +65,10 @@ func (i *Importer) galleryJSONToGallery(galleryJSON jsonschema.Gallery) models.G
if galleryJSON.Details != "" {
newGallery.Details = galleryJSON.Details
}
if galleryJSON.URL != "" {
newGallery.URL = galleryJSON.URL
if len(galleryJSON.URLs) > 0 {
newGallery.URLs = models.NewRelatedStrings(galleryJSON.URLs)
} else if galleryJSON.URL != "" {
newGallery.URLs = models.NewRelatedStrings([]string{galleryJSON.URL})
}
if galleryJSON.Date != "" {
d, err := models.ParseDate(galleryJSON.Date)

View File

@@ -66,7 +66,7 @@ func TestImporterPreImport(t *testing.T) {
Details: details,
Rating: &rating,
Organized: organized,
URL: url,
URLs: models.NewRelatedStrings([]string{url}),
Files: models.NewRelatedFiles([]models.File{}),
TagIDs: models.NewRelatedIDs([]int{}),
PerformerIDs: models.NewRelatedIDs([]int{}),

View File

@@ -59,7 +59,7 @@ type GalleryUpdateInput struct {
ClientMutationID *string `json:"clientMutationId"`
ID string `json:"id"`
Title *string `json:"title"`
URL *string `json:"url"`
Urls []string `json:"urls"`
Date *string `json:"date"`
Details *string `json:"details"`
Rating *int `json:"rating"`
@@ -70,6 +70,9 @@ type GalleryUpdateInput struct {
TagIds []string `json:"tag_ids"`
PerformerIds []string `json:"performer_ids"`
PrimaryFileID *string `json:"primary_file_id"`
// deprecated
URL *string `json:"url"`
}
type GalleryDestroyInput struct {

View File

@@ -21,7 +21,7 @@ type Gallery struct {
ZipFiles []string `json:"zip_files,omitempty"`
FolderPath string `json:"folder_path,omitempty"`
Title string `json:"title,omitempty"`
URL string `json:"url,omitempty"`
URLs []string `json:"urls,omitempty"`
Date string `json:"date,omitempty"`
Details string `json:"details,omitempty"`
Rating int `json:"rating,omitempty"`
@@ -32,6 +32,9 @@ type Gallery struct {
Tags []string `json:"tags,omitempty"`
CreatedAt json.JSONTime `json:"created_at,omitempty"`
UpdatedAt json.JSONTime `json:"updated_at,omitempty"`
// deprecated - for import only
URL string `json:"url,omitempty"`
}
func (s Gallery) Filename(basename string, hash string) string {

View File

@@ -533,6 +533,29 @@ func (_m *GalleryReaderWriter) GetTagIDs(ctx context.Context, relatedID int) ([]
return r0, r1
}
// GetURLs provides a mock function with given fields: ctx, relatedID
func (_m *GalleryReaderWriter) GetURLs(ctx context.Context, relatedID int) ([]string, error) {
ret := _m.Called(ctx, relatedID)
var r0 []string
if rf, ok := ret.Get(0).(func(context.Context, int) []string); ok {
r0 = rf(ctx, relatedID)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]string)
}
}
var r1 error
if rf, ok := ret.Get(1).(func(context.Context, int) error); ok {
r1 = rf(ctx, relatedID)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// Query provides a mock function with given fields: ctx, galleryFilter, findFilter
func (_m *GalleryReaderWriter) Query(ctx context.Context, galleryFilter *models.GalleryFilterType, findFilter *models.FindFilterType) ([]*models.Gallery, int, error) {
ret := _m.Called(ctx, galleryFilter, findFilter)

View File

@@ -11,7 +11,6 @@ type Gallery struct {
ID int `json:"id"`
Title string `json:"title"`
URL string `json:"url"`
Date *Date `json:"date"`
Details string `json:"details"`
// Rating expressed in 1-100 scale
@@ -31,9 +30,10 @@ type Gallery struct {
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
SceneIDs RelatedIDs `json:"scene_ids"`
TagIDs RelatedIDs `json:"tag_ids"`
PerformerIDs RelatedIDs `json:"performer_ids"`
URLs RelatedStrings `json:"urls"`
SceneIDs RelatedIDs `json:"scene_ids"`
TagIDs RelatedIDs `json:"tag_ids"`
PerformerIDs RelatedIDs `json:"performer_ids"`
}
func NewGallery() Gallery {
@@ -51,7 +51,7 @@ type GalleryPartial struct {
// Checksum OptionalString
// Zip OptionalBool
Title OptionalString
URL OptionalString
URLs *UpdateStrings
Date OptionalDate
Details OptionalString
// Rating expressed in 1-100 scale
@@ -81,6 +81,12 @@ func (g *Gallery) IsUserCreated() bool {
return g.PrimaryFileID == nil && g.FolderID == nil
}
func (g *Gallery) LoadURLs(ctx context.Context, l URLLoader) error {
return g.URLs.load(func() ([]string, error) {
return l.GetURLs(ctx, g.ID)
})
}
func (g *Gallery) LoadFiles(ctx context.Context, l FileLoader) error {
return g.Files.load(func() ([]File, error) {
return l.GetFiles(ctx, g.ID)

View File

@@ -63,6 +63,7 @@ type GalleryReader interface {
GalleryQueryer
GalleryCounter
URLLoader
FileIDLoader
ImageIDLoader
SceneIDLoader

View File

@@ -5,18 +5,24 @@ import "github.com/stashapp/stash/pkg/models"
type ScrapedGallery struct {
Title *string `json:"title"`
Details *string `json:"details"`
URL *string `json:"url"`
URLs []string `json:"urls"`
Date *string `json:"date"`
Studio *models.ScrapedStudio `json:"studio"`
Tags []*models.ScrapedTag `json:"tags"`
Performers []*models.ScrapedPerformer `json:"performers"`
// deprecated
URL *string `json:"url"`
}
func (ScrapedGallery) IsScrapedContent() {}
type ScrapedGalleryInput struct {
Title *string `json:"title"`
Details *string `json:"details"`
URL *string `json:"url"`
Date *string `json:"date"`
Title *string `json:"title"`
Details *string `json:"details"`
URLs []string `json:"urls"`
Date *string `json:"date"`
// deprecated
URL *string `json:"url"`
}

View File

@@ -66,8 +66,8 @@ func queryURLParametersFromGallery(gallery *models.Gallery) queryURLParameters {
ret["title"] = gallery.Title
}
if gallery.URL != "" {
ret["url"] = gallery.URL
if len(gallery.URLs.List()) > 0 {
ret["url"] = gallery.URLs.List()[0]
}
return ret

View File

@@ -354,11 +354,18 @@ func galleryToUpdateInput(gallery *models.Gallery) models.GalleryUpdateInput {
// fallback to file basename if title is empty
title := gallery.GetTitle()
var url *string
urls := gallery.URLs.List()
if len(urls) > 0 {
url = &urls[0]
}
return models.GalleryUpdateInput{
ID: strconv.Itoa(gallery.ID),
Title: &title,
Details: &gallery.Details,
URL: &gallery.URL,
URL: url,
Urls: urls,
Date: dateToStringPtr(gallery.Date),
}
}

View File

@@ -33,7 +33,7 @@ const (
dbConnTimeout = 30
)
var appSchemaVersion uint = 50
var appSchemaVersion uint = 51
//go:embed migrations/*.sql
var migrationsBox embed.FS

View File

@@ -26,12 +26,13 @@ const (
galleriesImagesTable = "galleries_images"
galleriesScenesTable = "scenes_galleries"
galleryIDColumn = "gallery_id"
galleriesURLsTable = "gallery_urls"
galleriesURLColumn = "url"
)
type galleryRow struct {
ID int `db:"id" goqu:"skipinsert"`
Title zero.String `db:"title"`
URL zero.String `db:"url"`
Date NullDate `db:"date"`
Details zero.String `db:"details"`
// expressed as 1-100
@@ -46,7 +47,6 @@ type galleryRow struct {
func (r *galleryRow) fromGallery(o models.Gallery) {
r.ID = o.ID
r.Title = zero.StringFrom(o.Title)
r.URL = zero.StringFrom(o.URL)
r.Date = NullDateFromDatePtr(o.Date)
r.Details = zero.StringFrom(o.Details)
r.Rating = intFromPtr(o.Rating)
@@ -70,7 +70,6 @@ func (r *galleryQueryRow) resolve() *models.Gallery {
ret := &models.Gallery{
ID: r.ID,
Title: r.Title.String,
URL: r.URL.String,
Date: r.Date.DatePtr(),
Details: r.Details.String,
Rating: nullIntPtr(r.Rating),
@@ -97,7 +96,6 @@ type galleryRowRecord struct {
func (r *galleryRowRecord) fromPartial(o models.GalleryPartial) {
r.setNullString("title", o.Title)
r.setNullString("url", o.URL)
r.setNullDate("date", o.Date)
r.setNullString("details", o.Details)
r.setNullInt("rating", o.Rating)
@@ -178,6 +176,12 @@ func (qb *GalleryStore) Create(ctx context.Context, newObject *models.Gallery, f
}
}
if newObject.URLs.Loaded() {
const startPos = 0
if err := galleriesURLsTableMgr.insertJoins(ctx, id, startPos, newObject.URLs.List()); err != nil {
return err
}
}
if newObject.PerformerIDs.Loaded() {
if err := galleriesPerformersTableMgr.insertJoins(ctx, id, newObject.PerformerIDs.List()); err != nil {
return err
@@ -212,6 +216,11 @@ func (qb *GalleryStore) Update(ctx context.Context, updatedObject *models.Galler
return err
}
if updatedObject.URLs.Loaded() {
if err := galleriesURLsTableMgr.replaceJoins(ctx, updatedObject.ID, updatedObject.URLs.List()); err != nil {
return err
}
}
if updatedObject.PerformerIDs.Loaded() {
if err := galleriesPerformersTableMgr.replaceJoins(ctx, updatedObject.ID, updatedObject.PerformerIDs.List()); err != nil {
return err
@@ -257,6 +266,11 @@ func (qb *GalleryStore) UpdatePartial(ctx context.Context, id int, partial model
}
}
if partial.URLs != nil {
if err := galleriesURLsTableMgr.modifyJoins(ctx, id, partial.URLs.Values, partial.URLs.Mode); err != nil {
return nil, err
}
}
if partial.PerformerIDs != nil {
if err := galleriesPerformersTableMgr.modifyJoins(ctx, id, partial.PerformerIDs.IDs, partial.PerformerIDs.Mode); err != nil {
return nil, err
@@ -669,7 +683,7 @@ func (qb *GalleryStore) makeFilter(ctx context.Context, galleryFilter *models.Ga
query.handleCriterion(ctx, intCriterionHandler(galleryFilter.Rating100, "galleries.rating", nil))
// legacy rating handler
query.handleCriterion(ctx, rating5CriterionHandler(galleryFilter.Rating, "galleries.rating", nil))
query.handleCriterion(ctx, stringCriterionHandler(galleryFilter.URL, "galleries.url"))
query.handleCriterion(ctx, galleryURLsCriterionHandler(galleryFilter.URL))
query.handleCriterion(ctx, boolCriterionHandler(galleryFilter.Organized, "galleries.organized", nil))
query.handleCriterion(ctx, galleryIsMissingCriterionHandler(qb, galleryFilter.IsMissing))
query.handleCriterion(ctx, galleryTagsCriterionHandler(qb, galleryFilter.Tags))
@@ -793,6 +807,18 @@ func (qb *GalleryStore) QueryCount(ctx context.Context, galleryFilter *models.Ga
return query.executeCount(ctx)
}
func galleryURLsCriterionHandler(url *models.StringCriterionInput) criterionHandlerFunc {
h := stringListCriterionHandlerBuilder{
joinTable: galleriesURLsTable,
stringColumn: galleriesURLColumn,
addJoinTable: func(f *filterBuilder) {
galleriesURLsTableMgr.join(f, "", "galleries.id")
},
}
return h.handler(url)
}
func (qb *GalleryStore) galleryPathCriterionHandler(c *models.StringCriterionInput) criterionHandlerFunc {
return func(ctx context.Context, f *filterBuilder) {
if c != nil {
@@ -874,6 +900,9 @@ func galleryIsMissingCriterionHandler(qb *GalleryStore, isMissing *string) crite
return func(ctx context.Context, f *filterBuilder) {
if isMissing != nil && *isMissing != "" {
switch *isMissing {
case "url":
galleriesURLsTableMgr.join(f, "", "galleries.id")
f.addWhere("gallery_urls.url IS NULL")
case "scenes":
f.addLeftJoin("scenes_galleries", "scenes_join", "scenes_join.gallery_id = galleries.id")
f.addWhere("scenes_join.gallery_id IS NULL")
@@ -1107,6 +1136,10 @@ func (qb *GalleryStore) setGallerySort(query *queryBuilder, findFilter *models.F
query.sortAndPagination += ", COALESCE(galleries.title, galleries.id) COLLATE NATURAL_CI ASC"
}
func (qb *GalleryStore) GetURLs(ctx context.Context, galleryID int) ([]string, error) {
return galleriesURLsTableMgr.get(ctx, galleryID)
}
func (qb *GalleryStore) filesRepository() *filesRepository {
return &filesRepository{
repository: repository{

View File

@@ -17,6 +17,11 @@ import (
var invalidID = -1
func loadGalleryRelationships(ctx context.Context, expected models.Gallery, actual *models.Gallery) error {
if expected.URLs.Loaded() {
if err := actual.LoadURLs(ctx, db.Gallery); err != nil {
return err
}
}
if expected.SceneIDs.Loaded() {
if err := actual.LoadSceneIDs(ctx, db.Gallery); err != nil {
return err
@@ -72,7 +77,7 @@ func Test_galleryQueryBuilder_Create(t *testing.T) {
"full",
models.Gallery{
Title: title,
URL: url,
URLs: models.NewRelatedStrings([]string{url}),
Date: &date,
Details: details,
Rating: &rating,
@@ -90,7 +95,7 @@ func Test_galleryQueryBuilder_Create(t *testing.T) {
"with file",
models.Gallery{
Title: title,
URL: url,
URLs: models.NewRelatedStrings([]string{url}),
Date: &date,
Details: details,
Rating: &rating,
@@ -222,7 +227,7 @@ func Test_galleryQueryBuilder_Update(t *testing.T) {
&models.Gallery{
ID: galleryIDs[galleryIdxWithScene],
Title: title,
URL: url,
URLs: models.NewRelatedStrings([]string{url}),
Date: &date,
Details: details,
Rating: &rating,
@@ -243,6 +248,7 @@ func Test_galleryQueryBuilder_Update(t *testing.T) {
"clear nullables",
&models.Gallery{
ID: galleryIDs[galleryIdxWithImage],
URLs: models.NewRelatedStrings([]string{}),
SceneIDs: models.NewRelatedIDs([]int{}),
TagIDs: models.NewRelatedIDs([]int{}),
PerformerIDs: models.NewRelatedIDs([]int{}),
@@ -384,7 +390,7 @@ func clearGalleryPartial() models.GalleryPartial {
return models.GalleryPartial{
Title: models.OptionalString{Set: true, Null: true},
Details: models.OptionalString{Set: true, Null: true},
URL: models.OptionalString{Set: true, Null: true},
URLs: &models.UpdateStrings{Mode: models.RelationshipUpdateModeSet},
Date: models.OptionalDate{Set: true, Null: true},
Rating: models.OptionalInt{Set: true, Null: true},
StudioID: models.OptionalInt{Set: true, Null: true},
@@ -416,9 +422,12 @@ func Test_galleryQueryBuilder_UpdatePartial(t *testing.T) {
"full",
galleryIDs[galleryIdxWithImage],
models.GalleryPartial{
Title: models.NewOptionalString(title),
Details: models.NewOptionalString(details),
URL: models.NewOptionalString(url),
Title: models.NewOptionalString(title),
Details: models.NewOptionalString(details),
URLs: &models.UpdateStrings{
Values: []string{url},
Mode: models.RelationshipUpdateModeSet,
},
Date: models.NewOptionalDate(date),
Rating: models.NewOptionalInt(rating),
Organized: models.NewOptionalBool(true),
@@ -443,7 +452,7 @@ func Test_galleryQueryBuilder_UpdatePartial(t *testing.T) {
ID: galleryIDs[galleryIdxWithImage],
Title: title,
Details: details,
URL: url,
URLs: models.NewRelatedStrings([]string{url}),
Date: &date,
Rating: &rating,
Organized: true,
@@ -1653,7 +1662,13 @@ func TestGalleryQueryURL(t *testing.T) {
verifyFn := func(g *models.Gallery) {
t.Helper()
verifyString(t, g.URL, urlCriterion)
urls := g.URLs.List()
var url string
if len(urls) > 0 {
url = urls[0]
}
verifyString(t, url, urlCriterion)
}
verifyGalleryQuery(t, filter, verifyFn)
@@ -1683,6 +1698,12 @@ func verifyGalleryQuery(t *testing.T, filter models.GalleryFilterType, verifyFn
galleries := queryGallery(ctx, t, sqb, &filter, nil)
for _, g := range galleries {
if err := g.LoadURLs(ctx, sqb); err != nil {
t.Errorf("Error loading gallery URLs: %v", err)
}
}
// assume it should find at least one
assert.Greater(t, len(galleries), 0)

View File

@@ -0,0 +1,76 @@
PRAGMA foreign_keys=OFF;
CREATE TABLE `gallery_urls` (
`gallery_id` integer NOT NULL,
`position` integer NOT NULL,
`url` varchar(255) NOT NULL,
foreign key(`gallery_id`) references `galleries`(`id`) on delete CASCADE,
PRIMARY KEY(`gallery_id`, `position`, `url`)
);
CREATE INDEX `gallery_urls_url` on `gallery_urls` (`url`);
-- drop url
CREATE TABLE `galleries_new` (
`id` integer not null primary key autoincrement,
`folder_id` integer,
`title` varchar(255),
`date` date,
`details` text,
`studio_id` integer,
`rating` tinyint,
`organized` boolean not null default '0',
`created_at` datetime not null,
`updated_at` datetime not null,
foreign key(`studio_id`) references `studios`(`id`) on delete SET NULL,
foreign key(`folder_id`) references `folders`(`id`) on delete SET NULL
);
INSERT INTO `galleries_new`
(
`id`,
`folder_id`,
`title`,
`date`,
`details`,
`studio_id`,
`rating`,
`organized`,
`created_at`,
`updated_at`
)
SELECT
`id`,
`folder_id`,
`title`,
`date`,
`details`,
`studio_id`,
`rating`,
`organized`,
`created_at`,
`updated_at`
FROM `galleries`;
INSERT INTO `gallery_urls`
(
`gallery_id`,
`position`,
`url`
)
SELECT
`id`,
'0',
`url`
FROM `galleries`
WHERE `galleries`.`url` IS NOT NULL AND `galleries`.`url` != '';
DROP INDEX `index_galleries_on_studio_id`;
DROP INDEX `index_galleries_on_folder_id_unique`;
DROP TABLE `galleries`;
ALTER TABLE `galleries_new` rename to `galleries`;
CREATE INDEX `index_galleries_on_studio_id` on `galleries` (`studio_id`);
CREATE UNIQUE INDEX `index_galleries_on_folder_id_unique` on `galleries` (`folder_id`);
PRAGMA foreign_keys=ON;

View File

@@ -1291,6 +1291,9 @@ func sceneIsMissingCriterionHandler(qb *SceneStore, isMissing *string) criterion
return func(ctx context.Context, f *filterBuilder) {
if isMissing != nil && *isMissing != "" {
switch *isMissing {
case "url":
scenesURLsTableMgr.join(f, "", "scenes.id")
f.addWhere("scene_urls.url IS NULL")
case "galleries":
qb.galleriesRepository().join(f, "galleries_join", "scenes.id")
f.addWhere("galleries_join.scene_id IS NULL")

View File

@@ -1213,7 +1213,16 @@ func getGalleryNullStringValue(index int, field string) sql.NullString {
}
func getGalleryNullStringPtr(index int, field string) *string {
return getStringPtr(getPrefixedStringValue("gallery", index, field))
return getStringPtrFromNullString(getPrefixedNullStringValue("gallery", index, field))
}
func getGalleryEmptyString(index int, field string) string {
v := getGalleryNullStringPtr(index, field)
if v == nil {
return ""
}
return *v
}
func getGalleryBasename(index int) string {
@@ -1245,8 +1254,10 @@ func makeGallery(i int, includeScenes bool) *models.Gallery {
tids := indexesToIDs(tagIDs, galleryTags[i])
ret := &models.Gallery{
Title: getGalleryStringValue(i, titleField),
URL: getGalleryNullStringValue(i, urlField).String,
Title: getGalleryStringValue(i, titleField),
URLs: models.NewRelatedStrings([]string{
getGalleryEmptyString(i, urlField),
}),
Rating: getIntPtr(getRating(i)),
Date: getObjectDate(i),
StudioID: studioID,

View File

@@ -19,6 +19,7 @@ var (
galleriesTagsJoinTable = goqu.T(galleriesTagsTable)
performersGalleriesJoinTable = goqu.T(performersGalleriesTable)
galleriesScenesJoinTable = goqu.T(galleriesScenesTable)
galleriesURLsJoinTable = goqu.T(galleriesURLsTable)
scenesFilesJoinTable = goqu.T(scenesFilesTable)
scenesTagsJoinTable = goqu.T(scenesTagsTable)
@@ -122,6 +123,14 @@ var (
table: goqu.T(galleriesChaptersTable),
idColumn: goqu.T(galleriesChaptersTable).Col(idColumn),
}
galleriesURLsTableMgr = &orderedValueTable[string]{
table: table{
table: galleriesURLsJoinTable,
idColumn: galleriesURLsJoinTable.Col(galleryIDColumn),
},
valueColumn: galleriesURLsJoinTable.Col(galleriesURLColumn),
}
)
var (