Multiple image URLs (#4000)

* Backend changes - ported from scene impl
* Front end changes
* Refactor URL mutation code
This commit is contained in:
WithoutPants
2023-09-12 13:31:53 +10:00
committed by GitHub
parent 9f4d0af886
commit a25286bdcb
29 changed files with 457 additions and 173 deletions

View File

@@ -14,7 +14,7 @@ import (
func ToBasicJSON(image *models.Image) *jsonschema.Image {
newImageJSON := jsonschema.Image{
Title: image.Title,
URL: image.URL,
URLs: image.URLs.List(),
CreatedAt: json.JSONTime{Time: image.CreatedAt},
UpdatedAt: json.JSONTime{Time: image.UpdatedAt},
}
@@ -37,19 +37,6 @@ func ToBasicJSON(image *models.Image) *jsonschema.Image {
return &newImageJSON
}
// func getImageFileJSON(image *models.Image) *jsonschema.ImageFile {
// ret := &jsonschema.ImageFile{}
// f := image.PrimaryFile()
// ret.ModTime = json.JSONTime{Time: f.ModTime}
// ret.Size = f.Size
// ret.Width = f.Width
// ret.Height = f.Height
// return ret
// }
// GetStudioName returns the name of the provided image's studio. It returns an
// empty string if there is no studio assigned to the image.
func GetStudioName(ctx context.Context, reader models.StudioGetter, image *models.Image) (string, error) {

View File

@@ -53,7 +53,7 @@ func createFullImage(id int) models.Image {
OCounter: ocounter,
Rating: &rating,
Date: &dateObj,
URL: url,
URLs: models.NewRelatedStrings([]string{url}),
Organized: organized,
CreatedAt: createTime,
UpdatedAt: updateTime,
@@ -66,7 +66,7 @@ func createFullJSONImage() *jsonschema.Image {
OCounter: ocounter,
Rating: rating,
Date: date,
URL: url,
URLs: []string{url},
Organized: organized,
Files: []string{path},
CreatedAt: json.JSONTime{

View File

@@ -62,8 +62,6 @@ func (i *Importer) PreImport(ctx context.Context) error {
func (i *Importer) imageJSONToImage(imageJSON jsonschema.Image) models.Image {
newImage := models.Image{
// Checksum: imageJSON.Checksum,
// Path: i.Path,
PerformerIDs: models.NewRelatedIDs([]int{}),
TagIDs: models.NewRelatedIDs([]int{}),
GalleryIDs: models.NewRelatedIDs([]int{}),
@@ -81,9 +79,12 @@ func (i *Importer) imageJSONToImage(imageJSON jsonschema.Image) models.Image {
if imageJSON.Rating != 0 {
newImage.Rating = &imageJSON.Rating
}
if imageJSON.URL != "" {
newImage.URL = imageJSON.URL
if len(imageJSON.URLs) > 0 {
newImage.URLs = models.NewRelatedStrings(imageJSON.URLs)
} else if imageJSON.URL != "" {
newImage.URLs = models.NewRelatedStrings([]string{imageJSON.URL})
}
if imageJSON.Date != "" {
d, err := models.ParseDate(imageJSON.Date)
if err == nil {

View File

@@ -10,10 +10,14 @@ import (
)
type Image struct {
Title string `json:"title,omitempty"`
Studio string `json:"studio,omitempty"`
Rating int `json:"rating,omitempty"`
URL string `json:"url,omitempty"`
Title string `json:"title,omitempty"`
Studio string `json:"studio,omitempty"`
Rating int `json:"rating,omitempty"`
// deprecated - for import only
URL string `json:"url,omitempty"`
URLs []string `json:"urls,omitempty"`
Date string `json:"date,omitempty"`
Organized bool `json:"organized,omitempty"`
OCounter int `json:"o_counter,omitempty"`

View File

@@ -42,8 +42,10 @@ type Scene struct {
Title string `json:"title,omitempty"`
Code string `json:"code,omitempty"`
Studio string `json:"studio,omitempty"`
// deprecated - for import only
URL string `json:"url,omitempty"`
URL string `json:"url,omitempty"`
URLs []string `json:"urls,omitempty"`
Date string `json:"date,omitempty"`
Rating int `json:"rating,omitempty"`

View File

@@ -462,6 +462,29 @@ func (_m *ImageReaderWriter) GetTagIDs(ctx context.Context, relatedID int) ([]in
return r0, r1
}
// GetURLs provides a mock function with given fields: ctx, relatedID
func (_m *ImageReaderWriter) GetURLs(ctx context.Context, relatedID int) ([]string, error) {
ret := _m.Called(ctx, relatedID)
var r0 []string
if rf, ok := ret.Get(0).(func(context.Context, int) []string); ok {
r0 = rf(ctx, relatedID)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]string)
}
}
var r1 error
if rf, ok := ret.Get(1).(func(context.Context, int) error); ok {
r1 = rf(ctx, relatedID)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// IncrementOCounter provides a mock function with given fields: ctx, id
func (_m *ImageReaderWriter) IncrementOCounter(ctx context.Context, id int) (int, error) {
ret := _m.Called(ctx, id)

View File

@@ -13,12 +13,12 @@ type Image struct {
Title string `json:"title"`
// Rating expressed in 1-100 scale
Rating *int `json:"rating"`
Organized bool `json:"organized"`
OCounter int `json:"o_counter"`
StudioID *int `json:"studio_id"`
URL string `json:"url"`
Date *Date `json:"date"`
Rating *int `json:"rating"`
Organized bool `json:"organized"`
OCounter int `json:"o_counter"`
StudioID *int `json:"studio_id"`
URLs RelatedStrings `json:"urls"`
Date *Date `json:"date"`
// transient - not persisted
Files RelatedFiles
@@ -48,7 +48,7 @@ type ImagePartial struct {
Title OptionalString
// Rating expressed in 1-100 scale
Rating OptionalInt
URL OptionalString
URLs *UpdateStrings
Date OptionalDate
Organized OptionalBool
OCounter OptionalInt
@@ -69,6 +69,12 @@ func NewImagePartial() ImagePartial {
}
}
func (i *Image) LoadURLs(ctx context.Context, l URLLoader) error {
return i.URLs.load(func() ([]string, error) {
return l.GetURLs(ctx, i.ID)
})
}
func (i *Image) LoadFiles(ctx context.Context, l FileLoader) error {
return i.Files.load(func() ([]File, error) {
return l.GetFiles(ctx, i.ID)

View File

@@ -63,6 +63,7 @@ type ImageReader interface {
ImageQueryer
ImageCounter
URLLoader
FileIDLoader
GalleryIDLoader
PerformerIDLoader

View File

@@ -368,7 +368,6 @@ func (db *Anonymiser) anonymiseImages(ctx context.Context) error {
query := dialect.From(table).Select(
table.Col(idColumn),
table.Col("title"),
table.Col("url"),
).Where(table.Col(idColumn).Gt(lastID)).Limit(1000)
gotSome = false
@@ -378,20 +377,17 @@ func (db *Anonymiser) anonymiseImages(ctx context.Context) error {
var (
id int
title sql.NullString
url sql.NullString
)
if err := rows.Scan(
&id,
&title,
&url,
); err != nil {
return err
}
set := goqu.Record{}
db.obfuscateNullString(set, "title", title)
db.obfuscateNullString(set, "url", url)
if len(set) > 0 {
stmt := dialect.Update(table).Set(set).Where(table.Col(idColumn).Eq(id))
@@ -416,6 +412,10 @@ func (db *Anonymiser) anonymiseImages(ctx context.Context) error {
}
}
if err := db.anonymiseURLs(ctx, goqu.T(imagesURLsTable), "image_id"); err != nil {
return err
}
return nil
}

View File

@@ -33,7 +33,7 @@ const (
dbConnTimeout = 30
)
var appSchemaVersion uint = 49
var appSchemaVersion uint = 50
//go:embed migrations/*.sql
var migrationsBox embed.FS

View File

@@ -24,27 +24,27 @@ const (
performersImagesTable = "performers_images"
imagesTagsTable = "images_tags"
imagesFilesTable = "images_files"
imagesURLsTable = "image_urls"
imageURLColumn = "url"
)
type imageRow struct {
ID int `db:"id" goqu:"skipinsert"`
Title zero.String `db:"title"`
// expressed as 1-100
Rating null.Int `db:"rating"`
URL zero.String `db:"url"`
Date NullDate `db:"date"`
Organized bool `db:"organized"`
OCounter int `db:"o_counter"`
StudioID null.Int `db:"studio_id,omitempty"`
CreatedAt Timestamp `db:"created_at"`
UpdatedAt Timestamp `db:"updated_at"`
Rating null.Int `db:"rating"`
Date NullDate `db:"date"`
Organized bool `db:"organized"`
OCounter int `db:"o_counter"`
StudioID null.Int `db:"studio_id,omitempty"`
CreatedAt Timestamp `db:"created_at"`
UpdatedAt Timestamp `db:"updated_at"`
}
func (r *imageRow) fromImage(i models.Image) {
r.ID = i.ID
r.Title = zero.StringFrom(i.Title)
r.Rating = intFromPtr(i.Rating)
r.URL = zero.StringFrom(i.URL)
r.Date = NullDateFromDatePtr(i.Date)
r.Organized = i.Organized
r.OCounter = i.OCounter
@@ -66,7 +66,6 @@ func (r *imageQueryRow) resolve() *models.Image {
ID: r.ID,
Title: r.Title.String,
Rating: nullIntPtr(r.Rating),
URL: r.URL.String,
Date: r.Date.DatePtr(),
Organized: r.Organized,
OCounter: r.OCounter,
@@ -93,7 +92,6 @@ type imageRowRecord struct {
func (r *imageRowRecord) fromPartial(i models.ImagePartial) {
r.setNullString("title", i.Title)
r.setNullInt("rating", i.Rating)
r.setNullString("url", i.URL)
r.setNullDate("date", i.Date)
r.setBool("organized", i.Organized)
r.setInt("o_counter", i.OCounter)
@@ -176,6 +174,13 @@ func (qb *ImageStore) Create(ctx context.Context, newObject *models.Image, fileI
}
}
if newObject.URLs.Loaded() {
const startPos = 0
if err := imagesURLsTableMgr.insertJoins(ctx, id, startPos, newObject.URLs.List()); err != nil {
return err
}
}
if newObject.PerformerIDs.Loaded() {
if err := imagesPerformersTableMgr.insertJoins(ctx, id, newObject.PerformerIDs.List()); err != nil {
return err
@@ -223,6 +228,12 @@ func (qb *ImageStore) UpdatePartial(ctx context.Context, id int, partial models.
return nil, err
}
}
if partial.URLs != nil {
if err := imagesURLsTableMgr.modifyJoins(ctx, id, partial.URLs.Values, partial.URLs.Mode); err != nil {
return nil, err
}
}
if partial.PerformerIDs != nil {
if err := imagesPerformersTableMgr.modifyJoins(ctx, id, partial.PerformerIDs.IDs, partial.PerformerIDs.Mode); err != nil {
return nil, err
@@ -251,6 +262,12 @@ func (qb *ImageStore) Update(ctx context.Context, updatedObject *models.Image) e
return err
}
if updatedObject.URLs.Loaded() {
if err := imagesURLsTableMgr.replaceJoins(ctx, updatedObject.ID, updatedObject.URLs.List()); err != nil {
return err
}
}
if updatedObject.PerformerIDs.Loaded() {
if err := imagesPerformersTableMgr.replaceJoins(ctx, updatedObject.ID, updatedObject.PerformerIDs.List()); err != nil {
return err
@@ -664,7 +681,7 @@ func (qb *ImageStore) makeFilter(ctx context.Context, imageFilter *models.ImageF
query.handleCriterion(ctx, intCriterionHandler(imageFilter.OCounter, "images.o_counter", nil))
query.handleCriterion(ctx, boolCriterionHandler(imageFilter.Organized, "images.organized", nil))
query.handleCriterion(ctx, dateCriterionHandler(imageFilter.Date, "images.date"))
query.handleCriterion(ctx, stringCriterionHandler(imageFilter.URL, "images.url"))
query.handleCriterion(ctx, imageURLsCriterionHandler(imageFilter.URL))
query.handleCriterion(ctx, resolutionCriterionHandler(imageFilter.Resolution, "image_files.height", "image_files.width", qb.addImageFilesTable))
query.handleCriterion(ctx, imageIsMissingCriterionHandler(qb, imageFilter.IsMissing))
@@ -855,6 +872,18 @@ func imageIsMissingCriterionHandler(qb *ImageStore, isMissing *string) criterion
}
}
func imageURLsCriterionHandler(url *models.StringCriterionInput) criterionHandlerFunc {
h := stringListCriterionHandlerBuilder{
joinTable: imagesURLsTable,
stringColumn: imageURLColumn,
addJoinTable: func(f *filterBuilder) {
imagesURLsTableMgr.join(f, "", "images.id")
},
}
return h.handler(url)
}
func (qb *ImageStore) getMultiCriterionHandlerBuilder(foreignTable, joinTable, foreignFK string, addJoinsFunc func(f *filterBuilder)) multiCriterionHandlerBuilder {
return multiCriterionHandlerBuilder{
primaryTable: imageTable,
@@ -1097,3 +1126,7 @@ func (qb *ImageStore) UpdateTags(ctx context.Context, imageID int, tagIDs []int)
// Delete the existing joins and then create new ones
return qb.tagsRepository().replace(ctx, imageID, tagIDs)
}
func (qb *ImageStore) GetURLs(ctx context.Context, imageID int) ([]string, error) {
return imagesURLsTableMgr.get(ctx, imageID)
}

View File

@@ -15,6 +15,11 @@ import (
)
func loadImageRelationships(ctx context.Context, expected models.Image, actual *models.Image) error {
if expected.URLs.Loaded() {
if err := actual.LoadURLs(ctx, db.Image); err != nil {
return err
}
}
if expected.GalleryIDs.Loaded() {
if err := actual.LoadGalleryIDs(ctx, db.Image); err != nil {
return err
@@ -74,7 +79,7 @@ func Test_imageQueryBuilder_Create(t *testing.T) {
Title: title,
Rating: &rating,
Date: &date,
URL: url,
URLs: models.NewRelatedStrings([]string{url}),
Organized: true,
OCounter: ocounter,
StudioID: &studioIDs[studioIdxWithImage],
@@ -92,7 +97,7 @@ func Test_imageQueryBuilder_Create(t *testing.T) {
Title: title,
Rating: &rating,
Date: &date,
URL: url,
URLs: models.NewRelatedStrings([]string{url}),
Organized: true,
OCounter: ocounter,
StudioID: &studioIDs[studioIdxWithImage],
@@ -229,7 +234,7 @@ func Test_imageQueryBuilder_Update(t *testing.T) {
ID: imageIDs[imageIdxWithGallery],
Title: title,
Rating: &rating,
URL: url,
URLs: models.NewRelatedStrings([]string{url}),
Date: &date,
Organized: true,
OCounter: ocounter,
@@ -378,7 +383,7 @@ func clearImagePartial() models.ImagePartial {
return models.ImagePartial{
Title: models.OptionalString{Set: true, Null: true},
Rating: models.OptionalInt{Set: true, Null: true},
URL: models.OptionalString{Set: true, Null: true},
URLs: &models.UpdateStrings{Mode: models.RelationshipUpdateModeSet},
Date: models.OptionalDate{Set: true, Null: true},
StudioID: models.OptionalInt{Set: true, Null: true},
GalleryIDs: &models.UpdateIDs{Mode: models.RelationshipUpdateModeSet},
@@ -409,9 +414,12 @@ func Test_imageQueryBuilder_UpdatePartial(t *testing.T) {
"full",
imageIDs[imageIdx1WithGallery],
models.ImagePartial{
Title: models.NewOptionalString(title),
Rating: models.NewOptionalInt(rating),
URL: models.NewOptionalString(url),
Title: models.NewOptionalString(title),
Rating: models.NewOptionalInt(rating),
URLs: &models.UpdateStrings{
Values: []string{url},
Mode: models.RelationshipUpdateModeSet,
},
Date: models.NewOptionalDate(date),
Organized: models.NewOptionalBool(true),
OCounter: models.NewOptionalInt(ocounter),
@@ -435,7 +443,7 @@ func Test_imageQueryBuilder_UpdatePartial(t *testing.T) {
ID: imageIDs[imageIdx1WithGallery],
Title: title,
Rating: &rating,
URL: url,
URLs: models.NewRelatedStrings([]string{url}),
Date: &date,
Organized: true,
OCounter: ocounter,
@@ -1519,6 +1527,67 @@ func imageQueryQ(ctx context.Context, t *testing.T, sqb models.ImageReader, q st
assert.Len(t, images, totalImages)
}
func verifyImageQuery(t *testing.T, filter models.ImageFilterType, verifyFn func(ctx context.Context, s *models.Image)) {
t.Helper()
withTxn(func(ctx context.Context) error {
t.Helper()
sqb := db.Image
images := queryImages(ctx, t, sqb, &filter, nil)
// assume it should find at least one
assert.Greater(t, len(images), 0)
for _, image := range images {
verifyFn(ctx, image)
}
return nil
})
}
func TestImageQueryURL(t *testing.T) {
const imageIdx = 1
imageURL := getImageStringValue(imageIdx, urlField)
urlCriterion := models.StringCriterionInput{
Value: imageURL,
Modifier: models.CriterionModifierEquals,
}
filter := models.ImageFilterType{
URL: &urlCriterion,
}
verifyFn := func(ctx context.Context, o *models.Image) {
t.Helper()
if err := o.LoadURLs(ctx, db.Image); err != nil {
t.Errorf("Error loading scene URLs: %v", err)
}
urls := o.URLs.List()
var url string
if len(urls) > 0 {
url = urls[0]
}
verifyString(t, url, urlCriterion)
}
verifyImageQuery(t, filter, verifyFn)
urlCriterion.Modifier = models.CriterionModifierNotEquals
verifyImageQuery(t, filter, verifyFn)
urlCriterion.Modifier = models.CriterionModifierMatchesRegex
urlCriterion.Value = "image_.*1_URL"
verifyImageQuery(t, filter, verifyFn)
urlCriterion.Modifier = models.CriterionModifierNotMatchesRegex
verifyImageQuery(t, filter, verifyFn)
urlCriterion.Modifier = models.CriterionModifierIsNull
urlCriterion.Value = ""
verifyImageQuery(t, filter, verifyFn)
urlCriterion.Modifier = models.CriterionModifierNotNull
verifyImageQuery(t, filter, verifyFn)
}
func TestImageQueryPath(t *testing.T) {
const imageIdx = 1
imagePath := getFilePath(folderIdxWithImageFiles, getImageBasename(imageIdx))

View File

@@ -0,0 +1,70 @@
PRAGMA foreign_keys=OFF;
CREATE TABLE `image_urls` (
`image_id` integer NOT NULL,
`position` integer NOT NULL,
`url` varchar(255) NOT NULL,
foreign key(`image_id`) references `images`(`id`) on delete CASCADE,
PRIMARY KEY(`image_id`, `position`, `url`)
);
CREATE INDEX `image_urls_url` on `image_urls` (`url`);
-- drop url
CREATE TABLE "images_new" (
`id` integer not null primary key autoincrement,
`title` varchar(255),
`rating` tinyint,
`studio_id` integer,
`o_counter` tinyint not null default 0,
`organized` boolean not null default '0',
`created_at` datetime not null,
`updated_at` datetime not null,
`date` date,
foreign key(`studio_id`) references `studios`(`id`) on delete SET NULL
);
INSERT INTO `images_new`
(
`id`,
`title`,
`rating`,
`studio_id`,
`o_counter`,
`organized`,
`created_at`,
`updated_at`,
`date`
)
SELECT
`id`,
`title`,
`rating`,
`studio_id`,
`o_counter`,
`organized`,
`created_at`,
`updated_at`,
`date`
FROM `images`;
INSERT INTO `image_urls`
(
`image_id`,
`position`,
`url`
)
SELECT
`id`,
'0',
`url`
FROM `images`
WHERE `images`.`url` IS NOT NULL AND `images`.`url` != '';
DROP INDEX `index_images_on_studio_id`;
DROP TABLE `images`;
ALTER TABLE `images_new` rename to `images`;
CREATE INDEX `index_images_on_studio_id` on `images` (`studio_id`);
PRAGMA foreign_keys=ON;

View File

@@ -1113,6 +1113,19 @@ func getImageStringValue(index int, field string) string {
return fmt.Sprintf("image_%04d_%s", index, field)
}
func getImageNullStringPtr(index int, field string) *string {
return getStringPtrFromNullString(getPrefixedNullStringValue("image", index, field))
}
func getImageEmptyString(index int, field string) string {
v := getImageNullStringPtr(index, field)
if v == nil {
return ""
}
return *v
}
func getImageBasename(index int) string {
return getImageStringValue(index, pathField)
}
@@ -1148,10 +1161,12 @@ func makeImage(i int) *models.Image {
tids := indexesToIDs(tagIDs, imageTags[i])
return &models.Image{
Title: title,
Rating: getIntPtr(getRating(i)),
Date: getObjectDate(i),
URL: getImageStringValue(i, urlField),
Title: title,
Rating: getIntPtr(getRating(i)),
Date: getObjectDate(i),
URLs: models.NewRelatedStrings([]string{
getImageEmptyString(i, urlField),
}),
OCounter: getOCounter(i),
StudioID: studioID,
GalleryIDs: models.NewRelatedIDs(gids),

View File

@@ -13,6 +13,7 @@ var (
imagesTagsJoinTable = goqu.T(imagesTagsTable)
performersImagesJoinTable = goqu.T(performersImagesTable)
imagesFilesJoinTable = goqu.T(imagesFilesTable)
imagesURLsJoinTable = goqu.T(imagesURLsTable)
galleriesFilesJoinTable = goqu.T(galleriesFilesTable)
galleriesTagsJoinTable = goqu.T(galleriesTagsTable)
@@ -70,6 +71,14 @@ var (
},
fkColumn: performersImagesJoinTable.Col(performerIDColumn),
}
imagesURLsTableMgr = &orderedValueTable[string]{
table: table{
table: imagesURLsJoinTable,
idColumn: imagesURLsJoinTable.Col(imageIDColumn),
},
valueColumn: imagesURLsJoinTable.Col(imageURLColumn),
}
)
var (