Mirrors/stash
1
0
Fork 0
mirror of https://github.com/stashapp/stash.git synced 2025-12-18 04:44:37 +03:00
Code Issues Packages Projects Releases Wiki Activity

Performer urls (#4958)

Browse Source 
* Populate URLs from legacy fields
* Return nil properly in xpath/json scrapers
* Improve migration logging
This commit is contained in:
WithoutPants
2024-06-18 13:41:05 +10:00
committed by GitHub
parent fda4776d30
commit f26766033e
47 changed files with 992 additions and 379 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
pkg/sqlite/anonymise.go
View File

@@ -495,9 +495,6 @@ func (db *Anonymiser) anonymisePerformers(ctx context.Context) error {
table.Col(idColumn),
table.Col("name"),
table.Col("details"),
table.Col("url"),
table.Col("twitter"),
table.Col("instagram"),
table.Col("tattoos"),
table.Col("piercings"),
).Where(table.Col(idColumn).Gt(lastID)).Limit(1000)
@@ -510,9 +507,6 @@ func (db *Anonymiser) anonymisePerformers(ctx context.Context) error {
id int
name sql.NullString
details sql.NullString
url sql.NullString
twitter sql.NullString
instagram sql.NullString
tattoos sql.NullString
piercings sql.NullString
)
@@ -521,9 +515,6 @@ func (db *Anonymiser) anonymisePerformers(ctx context.Context) error {
&id,
&name,
&details,
&url,
&twitter,
&instagram,
&tattoos,
&piercings,
); err != nil {
@@ -533,9 +524,6 @@ func (db *Anonymiser) anonymisePerformers(ctx context.Context) error {
set := goqu.Record{}
db.obfuscateNullString(set, "name", name)
db.obfuscateNullString(set, "details", details)
db.obfuscateNullString(set, "url", url)
db.obfuscateNullString(set, "twitter", twitter)
db.obfuscateNullString(set, "instagram", instagram)
db.obfuscateNullString(set, "tattoos", tattoos)
db.obfuscateNullString(set, "piercings", piercings)
@@ -566,6 +554,10 @@ func (db *Anonymiser) anonymisePerformers(ctx context.Context) error {
return err
}
if err := db.anonymiseURLs(ctx, goqu.T(performerURLsTable), "performer_id"); err != nil {
return err
}
return nil
}

2
pkg/sqlite/database.go
View File

@@ -30,7 +30,7 @@ const (
dbConnTimeout = 30
)
var appSchemaVersion uint = 61
var appSchemaVersion uint = 62
//go:embed migrations/*.sql
var migrationsBox embed.FS

155
pkg/sqlite/migrations/62_performer_urls.up.sql Normal file
View File

@@ -0,0 +1,155 @@
PRAGMA foreign_keys=OFF;
CREATE TABLE `performer_urls` (
`performer_id` integer NOT NULL,
`position` integer NOT NULL,
`url` varchar(255) NOT NULL,
foreign key(`performer_id`) references `performers`(`id`) on delete CASCADE,
PRIMARY KEY(`performer_id`, `position`, `url`)
);
CREATE INDEX `performers_urls_url` on `performer_urls` (`url`);
-- drop url, twitter and instagram
-- make name not null
CREATE TABLE `performers_new` (
`id` integer not null primary key autoincrement,
`name` varchar(255) not null,
`disambiguation` varchar(255),
`gender` varchar(20),
`birthdate` date,
`ethnicity` varchar(255),
`country` varchar(255),
`eye_color` varchar(255),
`height` int,
`measurements` varchar(255),
`fake_tits` varchar(255),
`career_length` varchar(255),
`tattoos` varchar(255),
`piercings` varchar(255),
`favorite` boolean not null default '0',
`created_at` datetime not null,
`updated_at` datetime not null,
`details` text,
`death_date` date,
`hair_color` varchar(255),
`weight` integer,
`rating` tinyint,
`ignore_auto_tag` boolean not null default '0',
`image_blob` varchar(255) REFERENCES `blobs`(`checksum`),
`penis_length` float,
`circumcised` varchar[10]
);
INSERT INTO `performers_new`
(
`id`,
`name`,
`disambiguation`,
`gender`,
`birthdate`,
`ethnicity`,
`country`,
`eye_color`,
`height`,
`measurements`,
`fake_tits`,
`career_length`,
`tattoos`,
`piercings`,
`favorite`,
`created_at`,
`updated_at`,
`details`,
`death_date`,
`hair_color`,
`weight`,
`rating`,
`ignore_auto_tag`,
`image_blob`,
`penis_length`,
`circumcised`
)
SELECT
`id`,
`name`,
`disambiguation`,
`gender`,
`birthdate`,
`ethnicity`,
`country`,
`eye_color`,
`height`,
`measurements`,
`fake_tits`,
`career_length`,
`tattoos`,
`piercings`,
`favorite`,
`created_at`,
`updated_at`,
`details`,
`death_date`,
`hair_color`,
`weight`,
`rating`,
`ignore_auto_tag`,
`image_blob`,
`penis_length`,
`circumcised`
FROM `performers`;
INSERT INTO `performer_urls`
(
`performer_id`,
`position`,
`url`
)
SELECT
`id`,
'0',
`url`
FROM `performers`
WHERE `performers`.`url` IS NOT NULL AND `performers`.`url` != '';
INSERT INTO `performer_urls`
(
`performer_id`,
`position`,
`url`
)
SELECT
`id`,
(SELECT count(*) FROM `performer_urls` WHERE `performer_id` = `performers`.`id`)+1,
CASE
WHEN `twitter` LIKE 'http%://%' THEN `twitter`
ELSE 'https://www.twitter.com/' || `twitter`
END
FROM `performers`
WHERE `performers`.`twitter` IS NOT NULL AND `performers`.`twitter` != '';
INSERT INTO `performer_urls`
(
`performer_id`,
`position`,
`url`
)
SELECT
`id`,
(SELECT count(*) FROM `performer_urls` WHERE `performer_id` = `performers`.`id`)+1,
CASE
WHEN `instagram` LIKE 'http%://%' THEN `instagram`
ELSE 'https://www.instagram.com/' || `instagram`
END
FROM `performers`
WHERE `performers`.`instagram` IS NOT NULL AND `performers`.`instagram` != '';
DROP INDEX `performers_name_disambiguation_unique`;
DROP INDEX `performers_name_unique`;
DROP TABLE `performers`;
ALTER TABLE `performers_new` rename to `performers`;
CREATE UNIQUE INDEX `performers_name_disambiguation_unique` on `performers` (`name`, `disambiguation`) WHERE `disambiguation` IS NOT NULL;
CREATE UNIQUE INDEX `performers_name_unique` on `performers` (`name`) WHERE `disambiguation` IS NULL;
PRAGMA foreign_keys=ON;

38
pkg/sqlite/performer.go
View File

@@ -23,6 +23,9 @@ const (
performerAliasColumn = "alias"
performersTagsTable = "performers_tags"
performerURLsTable = "performer_urls"
performerURLColumn = "url"
performerImageBlobColumn = "image_blob"
)
@@ -31,9 +34,6 @@ type performerRow struct {
Name null.String `db:"name"` // TODO: make schema non-nullable
Disambigation zero.String `db:"disambiguation"`
Gender zero.String `db:"gender"`
URL zero.String `db:"url"`
Twitter zero.String `db:"twitter"`
Instagram zero.String `db:"instagram"`
Birthdate NullDate `db:"birthdate"`
Ethnicity zero.String `db:"ethnicity"`
Country zero.String `db:"country"`
@@ -68,9 +68,6 @@ func (r *performerRow) fromPerformer(o models.Performer) {
if o.Gender != nil && o.Gender.IsValid() {
r.Gender = zero.StringFrom(o.Gender.String())
}
r.URL = zero.StringFrom(o.URL)
r.Twitter = zero.StringFrom(o.Twitter)
r.Instagram = zero.StringFrom(o.Instagram)
r.Birthdate = NullDateFromDatePtr(o.Birthdate)
r.Ethnicity = zero.StringFrom(o.Ethnicity)
r.Country = zero.StringFrom(o.Country)
@@ -101,9 +98,6 @@ func (r *performerRow) resolve() *models.Performer {
ID: r.ID,
Name: r.Name.String,
Disambiguation: r.Disambigation.String,
URL: r.URL.String,
Twitter: r.Twitter.String,
Instagram: r.Instagram.String,
Birthdate: r.Birthdate.DatePtr(),
Ethnicity: r.Ethnicity.String,
Country: r.Country.String,
@@ -148,9 +142,6 @@ func (r *performerRowRecord) fromPartial(o models.PerformerPartial) {
r.setString("name", o.Name)
r.setNullString("disambiguation", o.Disambiguation)
r.setNullString("gender", o.Gender)
r.setNullString("url", o.URL)
r.setNullString("twitter", o.Twitter)
r.setNullString("instagram", o.Instagram)
r.setNullDate("birthdate", o.Birthdate)
r.setNullString("ethnicity", o.Ethnicity)
r.setNullString("country", o.Country)
@@ -272,6 +263,13 @@ func (qb *PerformerStore) Create(ctx context.Context, newObject *models.Performe
}
}
if newObject.URLs.Loaded() {
const startPos = 0
if err := performersURLsTableMgr.insertJoins(ctx, id, startPos, newObject.URLs.List()); err != nil {
return err
}
}
if newObject.TagIDs.Loaded() {
if err := performersTagsTableMgr.insertJoins(ctx, id, newObject.TagIDs.List()); err != nil {
return err
@@ -315,6 +313,12 @@ func (qb *PerformerStore) UpdatePartial(ctx context.Context, id int, partial mod
}
}
if partial.URLs != nil {
if err := performersURLsTableMgr.modifyJoins(ctx, id, partial.URLs.Values, partial.URLs.Mode); err != nil {
return nil, err
}
}
if partial.TagIDs != nil {
if err := performersTagsTableMgr.modifyJoins(ctx, id, partial.TagIDs.IDs, partial.TagIDs.Mode); err != nil {
return nil, err
@@ -343,6 +347,12 @@ func (qb *PerformerStore) Update(ctx context.Context, updatedObject *models.Perf
}
}
if updatedObject.URLs.Loaded() {
if err := performersURLsTableMgr.replaceJoins(ctx, updatedObject.ID, updatedObject.URLs.List()); err != nil {
return err
}
}
if updatedObject.TagIDs.Loaded() {
if err := performersTagsTableMgr.replaceJoins(ctx, updatedObject.ID, updatedObject.TagIDs.List()); err != nil {
return err
@@ -785,6 +795,10 @@ func (qb *PerformerStore) GetAliases(ctx context.Context, performerID int) ([]st
return performersAliasesTableMgr.get(ctx, performerID)
}
func (qb *PerformerStore) GetURLs(ctx context.Context, performerID int) ([]string, error) {
return performersURLsTableMgr.get(ctx, performerID)
}
func (qb *PerformerStore) GetStashIDs(ctx context.Context, performerID int) ([]models.StashID, error) {
return performersStashIDsTableMgr.get(ctx, performerID)
}

19
pkg/sqlite/performer_filter.go
View File

@@ -134,7 +134,7 @@ func (qb *performerFilterHandler) criterionHandler() criterionHandler {
stringCriterionHandler(filter.Piercings, tableName+".piercings"),
intCriterionHandler(filter.Rating100, tableName+".rating", nil),
stringCriterionHandler(filter.HairColor, tableName+".hair_color"),
stringCriterionHandler(filter.URL, tableName+".url"),
qb.urlsCriterionHandler(filter.URL),
intCriterionHandler(filter.Weight, tableName+".weight", nil),
criterionHandlerFunc(func(ctx context.Context, f *filterBuilder) {
if filter.StashID != nil {
@@ -211,6 +211,9 @@ func (qb *performerFilterHandler) performerIsMissingCriterionHandler(isMissing *
return func(ctx context.Context, f *filterBuilder) {
if isMissing != nil && *isMissing != "" {
switch *isMissing {
case "url":
performersURLsTableMgr.join(f, "", "performers.id")
f.addWhere("performer_urls.url IS NULL")
case "scenes": // Deprecated: use `scene_count == 0` filter instead
f.addLeftJoin(performersScenesTable, "scenes_join", "scenes_join.performer_id = performers.id")
f.addWhere("scenes_join.scene_id IS NULL")
@@ -241,6 +244,20 @@ func (qb *performerFilterHandler) performerAgeFilterCriterionHandler(age *models
}
}
func (qb *performerFilterHandler) urlsCriterionHandler(url *models.StringCriterionInput) criterionHandlerFunc {
h := stringListCriterionHandlerBuilder{
primaryTable: performerTable,
primaryFK: performerIDColumn,
joinTable: performerURLsTable,
stringColumn: performerURLColumn,
addJoinTable: func(f *filterBuilder) {
performersURLsTableMgr.join(f, "", "performers.id")
},
}
return h.handler(url)
}
func (qb *performerFilterHandler) aliasCriterionHandler(alias *models.StringCriterionInput) criterionHandlerFunc {
h := stringListCriterionHandlerBuilder{
primaryTable: performerTable,

72
pkg/sqlite/performer_test.go
View File

@@ -22,6 +22,11 @@ func loadPerformerRelationships(ctx context.Context, expected models.Performer,
return err
}
}
if expected.URLs.Loaded() {
if err := actual.LoadURLs(ctx, db.Performer); err != nil {
return err
}
}
if expected.TagIDs.Loaded() {
if err := actual.LoadTagIDs(ctx, db.Performer); err != nil {
return err
@@ -45,6 +50,7 @@ func Test_PerformerStore_Create(t *testing.T) {
url = "url"
twitter = "twitter"
instagram = "instagram"
urls = []string{url, twitter, instagram}
rating = 3
ethnicity = "ethnicity"
country = "country"
@@ -84,9 +90,7 @@ func Test_PerformerStore_Create(t *testing.T) {
Name: name,
Disambiguation: disambiguation,
Gender: &gender,
URL: url,
Twitter: twitter,
Instagram: instagram,
URLs: models.NewRelatedStrings(urls),
Birthdate: &birthdate,
Ethnicity: ethnicity,
Country: country,
@@ -193,6 +197,7 @@ func Test_PerformerStore_Update(t *testing.T) {
url = "url"
twitter = "twitter"
instagram = "instagram"
urls = []string{url, twitter, instagram}
rating = 3
ethnicity = "ethnicity"
country = "country"
@@ -233,9 +238,7 @@ func Test_PerformerStore_Update(t *testing.T) {
Name: name,
Disambiguation: disambiguation,
Gender: &gender,
URL: url,
Twitter: twitter,
Instagram: instagram,
URLs: models.NewRelatedStrings(urls),
Birthdate: &birthdate,
Ethnicity: ethnicity,
Country: country,
@@ -277,6 +280,7 @@ func Test_PerformerStore_Update(t *testing.T) {
&models.Performer{
ID: performerIDs[performerIdxWithGallery],
Aliases: models.NewRelatedStrings([]string{}),
URLs: models.NewRelatedStrings([]string{}),
TagIDs: models.NewRelatedIDs([]int{}),
StashIDs: models.NewRelatedStashIDs([]models.StashID{}),
},
@@ -341,9 +345,7 @@ func clearPerformerPartial() models.PerformerPartial {
return models.PerformerPartial{
Disambiguation: nullString,
Gender: nullString,
URL: nullString,
Twitter: nullString,
Instagram: nullString,
URLs: &models.UpdateStrings{Mode: models.RelationshipUpdateModeSet},
Birthdate: nullDate,
Ethnicity: nullString,
Country: nullString,
@@ -376,6 +378,7 @@ func Test_PerformerStore_UpdatePartial(t *testing.T) {
url = "url"
twitter = "twitter"
instagram = "instagram"
urls = []string{url, twitter, instagram}
rating = 3
ethnicity = "ethnicity"
country = "country"
@@ -418,21 +421,22 @@ func Test_PerformerStore_UpdatePartial(t *testing.T) {
Name: models.NewOptionalString(name),
Disambiguation: models.NewOptionalString(disambiguation),
Gender: models.NewOptionalString(gender.String()),
URL: models.NewOptionalString(url),
Twitter: models.NewOptionalString(twitter),
Instagram: models.NewOptionalString(instagram),
Birthdate: models.NewOptionalDate(birthdate),
Ethnicity: models.NewOptionalString(ethnicity),
Country: models.NewOptionalString(country),
EyeColor: models.NewOptionalString(eyeColor),
Height: models.NewOptionalInt(height),
Measurements: models.NewOptionalString(measurements),
FakeTits: models.NewOptionalString(fakeTits),
PenisLength: models.NewOptionalFloat64(penisLength),
Circumcised: models.NewOptionalString(circumcised.String()),
CareerLength: models.NewOptionalString(careerLength),
Tattoos: models.NewOptionalString(tattoos),
Piercings: models.NewOptionalString(piercings),
URLs: &models.UpdateStrings{
Values: urls,
Mode: models.RelationshipUpdateModeSet,
},
Birthdate: models.NewOptionalDate(birthdate),
Ethnicity: models.NewOptionalString(ethnicity),
Country: models.NewOptionalString(country),
EyeColor: models.NewOptionalString(eyeColor),
Height: models.NewOptionalInt(height),
Measurements: models.NewOptionalString(measurements),
FakeTits: models.NewOptionalString(fakeTits),
PenisLength: models.NewOptionalFloat64(penisLength),
Circumcised: models.NewOptionalString(circumcised.String()),
CareerLength: models.NewOptionalString(careerLength),
Tattoos: models.NewOptionalString(tattoos),
Piercings: models.NewOptionalString(piercings),
Aliases: &models.UpdateStrings{
Values: aliases,
Mode: models.RelationshipUpdateModeSet,
@@ -469,9 +473,7 @@ func Test_PerformerStore_UpdatePartial(t *testing.T) {
Name: name,
Disambiguation: disambiguation,
Gender: &gender,
URL: url,
Twitter: twitter,
Instagram: instagram,
URLs: models.NewRelatedStrings(urls),
Birthdate: &birthdate,
Ethnicity: ethnicity,
Country: country,
@@ -516,6 +518,7 @@ func Test_PerformerStore_UpdatePartial(t *testing.T) {
ID: performerIDs[performerIdxWithTwoTags],
Name: getPerformerStringValue(performerIdxWithTwoTags, "Name"),
Favorite: getPerformerBoolValue(performerIdxWithTwoTags),
URLs: models.NewRelatedStrings([]string{}),
Aliases: models.NewRelatedStrings([]string{}),
TagIDs: models.NewRelatedIDs([]int{}),
StashIDs: models.NewRelatedStashIDs([]models.StashID{}),
@@ -1290,7 +1293,14 @@ func TestPerformerQueryURL(t *testing.T) {
verifyFn := func(g *models.Performer) {
t.Helper()
verifyString(t, g.URL, urlCriterion)
urls := g.URLs.List()
var url string
if len(urls) > 0 {
url = urls[0]
}
verifyString(t, url, urlCriterion)
}
verifyPerformerQuery(t, filter, verifyFn)
@@ -1318,6 +1328,12 @@ func verifyPerformerQuery(t *testing.T, filter models.PerformerFilterType, verif
t.Helper()
performers := queryPerformers(ctx, t, &filter, nil)
for _, performer := range performers {
if err := performer.LoadURLs(ctx, db.Performer); err != nil {
t.Errorf("Error loading movie relationships: %v", err)
}
}
// assume it should find at least one
assert.Greater(t, len(performers), 0)

33
pkg/sqlite/setup_test.go
View File

@@ -1374,6 +1374,15 @@ func getPerformerNullStringValue(index int, field string) string {
return ret.String
}
func getPerformerEmptyString(index int, field string) string {
v := getPrefixedNullStringValue("performer", index, field)
if !v.Valid {
return ""
}
return v.String
}
func getPerformerBoolValue(index int) bool {
index = index % 2
return index == 1
@@ -1479,17 +1488,19 @@ func createPerformers(ctx context.Context, n int, o int) error {
Name: getPerformerStringValue(index, name),
Disambiguation: getPerformerStringValue(index, "disambiguation"),
Aliases: models.NewRelatedStrings(performerAliases(index)),
URL: getPerformerNullStringValue(i, urlField),
Favorite: getPerformerBoolValue(i),
Birthdate: getPerformerBirthdate(i),
DeathDate: getPerformerDeathDate(i),
Details: getPerformerStringValue(i, "Details"),
Ethnicity: getPerformerStringValue(i, "Ethnicity"),
PenisLength: getPerformerPenisLength(i),
Circumcised: getPerformerCircumcised(i),
Rating: getIntPtr(getRating(i)),
IgnoreAutoTag: getIgnoreAutoTag(i),
TagIDs: models.NewRelatedIDs(tids),
URLs: models.NewRelatedStrings([]string{
getPerformerEmptyString(i, urlField),
}),
Favorite: getPerformerBoolValue(i),
Birthdate: getPerformerBirthdate(i),
DeathDate: getPerformerDeathDate(i),
Details: getPerformerStringValue(i, "Details"),
Ethnicity: getPerformerStringValue(i, "Ethnicity"),
PenisLength: getPerformerPenisLength(i),
Circumcised: getPerformerCircumcised(i),
Rating: getIntPtr(getRating(i)),
IgnoreAutoTag: getIgnoreAutoTag(i),
TagIDs: models.NewRelatedIDs(tids),
}
careerLength := getPerformerCareerLength(i)

9
pkg/sqlite/tables.go
View File

@@ -29,6 +29,7 @@ var (
scenesURLsJoinTable = goqu.T(scenesURLsTable)
performersAliasesJoinTable = goqu.T(performersAliasesTable)
performersURLsJoinTable = goqu.T(performerURLsTable)
performersTagsJoinTable = goqu.T(performersTagsTable)
performersStashIDsJoinTable = goqu.T("performer_stash_ids")
@@ -255,6 +256,14 @@ var (
stringColumn: performersAliasesJoinTable.Col(performerAliasColumn),
}
performersURLsTableMgr = &orderedValueTable[string]{
table: table{
table: performersURLsJoinTable,
idColumn: performersURLsJoinTable.Col(performerIDColumn),
},
valueColumn: performersURLsJoinTable.Col(performerURLColumn),
}
performersTagsTableMgr = &joinTable{
table: table{
table: performersTagsJoinTable,