From 5580525c2d919b7227adc3e5d899a5fa13227234 Mon Sep 17 00:00:00 2001 From: DingDongSoLong4 <99329275+DingDongSoLong4@users.noreply.github.com> Date: Thu, 13 Jul 2023 04:15:02 +0200 Subject: [PATCH] SQLite model refactoring, part 2 (#3839) * Treat empty image input as null * Add validation to models.Date * Allow zero dates in database * Make scene_markers.scene_id non-nullable * Drop scraped_items table * Remove movie/studio checksum * Add migration notes --------- Co-authored-by: WithoutPants <53250216+WithoutPants@users.noreply.github.com> --- graphql/documents/data/movie.graphql | 1 - graphql/documents/data/studio.graphql | 1 - graphql/schema/types/movie.graphql | 2 +- graphql/schema/types/studio.graphql | 2 +- internal/api/changeset_translator.go | 26 ++-- internal/api/resolver_model_movie.go | 6 + internal/api/resolver_model_studio.go | 6 + internal/api/resolver_mutation_gallery.go | 16 ++- internal/api/resolver_mutation_image.go | 10 +- internal/api/resolver_mutation_movie.go | 21 ++- internal/api/resolver_mutation_performer.go | 32 +++-- internal/api/resolver_mutation_scene.go | 23 ++- internal/api/resolver_mutation_studio.go | 20 +-- internal/autotag/integration_test.go | 1 - internal/identify/identify.go | 6 +- internal/identify/identify_test.go | 4 +- internal/identify/performer.go | 12 +- internal/identify/performer_test.go | 27 ++-- internal/identify/scene.go | 2 +- internal/identify/studio.go | 2 - internal/identify/studio_test.go | 9 +- internal/manager/filename_parser.go | 28 ++-- internal/manager/json_utils.go | 8 -- internal/manager/repository.go | 2 - internal/manager/task_export.go | 99 +------------ internal/manager/task_import.go | 85 ----------- internal/manager/task_stash_box_tag.go | 5 +- pkg/gallery/export_test.go | 14 +- pkg/gallery/import.go | 6 +- pkg/image/export_test.go | 14 +- pkg/image/import.go | 6 +- pkg/models/date.go | 16 ++- pkg/models/errors.go | 2 + pkg/models/jsonschema/scraped.go | 49 ------- pkg/models/mocks/ScrapedItemReaderWriter.go | 61 -------- pkg/models/mocks/transaction.go | 1 - pkg/models/model_movie.go | 5 - pkg/models/model_scene_test.go | 2 +- pkg/models/model_scraped_item.go | 36 ----- pkg/models/model_studio.go | 5 - pkg/models/repository.go | 1 - pkg/models/scraped.go | 21 --- pkg/movie/export_test.go | 14 +- pkg/movie/import.go | 10 +- pkg/performer/export_test.go | 4 +- pkg/performer/import.go | 12 +- pkg/performer/validate_test.go | 4 +- pkg/scene/export_test.go | 16 +-- pkg/scene/import.go | 6 +- pkg/sqlite/database.go | 2 +- pkg/sqlite/date.go | 14 +- pkg/sqlite/gallery.go | 3 +- pkg/sqlite/gallery_test.go | 18 +-- pkg/sqlite/image_test.go | 12 +- pkg/sqlite/migrations/48_cleanup.up.sql | 91 ++++++++++++ pkg/sqlite/migrations/48_premigrate.go | 150 ++++++++++++++++++++ pkg/sqlite/movies.go | 4 - pkg/sqlite/movies_test.go | 3 - pkg/sqlite/performer.go | 2 +- pkg/sqlite/performer_test.go | 12 +- pkg/sqlite/repository.go | 105 -------------- pkg/sqlite/scene.go | 3 +- pkg/sqlite/scene_marker.go | 7 +- pkg/sqlite/scene_test.go | 20 ++- pkg/sqlite/scraped_item.go | 81 ----------- pkg/sqlite/setup_test.go | 8 +- pkg/sqlite/sql.go | 4 +- pkg/sqlite/studio.go | 11 -- pkg/sqlite/transaction.go | 1 - pkg/studio/import.go | 4 - pkg/studio/import_test.go | 2 - pkg/utils/image.go | 4 + ui/v2.5/src/docs/en/MigrationNotes/48.md | 3 + ui/v2.5/src/docs/en/MigrationNotes/index.ts | 2 + 74 files changed, 520 insertions(+), 807 deletions(-) delete mode 100644 pkg/models/jsonschema/scraped.go delete mode 100644 pkg/models/mocks/ScrapedItemReaderWriter.go delete mode 100644 pkg/models/scraped.go create mode 100644 pkg/sqlite/migrations/48_cleanup.up.sql create mode 100644 pkg/sqlite/migrations/48_premigrate.go delete mode 100644 pkg/sqlite/scraped_item.go create mode 100644 ui/v2.5/src/docs/en/MigrationNotes/48.md diff --git a/graphql/documents/data/movie.graphql b/graphql/documents/data/movie.graphql index 1605e039e..4aaf90bc0 100644 --- a/graphql/documents/data/movie.graphql +++ b/graphql/documents/data/movie.graphql @@ -1,6 +1,5 @@ fragment MovieData on Movie { id - checksum name aliases duration diff --git a/graphql/documents/data/studio.graphql b/graphql/documents/data/studio.graphql index 73b9e0454..3badb9bf6 100644 --- a/graphql/documents/data/studio.graphql +++ b/graphql/documents/data/studio.graphql @@ -1,6 +1,5 @@ fragment StudioData on Studio { id - checksum name url parent_studio { diff --git a/graphql/schema/types/movie.graphql b/graphql/schema/types/movie.graphql index 7370cdc52..5220ad0da 100644 --- a/graphql/schema/types/movie.graphql +++ b/graphql/schema/types/movie.graphql @@ -1,7 +1,7 @@ type Movie { id: ID! - checksum: String! name: String! + checksum: String! @deprecated(reason: "MD5 hash of name, use name directly") aliases: String """Duration in seconds""" duration: Int diff --git a/graphql/schema/types/studio.graphql b/graphql/schema/types/studio.graphql index 1fda1df2d..5516ac23c 100644 --- a/graphql/schema/types/studio.graphql +++ b/graphql/schema/types/studio.graphql @@ -1,7 +1,7 @@ type Studio { id: ID! - checksum: String! name: String! + checksum: String! @deprecated(reason: "MD5 hash of name, use name directly") url: String parent_studio: Studio child_studios: [Studio!]! diff --git a/internal/api/changeset_translator.go b/internal/api/changeset_translator.go index c05d17978..e40b8fe0e 100644 --- a/internal/api/changeset_translator.go +++ b/internal/api/changeset_translator.go @@ -107,28 +107,36 @@ func (t changesetTranslator) optionalString(value *string, field string) models. return models.NewOptionalStringPtr(value) } -func (t changesetTranslator) optionalDate(value *string, field string) models.OptionalDate { +func (t changesetTranslator) optionalDate(value *string, field string) (models.OptionalDate, error) { if !t.hasField(field) { - return models.OptionalDate{} + return models.OptionalDate{}, nil } if value == nil || *value == "" { return models.OptionalDate{ Set: true, Null: true, - } + }, nil } - return models.NewOptionalDate(models.NewDate(*value)) + date, err := models.ParseDate(*value) + if err != nil { + return models.OptionalDate{}, err + } + + return models.NewOptionalDate(date), nil } -func (t changesetTranslator) datePtr(value *string, field string) *models.Date { - if value == nil { - return nil +func (t changesetTranslator) datePtr(value *string, field string) (*models.Date, error) { + if value == nil || *value == "" { + return nil, nil } - d := models.NewDate(*value) - return &d + date, err := models.ParseDate(*value) + if err != nil { + return nil, err + } + return &date, nil } func (t changesetTranslator) intPtrFromString(value *string, field string) (*int, error) { diff --git a/internal/api/resolver_model_movie.go b/internal/api/resolver_model_movie.go index 81e367b0b..8e60fda81 100644 --- a/internal/api/resolver_model_movie.go +++ b/internal/api/resolver_model_movie.go @@ -5,9 +5,15 @@ import ( "github.com/stashapp/stash/internal/api/loaders" "github.com/stashapp/stash/internal/api/urlbuilders" + "github.com/stashapp/stash/pkg/hash/md5" "github.com/stashapp/stash/pkg/models" ) +func (r *movieResolver) Checksum(ctx context.Context, obj *models.Movie) (string, error) { + // generate checksum from movie name + return md5.FromString(obj.Name), nil +} + func (r *movieResolver) Date(ctx context.Context, obj *models.Movie) (*string, error) { if obj.Date != nil { result := obj.Date.String() diff --git a/internal/api/resolver_model_studio.go b/internal/api/resolver_model_studio.go index 84f17eb84..f483a2904 100644 --- a/internal/api/resolver_model_studio.go +++ b/internal/api/resolver_model_studio.go @@ -6,6 +6,7 @@ import ( "github.com/stashapp/stash/internal/api/loaders" "github.com/stashapp/stash/internal/api/urlbuilders" "github.com/stashapp/stash/pkg/gallery" + "github.com/stashapp/stash/pkg/hash/md5" "github.com/stashapp/stash/pkg/image" "github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/movie" @@ -13,6 +14,11 @@ import ( "github.com/stashapp/stash/pkg/scene" ) +func (r *studioResolver) Checksum(ctx context.Context, obj *models.Studio) (string, error) { + // generate checksum from studio name + return md5.FromString(obj.Name), nil +} + func (r *studioResolver) ImagePath(ctx context.Context, obj *models.Studio) (*string, error) { var hasImage bool if err := r.withReadTxn(ctx, func(ctx context.Context) error { diff --git a/internal/api/resolver_mutation_gallery.go b/internal/api/resolver_mutation_gallery.go index e0c2730b9..9ea04e8bd 100644 --- a/internal/api/resolver_mutation_gallery.go +++ b/internal/api/resolver_mutation_gallery.go @@ -66,9 +66,9 @@ func (r *mutationResolver) GalleryCreate(ctx context.Context, input GalleryCreat UpdatedAt: currentTime, } - if input.Date != nil { - d := models.NewDate(*input.Date) - newGallery.Date = &d + newGallery.Date, err = translator.datePtr(input.Date, "date") + if err != nil { + return nil, fmt.Errorf("converting date: %w", err) } newGallery.StudioID, err = translator.intPtrFromString(input.StudioID, "studio_id") if err != nil { @@ -182,7 +182,10 @@ func (r *mutationResolver) galleryUpdate(ctx context.Context, input models.Galle updatedGallery.Details = translator.optionalString(input.Details, "details") updatedGallery.URL = translator.optionalString(input.URL, "url") - updatedGallery.Date = translator.optionalDate(input.Date, "date") + updatedGallery.Date, err = translator.optionalDate(input.Date, "date") + if err != nil { + return nil, fmt.Errorf("converting date: %w", err) + } updatedGallery.Rating = translator.ratingConversionOptional(input.Rating, input.Rating100) updatedGallery.StudioID, err = translator.optionalIntFromString(input.StudioID, "studio_id") if err != nil { @@ -262,7 +265,10 @@ func (r *mutationResolver) BulkGalleryUpdate(ctx context.Context, input BulkGall updatedGallery.Details = translator.optionalString(input.Details, "details") updatedGallery.URL = translator.optionalString(input.URL, "url") - updatedGallery.Date = translator.optionalDate(input.Date, "date") + updatedGallery.Date, err = translator.optionalDate(input.Date, "date") + if err != nil { + return nil, fmt.Errorf("converting date: %w", err) + } updatedGallery.Rating = translator.ratingConversionOptional(input.Rating, input.Rating100) updatedGallery.StudioID, err = translator.optionalIntFromString(input.StudioID, "studio_id") if err != nil { diff --git a/internal/api/resolver_mutation_image.go b/internal/api/resolver_mutation_image.go index fcbf064dc..6d5c3a88a 100644 --- a/internal/api/resolver_mutation_image.go +++ b/internal/api/resolver_mutation_image.go @@ -107,7 +107,10 @@ func (r *mutationResolver) imageUpdate(ctx context.Context, input ImageUpdateInp updatedImage.Title = translator.optionalString(input.Title, "title") updatedImage.Rating = translator.ratingConversionOptional(input.Rating, input.Rating100) updatedImage.URL = translator.optionalString(input.URL, "url") - updatedImage.Date = translator.optionalDate(input.Date, "date") + updatedImage.Date, err = translator.optionalDate(input.Date, "date") + if err != nil { + return nil, fmt.Errorf("converting date: %w", err) + } updatedImage.StudioID, err = translator.optionalIntFromString(input.StudioID, "studio_id") if err != nil { return nil, fmt.Errorf("converting studio id: %w", err) @@ -206,7 +209,10 @@ func (r *mutationResolver) BulkImageUpdate(ctx context.Context, input BulkImageU updatedImage.Title = translator.optionalString(input.Title, "title") updatedImage.Rating = translator.ratingConversionOptional(input.Rating, input.Rating100) updatedImage.URL = translator.optionalString(input.URL, "url") - updatedImage.Date = translator.optionalDate(input.Date, "date") + updatedImage.Date, err = translator.optionalDate(input.Date, "date") + if err != nil { + return nil, fmt.Errorf("converting date: %w", err) + } updatedImage.StudioID, err = translator.optionalIntFromString(input.StudioID, "studio_id") if err != nil { return nil, fmt.Errorf("converting studio id: %w", err) diff --git a/internal/api/resolver_mutation_movie.go b/internal/api/resolver_mutation_movie.go index 4534bc965..a27e8f98c 100644 --- a/internal/api/resolver_mutation_movie.go +++ b/internal/api/resolver_mutation_movie.go @@ -6,7 +6,6 @@ import ( "strconv" "time" - "github.com/stashapp/stash/pkg/hash/md5" "github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/plugin" "github.com/stashapp/stash/pkg/sliceutil/stringslice" @@ -30,18 +29,15 @@ func (r *mutationResolver) MovieCreate(ctx context.Context, input MovieCreateInp } // generate checksum from movie name rather than image - checksum := md5.FromString(input.Name) // Populate a new movie from the input currentTime := time.Now() newMovie := models.Movie{ - Checksum: checksum, Name: input.Name, CreatedAt: currentTime, UpdatedAt: currentTime, Aliases: translator.string(input.Aliases, "aliases"), Duration: input.Duration, - Date: translator.datePtr(input.Date, "date"), Rating: translator.ratingConversionInt(input.Rating, input.Rating100), Director: translator.string(input.Director, "director"), Synopsis: translator.string(input.Synopsis, "synopsis"), @@ -50,6 +46,10 @@ func (r *mutationResolver) MovieCreate(ctx context.Context, input MovieCreateInp var err error + newMovie.Date, err = translator.datePtr(input.Date, "date") + if err != nil { + return nil, fmt.Errorf("converting date: %w", err) + } newMovie.StudioID, err = translator.intPtrFromString(input.StudioID, "studio_id") if err != nil { return nil, fmt.Errorf("converting studio id: %w", err) @@ -123,16 +123,13 @@ func (r *mutationResolver) MovieUpdate(ctx context.Context, input MovieUpdateInp // Populate movie from the input updatedMovie := models.NewMoviePartial() - if input.Name != nil { - // generate checksum from movie name rather than image - checksum := md5.FromString(*input.Name) - updatedMovie.Name = models.NewOptionalString(*input.Name) - updatedMovie.Checksum = models.NewOptionalString(checksum) - } - + updatedMovie.Name = translator.optionalString(input.Name, "name") updatedMovie.Aliases = translator.optionalString(input.Aliases, "aliases") updatedMovie.Duration = translator.optionalInt(input.Duration, "duration") - updatedMovie.Date = translator.optionalDate(input.Date, "date") + updatedMovie.Date, err = translator.optionalDate(input.Date, "date") + if err != nil { + return nil, fmt.Errorf("converting date: %w", err) + } updatedMovie.Rating = translator.ratingConversionOptional(input.Rating, input.Rating100) updatedMovie.Director = translator.optionalString(input.Director, "director") updatedMovie.Synopsis = translator.optionalString(input.Synopsis, "synopsis") diff --git a/internal/api/resolver_mutation_performer.go b/internal/api/resolver_mutation_performer.go index 9fecd72d3..2c23f063a 100644 --- a/internal/api/resolver_mutation_performer.go +++ b/internal/api/resolver_mutation_performer.go @@ -75,13 +75,13 @@ func (r *mutationResolver) PerformerCreate(ctx context.Context, input PerformerC StashIDs: models.NewRelatedStashIDs(stashIDPtrSliceToSlice(input.StashIds)), } - if input.Birthdate != nil { - d := models.NewDate(*input.Birthdate) - newPerformer.Birthdate = &d + newPerformer.Birthdate, err = translator.datePtr(input.Birthdate, "birthdate") + if err != nil { + return nil, fmt.Errorf("converting birthdate: %w", err) } - if input.DeathDate != nil { - d := models.NewDate(*input.DeathDate) - newPerformer.DeathDate = &d + newPerformer.DeathDate, err = translator.datePtr(input.DeathDate, "death_date") + if err != nil { + return nil, fmt.Errorf("converting death date: %w", err) } // prefer height_cm over height @@ -157,7 +157,10 @@ func (r *mutationResolver) PerformerUpdate(ctx context.Context, input PerformerU updatedPerformer.Disambiguation = translator.optionalString(input.Disambiguation, "disambiguation") updatedPerformer.URL = translator.optionalString(input.URL, "url") updatedPerformer.Gender = translator.optionalString((*string)(input.Gender), "gender") - updatedPerformer.Birthdate = translator.optionalDate(input.Birthdate, "birthdate") + updatedPerformer.Birthdate, err = translator.optionalDate(input.Birthdate, "birthdate") + if err != nil { + return nil, fmt.Errorf("converting birthdate: %w", err) + } updatedPerformer.Ethnicity = translator.optionalString(input.Ethnicity, "ethnicity") updatedPerformer.Country = translator.optionalString(input.Country, "country") updatedPerformer.EyeColor = translator.optionalString(input.EyeColor, "eye_color") @@ -184,7 +187,10 @@ func (r *mutationResolver) PerformerUpdate(ctx context.Context, input PerformerU updatedPerformer.Favorite = translator.optionalBool(input.Favorite, "favorite") updatedPerformer.Rating = translator.ratingConversionOptional(input.Rating, input.Rating100) updatedPerformer.Details = translator.optionalString(input.Details, "details") - updatedPerformer.DeathDate = translator.optionalDate(input.DeathDate, "death_date") + updatedPerformer.DeathDate, err = translator.optionalDate(input.DeathDate, "death_date") + if err != nil { + return nil, fmt.Errorf("converting death date: %w", err) + } updatedPerformer.HairColor = translator.optionalString(input.HairColor, "hair_color") updatedPerformer.Weight = translator.optionalInt(input.Weight, "weight") updatedPerformer.IgnoreAutoTag = translator.optionalBool(input.IgnoreAutoTag, "ignore_auto_tag") @@ -282,7 +288,10 @@ func (r *mutationResolver) BulkPerformerUpdate(ctx context.Context, input BulkPe updatedPerformer.Disambiguation = translator.optionalString(input.Disambiguation, "disambiguation") updatedPerformer.URL = translator.optionalString(input.URL, "url") updatedPerformer.Gender = translator.optionalString((*string)(input.Gender), "gender") - updatedPerformer.Birthdate = translator.optionalDate(input.Birthdate, "birthdate") + updatedPerformer.Birthdate, err = translator.optionalDate(input.Birthdate, "birthdate") + if err != nil { + return nil, fmt.Errorf("converting birthdate: %w", err) + } updatedPerformer.Ethnicity = translator.optionalString(input.Ethnicity, "ethnicity") updatedPerformer.Country = translator.optionalString(input.Country, "country") updatedPerformer.EyeColor = translator.optionalString(input.EyeColor, "eye_color") @@ -309,7 +318,10 @@ func (r *mutationResolver) BulkPerformerUpdate(ctx context.Context, input BulkPe updatedPerformer.Favorite = translator.optionalBool(input.Favorite, "favorite") updatedPerformer.Rating = translator.ratingConversionOptional(input.Rating, input.Rating100) updatedPerformer.Details = translator.optionalString(input.Details, "details") - updatedPerformer.DeathDate = translator.optionalDate(input.DeathDate, "death_date") + updatedPerformer.DeathDate, err = translator.optionalDate(input.DeathDate, "death_date") + if err != nil { + return nil, fmt.Errorf("converting death date: %w", err) + } updatedPerformer.HairColor = translator.optionalString(input.HairColor, "hair_color") updatedPerformer.Weight = translator.optionalInt(input.Weight, "weight") updatedPerformer.IgnoreAutoTag = translator.optionalBool(input.IgnoreAutoTag, "ignore_auto_tag") diff --git a/internal/api/resolver_mutation_scene.go b/internal/api/resolver_mutation_scene.go index 3b96b9f07..ef1c603ab 100644 --- a/internal/api/resolver_mutation_scene.go +++ b/internal/api/resolver_mutation_scene.go @@ -67,7 +67,6 @@ func (r *mutationResolver) SceneCreate(ctx context.Context, input SceneCreateInp Code: translator.string(input.Code, "code"), Details: translator.string(input.Details, "details"), Director: translator.string(input.Director, "director"), - Date: translator.datePtr(input.Date, "date"), Rating: translator.ratingConversionInt(input.Rating, input.Rating100), Organized: translator.bool(input.Organized, "organized"), PerformerIDs: models.NewRelatedIDs(performerIDs), @@ -77,6 +76,10 @@ func (r *mutationResolver) SceneCreate(ctx context.Context, input SceneCreateInp StashIDs: models.NewRelatedStashIDs(stashIDPtrSliceToSlice(input.StashIds)), } + newScene.Date, err = translator.datePtr(input.Date, "date") + if err != nil { + return nil, fmt.Errorf("converting date: %w", err) + } newScene.StudioID, err = translator.intPtrFromString(input.StudioID, "studio_id") if err != nil { return nil, fmt.Errorf("converting studio id: %w", err) @@ -89,7 +92,7 @@ func (r *mutationResolver) SceneCreate(ctx context.Context, input SceneCreateInp } var coverImageData []byte - if input.CoverImage != nil && *input.CoverImage != "" { + if input.CoverImage != nil { var err error coverImageData, err = utils.ProcessImageInput(ctx, *input.CoverImage) if err != nil { @@ -169,16 +172,21 @@ func (r *mutationResolver) ScenesUpdate(ctx context.Context, input []*models.Sce func scenePartialFromInput(input models.SceneUpdateInput, translator changesetTranslator) (*models.ScenePartial, error) { updatedScene := models.NewScenePartial() + + var err error + updatedScene.Title = translator.optionalString(input.Title, "title") updatedScene.Code = translator.optionalString(input.Code, "code") updatedScene.Details = translator.optionalString(input.Details, "details") updatedScene.Director = translator.optionalString(input.Director, "director") - updatedScene.Date = translator.optionalDate(input.Date, "date") + updatedScene.Date, err = translator.optionalDate(input.Date, "date") + if err != nil { + return nil, fmt.Errorf("converting date: %w", err) + } updatedScene.Rating = translator.ratingConversionOptional(input.Rating, input.Rating100) updatedScene.OCounter = translator.optionalInt(input.OCounter, "o_counter") updatedScene.PlayCount = translator.optionalInt(input.PlayCount, "play_count") updatedScene.PlayDuration = translator.optionalFloat64(input.PlayDuration, "play_duration") - var err error updatedScene.StudioID, err = translator.optionalIntFromString(input.StudioID, "studio_id") if err != nil { return nil, fmt.Errorf("converting studio id: %w", err) @@ -355,7 +363,10 @@ func (r *mutationResolver) BulkSceneUpdate(ctx context.Context, input BulkSceneU updatedScene.Code = translator.optionalString(input.Code, "code") updatedScene.Details = translator.optionalString(input.Details, "details") updatedScene.Director = translator.optionalString(input.Director, "director") - updatedScene.Date = translator.optionalDate(input.Date, "date") + updatedScene.Date, err = translator.optionalDate(input.Date, "date") + if err != nil { + return nil, fmt.Errorf("converting date: %w", err) + } updatedScene.Rating = translator.ratingConversionOptional(input.Rating, input.Rating100) updatedScene.StudioID, err = translator.optionalIntFromString(input.StudioID, "studio_id") if err != nil { @@ -602,7 +613,7 @@ func (r *mutationResolver) SceneMerge(ctx context.Context, input SceneMergeInput } var coverImageData []byte - if input.Values.CoverImage != nil && *input.Values.CoverImage != "" { + if input.Values.CoverImage != nil { var err error coverImageData, err = utils.ProcessImageInput(ctx, *input.Values.CoverImage) if err != nil { diff --git a/internal/api/resolver_mutation_studio.go b/internal/api/resolver_mutation_studio.go index bcbcea9b0..a35dc0b04 100644 --- a/internal/api/resolver_mutation_studio.go +++ b/internal/api/resolver_mutation_studio.go @@ -6,13 +6,11 @@ import ( "strconv" "time" - "github.com/stashapp/stash/pkg/hash/md5" - "github.com/stashapp/stash/pkg/sliceutil/stringslice" - "github.com/stashapp/stash/pkg/studio" - "github.com/stashapp/stash/internal/manager" "github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/plugin" + "github.com/stashapp/stash/pkg/sliceutil/stringslice" + "github.com/stashapp/stash/pkg/studio" "github.com/stashapp/stash/pkg/utils" ) @@ -32,13 +30,9 @@ func (r *mutationResolver) StudioCreate(ctx context.Context, input StudioCreateI inputMap: getUpdateInputMap(ctx), } - // generate checksum from studio name rather than image - checksum := md5.FromString(input.Name) - // Populate a new studio from the input currentTime := time.Now() newStudio := models.Studio{ - Checksum: checksum, Name: input.Name, CreatedAt: currentTime, UpdatedAt: currentTime, @@ -57,7 +51,7 @@ func (r *mutationResolver) StudioCreate(ctx context.Context, input StudioCreateI // Process the base 64 encoded image string var imageData []byte - if input.Image != nil && *input.Image != "" { + if input.Image != nil { imageData, err = utils.ProcessImageInput(ctx, *input.Image) if err != nil { return nil, err @@ -120,13 +114,7 @@ func (r *mutationResolver) StudioUpdate(ctx context.Context, input StudioUpdateI // Populate studio from the input updatedStudio := models.NewStudioPartial() - if input.Name != nil { - // generate checksum from studio name rather than image - checksum := md5.FromString(*input.Name) - updatedStudio.Name = models.NewOptionalString(*input.Name) - updatedStudio.Checksum = models.NewOptionalString(checksum) - } - + updatedStudio.Name = translator.optionalString(input.Name, "name") updatedStudio.URL = translator.optionalString(input.URL, "url") updatedStudio.Details = translator.optionalString(input.Details, "details") updatedStudio.Rating = translator.ratingConversionOptional(input.Rating, input.Rating100) diff --git a/internal/autotag/integration_test.go b/internal/autotag/integration_test.go index b31d65666..1c7b0ee2d 100644 --- a/internal/autotag/integration_test.go +++ b/internal/autotag/integration_test.go @@ -99,7 +99,6 @@ func createPerformer(ctx context.Context, pqb models.PerformerWriter) error { func createStudio(ctx context.Context, qb models.StudioWriter, name string) (*models.Studio, error) { // create the studio studio := models.Studio{ - Checksum: name, Name: name, } diff --git a/internal/identify/identify.go b/internal/identify/identify.go index 8d45e1009..37f13dc66 100644 --- a/internal/identify/identify.go +++ b/internal/identify/identify.go @@ -370,8 +370,10 @@ func getScenePartial(scene *models.Scene, scraped *scraper.ScrapedScene, fieldOp } if scraped.Date != nil && (scene.Date == nil || scene.Date.String() != *scraped.Date) { if shouldSetSingleValueField(fieldOptions["date"], scene.Date != nil) { - d := models.NewDate(*scraped.Date) - partial.Date = models.NewOptionalDate(d) + d, err := models.ParseDate(*scraped.Date) + if err == nil { + partial.Date = models.NewOptionalDate(d) + } } } if scraped.Details != nil && (scene.Details != *scraped.Details) { diff --git a/internal/identify/identify_test.go b/internal/identify/identify_test.go index 6c9f92cb2..30dd72803 100644 --- a/internal/identify/identify_test.go +++ b/internal/identify/identify_test.go @@ -346,8 +346,8 @@ func Test_getScenePartial(t *testing.T) { scrapedURL = "scrapedURL" ) - originalDateObj := models.NewDate(originalDate) - scrapedDateObj := models.NewDate(scrapedDate) + originalDateObj, _ := models.ParseDate(originalDate) + scrapedDateObj, _ := models.ParseDate(scrapedDate) originalScene := &models.Scene{ Title: originalTitle, diff --git a/internal/identify/performer.go b/internal/identify/performer.go index a75bfb024..7fa300180 100644 --- a/internal/identify/performer.go +++ b/internal/identify/performer.go @@ -80,12 +80,16 @@ func scrapedToPerformerInput(performer *models.ScrapedPerformer) models.Performe ret.Disambiguation = *performer.Disambiguation } if performer.Birthdate != nil { - d := models.NewDate(*performer.Birthdate) - ret.Birthdate = &d + d, err := models.ParseDate(*performer.Birthdate) + if err == nil { + ret.Birthdate = &d + } } if performer.DeathDate != nil { - d := models.NewDate(*performer.DeathDate) - ret.DeathDate = &d + d, err := models.ParseDate(*performer.DeathDate) + if err == nil { + ret.DeathDate = &d + } } if performer.Gender != nil { v := models.GenderEnum(*performer.Gender) diff --git a/internal/identify/performer_test.go b/internal/identify/performer_test.go index 2e22837c4..f40e0cabb 100644 --- a/internal/identify/performer_test.go +++ b/internal/identify/performer_test.go @@ -244,13 +244,24 @@ func Test_scrapedToPerformerInput(t *testing.T) { return &ret } - dateToDatePtr := func(d models.Date) *models.Date { + dateFromInt := func(i int) *models.Date { + t := time.Date(2001, 1, i, 0, 0, 0, 0, time.UTC) + d := models.Date{Time: t} return &d } + dateStrFromInt := func(i int) *string { + s := dateFromInt(i).String() + return &s + } - genderPtr := func(g models.GenderEnum) *models.GenderEnum { + genderFromInt := func(i int) *models.GenderEnum { + g := models.AllGenderEnum[i%len(models.AllGenderEnum)] return &g } + genderStrFromInt := func(i int) *string { + s := genderFromInt(i).String() + return &s + } tests := []struct { name string @@ -262,9 +273,9 @@ func Test_scrapedToPerformerInput(t *testing.T) { &models.ScrapedPerformer{ Name: &name, Disambiguation: nextVal(), - Birthdate: nextVal(), - DeathDate: nextVal(), - Gender: nextVal(), + Birthdate: dateStrFromInt(*nextIntVal()), + DeathDate: dateStrFromInt(*nextIntVal()), + Gender: genderStrFromInt(*nextIntVal()), Ethnicity: nextVal(), Country: nextVal(), EyeColor: nextVal(), @@ -285,9 +296,9 @@ func Test_scrapedToPerformerInput(t *testing.T) { models.Performer{ Name: name, Disambiguation: *nextVal(), - Birthdate: dateToDatePtr(models.NewDate(*nextVal())), - DeathDate: dateToDatePtr(models.NewDate(*nextVal())), - Gender: genderPtr(models.GenderEnum(*nextVal())), + Birthdate: dateFromInt(*nextIntVal()), + DeathDate: dateFromInt(*nextIntVal()), + Gender: genderFromInt(*nextIntVal()), Ethnicity: *nextVal(), Country: *nextVal(), EyeColor: *nextVal(), diff --git a/internal/identify/scene.go b/internal/identify/scene.go index 7568d1b1f..73ac39d41 100644 --- a/internal/identify/scene.go +++ b/internal/identify/scene.go @@ -246,7 +246,7 @@ func (g sceneRelationships) stashIDs(ctx context.Context) ([]models.StashID, err func (g sceneRelationships) cover(ctx context.Context) ([]byte, error) { scraped := g.result.result.Image - if scraped == nil { + if scraped == nil || *scraped == "" { return nil, nil } diff --git a/internal/identify/studio.go b/internal/identify/studio.go index 682245d5b..c33541c0d 100644 --- a/internal/identify/studio.go +++ b/internal/identify/studio.go @@ -5,7 +5,6 @@ import ( "fmt" "time" - "github.com/stashapp/stash/pkg/hash/md5" "github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/utils" ) @@ -54,7 +53,6 @@ func scrapedToStudioInput(studio *models.ScrapedStudio) models.Studio { currentTime := time.Now() ret := models.Studio{ Name: studio.Name, - Checksum: md5.FromString(studio.Name), CreatedAt: currentTime, UpdatedAt: currentTime, } diff --git a/internal/identify/studio_test.go b/internal/identify/studio_test.go index def09641e..dd31f335c 100644 --- a/internal/identify/studio_test.go +++ b/internal/identify/studio_test.go @@ -117,7 +117,6 @@ func Test_createMissingStudio(t *testing.T) { func Test_scrapedToStudioInput(t *testing.T) { const name = "name" - const md5 = "b068931cc450442b63f5b3d276ea4297" url := "url" tests := []struct { @@ -132,9 +131,8 @@ func Test_scrapedToStudioInput(t *testing.T) { URL: &url, }, models.Studio{ - Name: name, - Checksum: md5, - URL: url, + Name: name, + URL: url, }, }, { @@ -143,8 +141,7 @@ func Test_scrapedToStudioInput(t *testing.T) { Name: name, }, models.Studio{ - Name: name, - Checksum: md5, + Name: name, }, }, } diff --git a/internal/manager/filename_parser.go b/internal/manager/filename_parser.go index 9ee876a8c..82fe686c0 100644 --- a/internal/manager/filename_parser.go +++ b/internal/manager/filename_parser.go @@ -262,10 +262,11 @@ func validateRating100(rating100 int) bool { return rating100 >= 1 && rating100 <= 100 } -func validateDate(dateStr string) bool { +// returns nil if invalid +func parseDate(dateStr string) *models.Date { splits := strings.Split(dateStr, "-") if len(splits) != 3 { - return false + return nil } year, _ := strconv.Atoi(splits[0]) @@ -274,19 +275,23 @@ func validateDate(dateStr string) bool { // assume year must be between 1900 and 2100 if year < 1900 || year > 2100 { - return false + return nil } if month < 1 || month > 12 { - return false + return nil } // not checking individual months to ensure date is in the correct range if d < 1 || d > 31 { - return false + return nil } - return true + ret, err := models.ParseDate(dateStr) + if err != nil { + return nil + } + return &ret } func (h *sceneHolder) setDate(field *parserField, value string) { @@ -315,9 +320,9 @@ func (h *sceneHolder) setDate(field *parserField, value string) { // ensure the date is valid // only set if new value is different from the old - if validateDate(fullDate) && h.scene.Date != nil && h.scene.Date.String() != fullDate { - d := models.NewDate(fullDate) - h.result.Date = &d + newDate := parseDate(fullDate) + if newDate != nil && h.scene.Date != nil && *h.scene.Date != *newDate { + h.result.Date = newDate } } @@ -346,10 +351,7 @@ func (h *sceneHolder) setField(field parserField, value interface{}) { v := value.(string) h.result.Title = v case "date": - if validateDate(value.(string)) { - d := models.NewDate(value.(string)) - h.result.Date = &d - } + h.result.Date = parseDate(value.(string)) case "rating": rating, _ := strconv.Atoi(value.(string)) if validateRating(rating) { diff --git a/internal/manager/json_utils.go b/internal/manager/json_utils.go index a2cb61b36..c90c95029 100644 --- a/internal/manager/json_utils.go +++ b/internal/manager/json_utils.go @@ -11,14 +11,6 @@ type jsonUtils struct { json paths.JSONPaths } -func (jp *jsonUtils) getScraped() ([]jsonschema.ScrapedItem, error) { - return jsonschema.LoadScrapedFile(jp.json.ScrapedFile) -} - -func (jp *jsonUtils) saveScaped(scraped []jsonschema.ScrapedItem) error { - return jsonschema.SaveScrapedFile(jp.json.ScrapedFile, scraped) -} - func (jp *jsonUtils) savePerformer(fn string, performer *jsonschema.Performer) error { return jsonschema.SavePerformerFile(filepath.Join(jp.json.Performers, fn), performer) } diff --git a/internal/manager/repository.go b/internal/manager/repository.go index a8f0b9e7d..f6f8176aa 100644 --- a/internal/manager/repository.go +++ b/internal/manager/repository.go @@ -56,7 +56,6 @@ type Repository struct { Performer models.PerformerReaderWriter Scene SceneReaderWriter SceneMarker models.SceneMarkerReaderWriter - ScrapedItem models.ScrapedItemReaderWriter Studio models.StudioReaderWriter Tag models.TagReaderWriter SavedFilter models.SavedFilterReaderWriter @@ -88,7 +87,6 @@ func sqliteRepository(d *sqlite.Database) Repository { Performer: txnRepo.Performer, Scene: d.Scene, SceneMarker: txnRepo.SceneMarker, - ScrapedItem: txnRepo.ScrapedItem, Studio: txnRepo.Studio, Tag: txnRepo.Tag, SavedFilter: txnRepo.SavedFilter, diff --git a/internal/manager/task_export.go b/internal/manager/task_export.go index 53fb3b389..f186d3eb4 100644 --- a/internal/manager/task_export.go +++ b/internal/manager/task_export.go @@ -173,10 +173,6 @@ func (t *ExportTask) Start(ctx context.Context, wg *sync.WaitGroup) { t.ExportStudios(ctx, workerCount, r) t.ExportTags(ctx, workerCount, r) - if t.full { - t.ExportScrapedItems(ctx, r) - } - return nil }) if txnErr != nil { @@ -296,13 +292,13 @@ func (t *ExportTask) populateMovieScenes(ctx context.Context, repo Repository) { } if err != nil { - logger.Errorf("[movies] failed to fetch movies: %s", err.Error()) + logger.Errorf("[movies] failed to fetch movies: %v", err) } for _, m := range movies { scenes, err := sceneReader.FindByMovieID(ctx, m.ID) if err != nil { - logger.Errorf("[movies] <%s> failed to fetch scenes for movie: %s", m.Checksum, err.Error()) + logger.Errorf("[movies] <%s> failed to fetch scenes for movie: %v", m.Name, err) continue } @@ -978,14 +974,14 @@ func (t *ExportTask) exportStudio(ctx context.Context, wg *sync.WaitGroup, jobCh newStudioJSON, err := studio.ToJSON(ctx, studioReader, s) if err != nil { - logger.Errorf("[studios] <%s> error getting studio JSON: %s", s.Checksum, err.Error()) + logger.Errorf("[studios] <%s> error getting studio JSON: %v", s.Name, err) continue } fn := newStudioJSON.Filename() if err := t.json.saveStudio(fn, newStudioJSON); err != nil { - logger.Errorf("[studios] <%s> failed to save json: %s", s.Checksum, err.Error()) + logger.Errorf("[studios] <%s> failed to save json: %v", s.Name, err) } } } @@ -1101,7 +1097,7 @@ func (t *ExportTask) exportMovie(ctx context.Context, wg *sync.WaitGroup, jobCha newMovieJSON, err := movie.ToJSON(ctx, movieReader, studioReader, m) if err != nil { - logger.Errorf("[movies] <%s> error getting tag JSON: %s", m.Checksum, err.Error()) + logger.Errorf("[movies] <%s> error getting tag JSON: %v", m.Name, err) continue } @@ -1114,90 +1110,7 @@ func (t *ExportTask) exportMovie(ctx context.Context, wg *sync.WaitGroup, jobCha fn := newMovieJSON.Filename() if err := t.json.saveMovie(fn, newMovieJSON); err != nil { - logger.Errorf("[movies] <%s> failed to save json: %s", fn, err.Error()) + logger.Errorf("[movies] <%s> failed to save json: %v", m.Name, err) } } } - -func (t *ExportTask) ExportScrapedItems(ctx context.Context, repo Repository) { - qb := repo.ScrapedItem - sqb := repo.Studio - scrapedItems, err := qb.All(ctx) - if err != nil { - logger.Errorf("[scraped sites] failed to fetch all items: %s", err.Error()) - } - - logger.Info("[scraped sites] exporting") - - scraped := []jsonschema.ScrapedItem{} - - for i, scrapedItem := range scrapedItems { - index := i + 1 - logger.Progressf("[scraped sites] %d of %d", index, len(scrapedItems)) - - var studioName string - if scrapedItem.StudioID.Valid { - studio, _ := sqb.Find(ctx, int(scrapedItem.StudioID.Int64)) - if studio != nil { - studioName = studio.Name - } - } - - newScrapedItemJSON := jsonschema.ScrapedItem{} - - if scrapedItem.Title.Valid { - newScrapedItemJSON.Title = scrapedItem.Title.String - } - if scrapedItem.Description.Valid { - newScrapedItemJSON.Description = scrapedItem.Description.String - } - if scrapedItem.URL.Valid { - newScrapedItemJSON.URL = scrapedItem.URL.String - } - if scrapedItem.Date != nil { - newScrapedItemJSON.Date = scrapedItem.Date.String() - } - if scrapedItem.Rating.Valid { - newScrapedItemJSON.Rating = scrapedItem.Rating.String - } - if scrapedItem.Tags.Valid { - newScrapedItemJSON.Tags = scrapedItem.Tags.String - } - if scrapedItem.Models.Valid { - newScrapedItemJSON.Models = scrapedItem.Models.String - } - if scrapedItem.Episode.Valid { - newScrapedItemJSON.Episode = int(scrapedItem.Episode.Int64) - } - if scrapedItem.GalleryFilename.Valid { - newScrapedItemJSON.GalleryFilename = scrapedItem.GalleryFilename.String - } - if scrapedItem.GalleryURL.Valid { - newScrapedItemJSON.GalleryURL = scrapedItem.GalleryURL.String - } - if scrapedItem.VideoFilename.Valid { - newScrapedItemJSON.VideoFilename = scrapedItem.VideoFilename.String - } - if scrapedItem.VideoURL.Valid { - newScrapedItemJSON.VideoURL = scrapedItem.VideoURL.String - } - - newScrapedItemJSON.Studio = studioName - updatedAt := json.JSONTime{Time: scrapedItem.UpdatedAt} // TODO keeping ruby format - newScrapedItemJSON.UpdatedAt = updatedAt - - scraped = append(scraped, newScrapedItemJSON) - } - - scrapedJSON, err := t.json.getScraped() - if err != nil { - logger.Debugf("[scraped sites] error reading json: %s", err.Error()) - } - if !jsonschema.CompareJSON(scrapedJSON, scraped) { - if err := t.json.saveScaped(scraped); err != nil { - logger.Errorf("[scraped sites] failed to save json: %s", err.Error()) - } - } - - logger.Infof("[scraped sites] export complete") -} diff --git a/internal/manager/task_import.go b/internal/manager/task_import.go index 2cd226427..aa0e7ec63 100644 --- a/internal/manager/task_import.go +++ b/internal/manager/task_import.go @@ -3,13 +3,11 @@ package manager import ( "archive/zip" "context" - "database/sql" "errors" "fmt" "io" "os" "path/filepath" - "time" "github.com/99designs/gqlgen/graphql" "github.com/stashapp/stash/pkg/fsutil" @@ -17,7 +15,6 @@ import ( "github.com/stashapp/stash/pkg/image" "github.com/stashapp/stash/pkg/logger" "github.com/stashapp/stash/pkg/models" - "github.com/stashapp/stash/pkg/models/json" "github.com/stashapp/stash/pkg/models/jsonschema" "github.com/stashapp/stash/pkg/models/paths" "github.com/stashapp/stash/pkg/movie" @@ -25,7 +22,6 @@ import ( "github.com/stashapp/stash/pkg/scene" "github.com/stashapp/stash/pkg/studio" "github.com/stashapp/stash/pkg/tag" - "github.com/stashapp/stash/pkg/utils" ) type ImportTask struct { @@ -38,7 +34,6 @@ type ImportTask struct { DuplicateBehaviour ImportDuplicateEnum MissingRefBehaviour models.ImportMissingRefEnum - scraped []jsonschema.ScrapedItem fileNamingAlgorithm models.HashAlgorithm } @@ -112,12 +107,6 @@ func (t *ImportTask) Start(ctx context.Context) { t.MissingRefBehaviour = models.ImportMissingRefEnumFail } - scraped, _ := t.json.getScraped() - if scraped == nil { - logger.Warn("missing scraped json") - } - t.scraped = scraped - if t.Reset { err := t.txnManager.Reset() @@ -134,7 +123,6 @@ func (t *ImportTask) Start(ctx context.Context) { t.ImportFiles(ctx) t.ImportGalleries(ctx) - t.ImportScrapedItems(ctx) t.ImportScenes(ctx) t.ImportImages(ctx) } @@ -614,61 +602,6 @@ func (t *ImportTask) ImportTag(ctx context.Context, tagJSON *jsonschema.Tag, pen return nil } -func (t *ImportTask) ImportScrapedItems(ctx context.Context) { - if err := t.txnManager.WithTxn(ctx, func(ctx context.Context) error { - logger.Info("[scraped sites] importing") - r := t.txnManager - qb := r.ScrapedItem - sqb := r.Studio - currentTime := time.Now() - - for i, mappingJSON := range t.scraped { - index := i + 1 - logger.Progressf("[scraped sites] %d of %d", index, len(t.scraped)) - - newScrapedItem := models.ScrapedItem{ - Title: sql.NullString{String: mappingJSON.Title, Valid: true}, - Description: sql.NullString{String: mappingJSON.Description, Valid: true}, - URL: sql.NullString{String: mappingJSON.URL, Valid: true}, - Rating: sql.NullString{String: mappingJSON.Rating, Valid: true}, - Tags: sql.NullString{String: mappingJSON.Tags, Valid: true}, - Models: sql.NullString{String: mappingJSON.Models, Valid: true}, - Episode: sql.NullInt64{Int64: int64(mappingJSON.Episode), Valid: true}, - GalleryFilename: sql.NullString{String: mappingJSON.GalleryFilename, Valid: true}, - GalleryURL: sql.NullString{String: mappingJSON.GalleryURL, Valid: true}, - VideoFilename: sql.NullString{String: mappingJSON.VideoFilename, Valid: true}, - VideoURL: sql.NullString{String: mappingJSON.VideoURL, Valid: true}, - CreatedAt: currentTime, - UpdatedAt: t.getTimeFromJSONTime(mappingJSON.UpdatedAt), - } - - time, err := utils.ParseDateStringAsTime(mappingJSON.Date) - if err == nil { - newScrapedItem.Date = &models.Date{Time: time} - } - - studio, err := sqb.FindByName(ctx, mappingJSON.Studio, false) - if err != nil { - logger.Errorf("[scraped sites] failed to fetch studio: %s", err.Error()) - } - if studio != nil { - newScrapedItem.StudioID = sql.NullInt64{Int64: int64(studio.ID), Valid: true} - } - - _, err = qb.Create(ctx, newScrapedItem) - if err != nil { - logger.Errorf("[scraped sites] <%s> failed to create: %s", newScrapedItem.Title.String, err.Error()) - } - } - - return nil - }); err != nil { - logger.Errorf("[scraped sites] import failed to commit: %s", err.Error()) - } - - logger.Info("[scraped sites] import complete") -} - func (t *ImportTask) ImportScenes(ctx context.Context) { logger.Info("[scenes] importing") @@ -799,21 +732,3 @@ func (t *ImportTask) ImportImages(ctx context.Context) { logger.Info("[images] import complete") } - -var currentLocation = time.Now().Location() - -func (t *ImportTask) getTimeFromJSONTime(jsonTime json.JSONTime) time.Time { - if currentLocation != nil { - if jsonTime.IsZero() { - return time.Now().In(currentLocation) - } else { - return jsonTime.Time.In(currentLocation) - } - } else { - if jsonTime.IsZero() { - return time.Now() - } else { - return jsonTime.Time - } - } -} diff --git a/internal/manager/task_stash_box_tag.go b/internal/manager/task_stash_box_tag.go index 3f80e301f..17511dc15 100644 --- a/internal/manager/task_stash_box_tag.go +++ b/internal/manager/task_stash_box_tag.go @@ -283,7 +283,10 @@ func getDate(val *string) *models.Date { return nil } - ret := models.NewDate(*val) + ret, err := models.ParseDate(*val) + if err != nil { + return nil + } return &ret } diff --git a/pkg/gallery/export_test.go b/pkg/gallery/export_test.go index f4bb8ec9f..fcd90b9e9 100644 --- a/pkg/gallery/export_test.go +++ b/pkg/gallery/export_test.go @@ -28,13 +28,13 @@ const ( ) var ( - url = "url" - title = "title" - date = "2001-01-01" - dateObj = models.NewDate(date) - rating = 5 - organized = true - details = "details" + url = "url" + title = "title" + date = "2001-01-01" + dateObj, _ = models.ParseDate(date) + rating = 5 + organized = true + details = "details" ) const ( diff --git a/pkg/gallery/import.go b/pkg/gallery/import.go index 4e64bacd1..ea78aee50 100644 --- a/pkg/gallery/import.go +++ b/pkg/gallery/import.go @@ -71,8 +71,10 @@ func (i *Importer) galleryJSONToGallery(galleryJSON jsonschema.Gallery) models.G newGallery.URL = galleryJSON.URL } if galleryJSON.Date != "" { - d := models.NewDate(galleryJSON.Date) - newGallery.Date = &d + d, err := models.ParseDate(galleryJSON.Date) + if err == nil { + newGallery.Date = &d + } } if galleryJSON.Rating != 0 { newGallery.Rating = &galleryJSON.Rating diff --git a/pkg/image/export_test.go b/pkg/image/export_test.go index 0c78746ea..4c46aae95 100644 --- a/pkg/image/export_test.go +++ b/pkg/image/export_test.go @@ -23,13 +23,13 @@ const ( ) var ( - title = "title" - rating = 5 - url = "http://a.com" - date = "2001-01-01" - dateObj = models.NewDate(date) - organized = true - ocounter = 2 + title = "title" + rating = 5 + url = "http://a.com" + date = "2001-01-01" + dateObj, _ = models.ParseDate(date) + organized = true + ocounter = 2 ) const ( diff --git a/pkg/image/import.go b/pkg/image/import.go index d3fb8d48a..b709a92ae 100644 --- a/pkg/image/import.go +++ b/pkg/image/import.go @@ -89,8 +89,10 @@ func (i *Importer) imageJSONToImage(imageJSON jsonschema.Image) models.Image { newImage.URL = imageJSON.URL } if imageJSON.Date != "" { - d := models.NewDate(imageJSON.Date) - newImage.Date = &d + d, err := models.ParseDate(imageJSON.Date) + if err == nil { + newImage.Date = &d + } } return newImage diff --git a/pkg/models/date.go b/pkg/models/date.go index 5fbb8f5bf..c88aeee35 100644 --- a/pkg/models/date.go +++ b/pkg/models/date.go @@ -1,6 +1,10 @@ package models -import "time" +import ( + "time" + + "github.com/stashapp/stash/pkg/utils" +) // Date wraps a time.Time with a format of "YYYY-MM-DD" type Date struct { @@ -13,7 +17,11 @@ func (d Date) String() string { return d.Format(dateFormat) } -func NewDate(s string) Date { - t, _ := time.Parse(dateFormat, s) - return Date{t} +// ParseDate uses utils.ParseDateStringAsTime to parse a string into a date. +func ParseDate(s string) (Date, error) { + ret, err := utils.ParseDateStringAsTime(s) + if err != nil { + return Date{}, err + } + return Date{Time: ret}, nil } diff --git a/pkg/models/errors.go b/pkg/models/errors.go index 3af2ff84c..3cde92431 100644 --- a/pkg/models/errors.go +++ b/pkg/models/errors.go @@ -8,4 +8,6 @@ var ( // ErrConversion signifies conversion errors ErrConversion = errors.New("conversion error") + + ErrScraperSource = errors.New("invalid ScraperSource") ) diff --git a/pkg/models/jsonschema/scraped.go b/pkg/models/jsonschema/scraped.go deleted file mode 100644 index c6444a484..000000000 --- a/pkg/models/jsonschema/scraped.go +++ /dev/null @@ -1,49 +0,0 @@ -package jsonschema - -import ( - "fmt" - "os" - - jsoniter "github.com/json-iterator/go" - "github.com/stashapp/stash/pkg/models/json" -) - -type ScrapedItem struct { - Title string `json:"title,omitempty"` - Description string `json:"description,omitempty"` - URL string `json:"url,omitempty"` - Date string `json:"date,omitempty"` - Rating string `json:"rating,omitempty"` - Tags string `json:"tags,omitempty"` - Models string `json:"models,omitempty"` - Episode int `json:"episode,omitempty"` - GalleryFilename string `json:"gallery_filename,omitempty"` - GalleryURL string `json:"gallery_url,omitempty"` - VideoFilename string `json:"video_filename,omitempty"` - VideoURL string `json:"video_url,omitempty"` - Studio string `json:"studio,omitempty"` - UpdatedAt json.JSONTime `json:"updated_at,omitempty"` -} - -func LoadScrapedFile(filePath string) ([]ScrapedItem, error) { - var scraped []ScrapedItem - file, err := os.Open(filePath) - if err != nil { - return nil, err - } - defer file.Close() - var json = jsoniter.ConfigCompatibleWithStandardLibrary - jsonParser := json.NewDecoder(file) - err = jsonParser.Decode(&scraped) - if err != nil { - return nil, err - } - return scraped, nil -} - -func SaveScrapedFile(filePath string, scrapedItems []ScrapedItem) error { - if scrapedItems == nil { - return fmt.Errorf("scraped items must not be nil") - } - return marshalToFile(filePath, scrapedItems) -} diff --git a/pkg/models/mocks/ScrapedItemReaderWriter.go b/pkg/models/mocks/ScrapedItemReaderWriter.go deleted file mode 100644 index 7157ab855..000000000 --- a/pkg/models/mocks/ScrapedItemReaderWriter.go +++ /dev/null @@ -1,61 +0,0 @@ -// Code generated by mockery v2.10.0. DO NOT EDIT. - -package mocks - -import ( - context "context" - - models "github.com/stashapp/stash/pkg/models" - mock "github.com/stretchr/testify/mock" -) - -// ScrapedItemReaderWriter is an autogenerated mock type for the ScrapedItemReaderWriter type -type ScrapedItemReaderWriter struct { - mock.Mock -} - -// All provides a mock function with given fields: ctx -func (_m *ScrapedItemReaderWriter) All(ctx context.Context) ([]*models.ScrapedItem, error) { - ret := _m.Called(ctx) - - var r0 []*models.ScrapedItem - if rf, ok := ret.Get(0).(func(context.Context) []*models.ScrapedItem); ok { - r0 = rf(ctx) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]*models.ScrapedItem) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(context.Context) error); ok { - r1 = rf(ctx) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// Create provides a mock function with given fields: ctx, newObject -func (_m *ScrapedItemReaderWriter) Create(ctx context.Context, newObject models.ScrapedItem) (*models.ScrapedItem, error) { - ret := _m.Called(ctx, newObject) - - var r0 *models.ScrapedItem - if rf, ok := ret.Get(0).(func(context.Context, models.ScrapedItem) *models.ScrapedItem); ok { - r0 = rf(ctx, newObject) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(*models.ScrapedItem) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(context.Context, models.ScrapedItem) error); ok { - r1 = rf(ctx, newObject) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} diff --git a/pkg/models/mocks/transaction.go b/pkg/models/mocks/transaction.go index 8ce7176b2..e7a0163d4 100644 --- a/pkg/models/mocks/transaction.go +++ b/pkg/models/mocks/transaction.go @@ -52,7 +52,6 @@ func NewTxnRepository() models.Repository { Performer: &PerformerReaderWriter{}, Scene: &SceneReaderWriter{}, SceneMarker: &SceneMarkerReaderWriter{}, - ScrapedItem: &ScrapedItemReaderWriter{}, Studio: &StudioReaderWriter{}, Tag: &TagReaderWriter{}, SavedFilter: &SavedFilterReaderWriter{}, diff --git a/pkg/models/model_movie.go b/pkg/models/model_movie.go index e4279c750..cf7f997d8 100644 --- a/pkg/models/model_movie.go +++ b/pkg/models/model_movie.go @@ -2,13 +2,10 @@ package models import ( "time" - - "github.com/stashapp/stash/pkg/hash/md5" ) type Movie struct { ID int `json:"id"` - Checksum string `json:"checksum"` Name string `json:"name"` Aliases string `json:"aliases"` Duration *int `json:"duration"` @@ -24,7 +21,6 @@ type Movie struct { } type MoviePartial struct { - Checksum OptionalString Name OptionalString Aliases OptionalString Duration OptionalInt @@ -44,7 +40,6 @@ var DefaultMovieImage = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAGQAAABkC func NewMovie(name string) *Movie { currentTime := time.Now() return &Movie{ - Checksum: md5.FromString(name), Name: name, CreatedAt: currentTime, UpdatedAt: currentTime, diff --git a/pkg/models/model_scene_test.go b/pkg/models/model_scene_test.go index e3126acae..d47e86c7f 100644 --- a/pkg/models/model_scene_test.go +++ b/pkg/models/model_scene_test.go @@ -25,7 +25,7 @@ func TestScenePartial_UpdateInput(t *testing.T) { studioIDStr = "2" ) - dateObj := NewDate(date) + dateObj, _ := ParseDate(date) tests := []struct { name string diff --git a/pkg/models/model_scraped_item.go b/pkg/models/model_scraped_item.go index c6d35b8d8..2b354f0c8 100644 --- a/pkg/models/model_scraped_item.go +++ b/pkg/models/model_scraped_item.go @@ -1,10 +1,5 @@ package models -import ( - "database/sql" - "time" -) - type ScrapedStudio struct { // Set if studio matched StoredID *string `json:"stored_id"` @@ -79,34 +74,3 @@ type ScrapedMovie struct { } func (ScrapedMovie) IsScrapedContent() {} - -type ScrapedItem struct { - ID int `db:"id" json:"id"` - Title sql.NullString `db:"title" json:"title"` - Code sql.NullString `db:"code" json:"code"` - Description sql.NullString `db:"description" json:"description"` - Director sql.NullString `db:"director" json:"director"` - URL sql.NullString `db:"url" json:"url"` - Date *Date `db:"date" json:"date"` - Rating sql.NullString `db:"rating" json:"rating"` - Tags sql.NullString `db:"tags" json:"tags"` - Models sql.NullString `db:"models" json:"models"` - Episode sql.NullInt64 `db:"episode" json:"episode"` - GalleryFilename sql.NullString `db:"gallery_filename" json:"gallery_filename"` - GalleryURL sql.NullString `db:"gallery_url" json:"gallery_url"` - VideoFilename sql.NullString `db:"video_filename" json:"video_filename"` - VideoURL sql.NullString `db:"video_url" json:"video_url"` - StudioID sql.NullInt64 `db:"studio_id,omitempty" json:"studio_id"` - CreatedAt time.Time `db:"created_at" json:"created_at"` - UpdatedAt time.Time `db:"updated_at" json:"updated_at"` -} - -type ScrapedItems []*ScrapedItem - -func (s *ScrapedItems) Append(o interface{}) { - *s = append(*s, o.(*ScrapedItem)) -} - -func (s *ScrapedItems) New() interface{} { - return &ScrapedItem{} -} diff --git a/pkg/models/model_studio.go b/pkg/models/model_studio.go index f62fe2d8a..a20cf4d75 100644 --- a/pkg/models/model_studio.go +++ b/pkg/models/model_studio.go @@ -2,13 +2,10 @@ package models import ( "time" - - "github.com/stashapp/stash/pkg/hash/md5" ) type Studio struct { ID int `json:"id"` - Checksum string `json:"checksum"` Name string `json:"name"` URL string `json:"url"` ParentID *int `json:"parent_id"` @@ -21,7 +18,6 @@ type Studio struct { } type StudioPartial struct { - Checksum OptionalString Name OptionalString URL OptionalString ParentID OptionalInt @@ -36,7 +32,6 @@ type StudioPartial struct { func NewStudio(name string) *Studio { currentTime := time.Now() return &Studio{ - Checksum: md5.FromString(name), Name: name, CreatedAt: currentTime, UpdatedAt: currentTime, diff --git a/pkg/models/repository.go b/pkg/models/repository.go index 898f7f25f..fe0e21dc0 100644 --- a/pkg/models/repository.go +++ b/pkg/models/repository.go @@ -23,7 +23,6 @@ type Repository struct { Performer PerformerReaderWriter Scene SceneReaderWriter SceneMarker SceneMarkerReaderWriter - ScrapedItem ScrapedItemReaderWriter Studio StudioReaderWriter Tag TagReaderWriter SavedFilter SavedFilterReaderWriter diff --git a/pkg/models/scraped.go b/pkg/models/scraped.go deleted file mode 100644 index be424147b..000000000 --- a/pkg/models/scraped.go +++ /dev/null @@ -1,21 +0,0 @@ -package models - -import ( - "context" - "errors" -) - -var ErrScraperSource = errors.New("invalid ScraperSource") - -type ScrapedItemReader interface { - All(ctx context.Context) ([]*ScrapedItem, error) -} - -type ScrapedItemWriter interface { - Create(ctx context.Context, newObject ScrapedItem) (*ScrapedItem, error) -} - -type ScrapedItemReaderWriter interface { - ScrapedItemReader - ScrapedItemWriter -} diff --git a/pkg/movie/export_test.go b/pkg/movie/export_test.go index d43fe022a..2f037a758 100644 --- a/pkg/movie/export_test.go +++ b/pkg/movie/export_test.go @@ -32,13 +32,13 @@ const movieName = "testMovie" const movieAliases = "aliases" var ( - date = "2001-01-01" - dateObj = models.NewDate(date) - rating = 5 - duration = 100 - director = "director" - synopsis = "synopsis" - url = "url" + date = "2001-01-01" + dateObj, _ = models.ParseDate(date) + rating = 5 + duration = 100 + director = "director" + synopsis = "synopsis" + url = "url" ) const studioName = "studio" diff --git a/pkg/movie/import.go b/pkg/movie/import.go index ed404c738..c9ee3adc5 100644 --- a/pkg/movie/import.go +++ b/pkg/movie/import.go @@ -4,7 +4,6 @@ import ( "context" "fmt" - "github.com/stashapp/stash/pkg/hash/md5" "github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/models/jsonschema" "github.com/stashapp/stash/pkg/studio" @@ -58,10 +57,7 @@ func (i *Importer) PreImport(ctx context.Context) error { } func (i *Importer) movieJSONToMovie(movieJSON jsonschema.Movie) models.Movie { - checksum := md5.FromString(movieJSON.Name) - newMovie := models.Movie{ - Checksum: checksum, Name: movieJSON.Name, Aliases: movieJSON.Aliases, Director: movieJSON.Director, @@ -72,8 +68,10 @@ func (i *Importer) movieJSONToMovie(movieJSON jsonschema.Movie) models.Movie { } if movieJSON.Date != "" { - d := models.NewDate(movieJSON.Date) - newMovie.Date = &d + d, err := models.ParseDate(movieJSON.Date) + if err == nil { + newMovie.Date = &d + } } if movieJSON.Rating != 0 { newMovie.Rating = &movieJSON.Rating diff --git a/pkg/performer/export_test.go b/pkg/performer/export_test.go index c5965404a..d63a9e05e 100644 --- a/pkg/performer/export_test.go +++ b/pkg/performer/export_test.go @@ -64,8 +64,8 @@ var stashIDs = []models.StashID{ const image = "aW1hZ2VCeXRlcw==" -var birthDate = models.NewDate("2001-01-01") -var deathDate = models.NewDate("2021-02-02") +var birthDate, _ = models.ParseDate("2001-01-01") +var deathDate, _ = models.ParseDate("2021-02-02") var ( createTime = time.Date(2001, 01, 01, 0, 0, 0, 0, time.Local) diff --git a/pkg/performer/import.go b/pkg/performer/import.go index ede9f4daa..f84030a6e 100644 --- a/pkg/performer/import.go +++ b/pkg/performer/import.go @@ -223,22 +223,18 @@ func performerJSONToPerformer(performerJSON jsonschema.Performer) models.Perform } if performerJSON.Birthdate != "" { - d, err := utils.ParseDateStringAsTime(performerJSON.Birthdate) + date, err := models.ParseDate(performerJSON.Birthdate) if err == nil { - newPerformer.Birthdate = &models.Date{ - Time: d, - } + newPerformer.Birthdate = &date } } if performerJSON.Rating != 0 { newPerformer.Rating = &performerJSON.Rating } if performerJSON.DeathDate != "" { - d, err := utils.ParseDateStringAsTime(performerJSON.DeathDate) + date, err := models.ParseDate(performerJSON.DeathDate) if err == nil { - newPerformer.DeathDate = &models.Date{ - Time: d, - } + newPerformer.DeathDate = &date } } diff --git a/pkg/performer/validate_test.go b/pkg/performer/validate_test.go index dbbb7fc98..86e69bc1d 100644 --- a/pkg/performer/validate_test.go +++ b/pkg/performer/validate_test.go @@ -16,8 +16,8 @@ func TestValidateDeathDate(t *testing.T) { date4 := "2004-01-01" empty := "" - md2 := models.NewDate(date2) - md3 := models.NewDate(date3) + md2, _ := models.ParseDate(date2) + md3, _ := models.ParseDate(date3) emptyPerformer := models.Performer{} invalidPerformer := models.Performer{ diff --git a/pkg/scene/export_test.go b/pkg/scene/export_test.go index 224a30a2d..85a63aa55 100644 --- a/pkg/scene/export_test.go +++ b/pkg/scene/export_test.go @@ -36,14 +36,14 @@ const ( ) var ( - url = "url" - title = "title" - date = "2001-01-01" - dateObj = models.NewDate(date) - rating = 5 - ocounter = 2 - organized = true - details = "details" + url = "url" + title = "title" + date = "2001-01-01" + dateObj, _ = models.ParseDate(date) + rating = 5 + ocounter = 2 + organized = true + details = "details" ) var ( diff --git a/pkg/scene/import.go b/pkg/scene/import.go index 1c00f8015..f7813511f 100644 --- a/pkg/scene/import.go +++ b/pkg/scene/import.go @@ -98,8 +98,10 @@ func (i *Importer) sceneJSONToScene(sceneJSON jsonschema.Scene) models.Scene { } if sceneJSON.Date != "" { - d := models.NewDate(sceneJSON.Date) - newScene.Date = &d + d, err := models.ParseDate(sceneJSON.Date) + if err == nil { + newScene.Date = &d + } } if sceneJSON.Rating != 0 { newScene.Rating = &sceneJSON.Rating diff --git a/pkg/sqlite/database.go b/pkg/sqlite/database.go index db5d72c21..12562b5c6 100644 --- a/pkg/sqlite/database.go +++ b/pkg/sqlite/database.go @@ -32,7 +32,7 @@ const ( dbConnTimeout = 30 ) -var appSchemaVersion uint = 47 +var appSchemaVersion uint = 48 //go:embed migrations/*.sql var migrationsBox embed.FS diff --git a/pkg/sqlite/date.go b/pkg/sqlite/date.go index 67eaf493e..ec41b612c 100644 --- a/pkg/sqlite/date.go +++ b/pkg/sqlite/date.go @@ -41,23 +41,13 @@ func (d *NullDate) Scan(value interface{}) error { return nil } - // Zero dates, which primarily come from empty strings in the DB, are treated as being invalid. - // TODO: add migration to remove invalid dates from the database and remove this. - // Ensure elsewhere that empty date inputs resolve to a null date and not a zero date. - // Zero dates shouldn't be invalid. - if d.Date.IsZero() { - d.Valid = false - } else { - d.Valid = true - } - + d.Valid = true return nil } // Value implements the driver Valuer interface. func (d NullDate) Value() (driver.Value, error) { - // TODO: don't ignore zero value, as above - if !d.Valid || d.Date.IsZero() { + if !d.Valid { return nil, nil } diff --git a/pkg/sqlite/gallery.go b/pkg/sqlite/gallery.go index 91c90fba8..b7ece948d 100644 --- a/pkg/sqlite/gallery.go +++ b/pkg/sqlite/gallery.go @@ -884,7 +884,7 @@ func galleryIsMissingCriterionHandler(qb *GalleryStore, isMissing *string) crite qb.performersRepository().join(f, "performers_join", "galleries.id") f.addWhere("performers_join.gallery_id IS NULL") case "date": - f.addWhere("galleries.date IS NULL OR galleries.date IS \"\" OR galleries.date IS \"0001-01-01\"") + f.addWhere("galleries.date IS NULL OR galleries.date IS \"\"") case "tags": qb.tagsRepository().join(f, "tags_join", "galleries.id") f.addWhere("tags_join.gallery_id IS NULL") @@ -1008,7 +1008,6 @@ func galleryPerformerAgeCriterionHandler(performerAge *models.IntCriterionInput) f.addWhere("galleries.date != '' AND performers.birthdate != ''") f.addWhere("galleries.date IS NOT NULL AND performers.birthdate IS NOT NULL") - f.addWhere("galleries.date != '0001-01-01' AND performers.birthdate != '0001-01-01'") ageCalc := "cast(strftime('%Y.%m%d', galleries.date) - strftime('%Y.%m%d', performers.birthdate) as int)" whereClause, args := getIntWhereClause(ageCalc, performerAge.Modifier, performerAge.Value, performerAge.Value2) diff --git a/pkg/sqlite/gallery_test.go b/pkg/sqlite/gallery_test.go index ce4320c3a..d33d5ba2a 100644 --- a/pkg/sqlite/gallery_test.go +++ b/pkg/sqlite/gallery_test.go @@ -62,7 +62,7 @@ func Test_galleryQueryBuilder_Create(t *testing.T) { galleryFile = makeFileWithID(fileIdxStartGalleryFiles) ) - date := models.NewDate("2003-02-01") + date, _ := models.ParseDate("2003-02-01") tests := []struct { name string @@ -211,7 +211,7 @@ func Test_galleryQueryBuilder_Update(t *testing.T) { updatedAt = time.Date(2001, 1, 1, 0, 0, 0, 0, time.UTC) ) - date := models.NewDate("2003-02-01") + date, _ := models.ParseDate("2003-02-01") tests := []struct { name string @@ -403,7 +403,7 @@ func Test_galleryQueryBuilder_UpdatePartial(t *testing.T) { createdAt = time.Date(2001, 1, 1, 0, 0, 0, 0, time.UTC) updatedAt = time.Date(2001, 1, 1, 0, 0, 0, 0, time.UTC) - date = models.NewDate("2003-02-01") + date, _ = models.ParseDate("2003-02-01") ) tests := []struct { @@ -844,10 +844,6 @@ func makeGalleryWithID(index int) *models.Gallery { ret := makeGallery(index, includeScenes) ret.ID = galleryIDs[index] - if ret.Date != nil && ret.Date.IsZero() { - ret.Date = nil - } - ret.Files = models.NewRelatedFiles([]file.File{makeGalleryFile(index)}) return ret @@ -1932,12 +1928,12 @@ func TestGalleryQueryIsMissingDate(t *testing.T) { galleries := queryGallery(ctx, t, sqb, &galleryFilter, nil) - // three in four scenes have no date - assert.Len(t, galleries, int(math.Ceil(float64(totalGalleries)/4*3))) + // one in four galleries have no date + assert.Len(t, galleries, int(math.Ceil(float64(totalGalleries)/4))) - // ensure date is null, empty or "0001-01-01" + // ensure date is null for _, g := range galleries { - assert.True(t, g.Date == nil || g.Date.Time == time.Time{}) + assert.Nil(t, g.Date) } return nil diff --git a/pkg/sqlite/image_test.go b/pkg/sqlite/image_test.go index 6d9076ff1..4f3ebcc22 100644 --- a/pkg/sqlite/image_test.go +++ b/pkg/sqlite/image_test.go @@ -57,7 +57,7 @@ func Test_imageQueryBuilder_Create(t *testing.T) { rating = 60 ocounter = 5 url = "url" - date = models.NewDate("2003-02-01") + date, _ = models.ParseDate("2003-02-01") createdAt = time.Date(2001, 1, 1, 0, 0, 0, 0, time.UTC) updatedAt = time.Date(2001, 1, 1, 0, 0, 0, 0, time.UTC) @@ -216,7 +216,7 @@ func Test_imageQueryBuilder_Update(t *testing.T) { title = "title" rating = 60 url = "url" - date = models.NewDate("2003-02-01") + date, _ = models.ParseDate("2003-02-01") ocounter = 5 createdAt = time.Date(2001, 1, 1, 0, 0, 0, 0, time.UTC) updatedAt = time.Date(2001, 1, 1, 0, 0, 0, 0, time.UTC) @@ -396,7 +396,7 @@ func Test_imageQueryBuilder_UpdatePartial(t *testing.T) { title = "title" rating = 60 url = "url" - date = models.NewDate("2003-02-01") + date, _ = models.ParseDate("2003-02-01") ocounter = 5 createdAt = time.Date(2001, 1, 1, 0, 0, 0, 0, time.UTC) updatedAt = time.Date(2001, 1, 1, 0, 0, 0, 0, time.UTC) @@ -965,10 +965,6 @@ func makeImageWithID(index int) *models.Image { ret := makeImage(index) ret.ID = imageIDs[index] - if ret.Date != nil && ret.Date.IsZero() { - ret.Date = nil - } - ret.Files = models.NewRelatedFiles([]file.File{makeImageFile(index)}) return ret @@ -2877,7 +2873,7 @@ func TestImageQuerySorting(t *testing.T) { "date", models.SortDirectionEnumDesc, imageIdxWithTwoGalleries, - imageIdxWithPerformerParentTag, + imageIdxWithGrandChildStudio, }, } diff --git a/pkg/sqlite/migrations/48_cleanup.up.sql b/pkg/sqlite/migrations/48_cleanup.up.sql new file mode 100644 index 000000000..cbe8d8cb6 --- /dev/null +++ b/pkg/sqlite/migrations/48_cleanup.up.sql @@ -0,0 +1,91 @@ +PRAGMA foreign_keys=OFF; + +-- Cleanup old invalid dates +UPDATE `scenes` SET `date` = NULL WHERE `date` = '0001-01-01' OR `date` = ''; +UPDATE `galleries` SET `date` = NULL WHERE `date` = '0001-01-01' OR `date` = ''; +UPDATE `performers` SET `birthdate` = NULL WHERE `birthdate` = '0001-01-01' OR `birthdate` = ''; +UPDATE `performers` SET `death_date` = NULL WHERE `death_date` = '0001-01-01' OR `death_date` = ''; + +-- Delete scene markers with missing scenes +DELETE FROM `scene_markers` WHERE `scene_id` IS NULL; + +-- make scene_id not null +DROP INDEX `index_scene_markers_on_scene_id`; +DROP INDEX `index_scene_markers_on_primary_tag_id`; + +CREATE TABLE `scene_markers_new` ( + `id` INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + `title` VARCHAR(255) NOT NULL, + `seconds` FLOAT NOT NULL, + `primary_tag_id` INTEGER NOT NULL, + `scene_id` INTEGER NOT NULL, + `created_at` DATETIME NOT NULL, + `updated_at` DATETIME NOT NULL, + FOREIGN KEY(`primary_tag_id`) REFERENCES `tags`(`id`), + FOREIGN KEY(`scene_id`) REFERENCES `scenes`(`id`) +); +INSERT INTO `scene_markers_new` SELECT * FROM `scene_markers`; + +DROP TABLE `scene_markers`; +ALTER TABLE `scene_markers_new` RENAME TO `scene_markers`; + +CREATE INDEX `index_scene_markers_on_primary_tag_id` ON `scene_markers`(`primary_tag_id`); +CREATE INDEX `index_scene_markers_on_scene_id` ON `scene_markers`(`scene_id`); + +-- drop unused scraped items table +DROP TABLE IF EXISTS `scraped_items`; + +-- remove checksum from movies +DROP INDEX `movies_checksum_unique`; +DROP INDEX `movies_name_unique`; + +CREATE TABLE `movies_new` ( + `id` integer not null primary key autoincrement, + `name` varchar(255) not null, + `aliases` varchar(255), + `duration` integer, + `date` date, + `rating` tinyint, + `studio_id` integer REFERENCES `studios`(`id`) ON DELETE SET NULL, + `director` varchar(255), + `synopsis` text, + `url` varchar(255), + `created_at` datetime not null, + `updated_at` datetime not null, + `front_image_blob` varchar(255) REFERENCES `blobs`(`checksum`), + `back_image_blob` varchar(255) REFERENCES `blobs`(`checksum`) +); + +INSERT INTO `movies_new` SELECT `id`, `name`, `aliases`, `duration`, `date`, `rating`, `studio_id`, `director`, `synopsis`, `url`, `created_at`, `updated_at`, `front_image_blob`, `back_image_blob` FROM `movies`; + +DROP TABLE `movies`; +ALTER TABLE `movies_new` RENAME TO `movies`; + +CREATE UNIQUE INDEX `index_movies_on_name_unique` ON `movies`(`name`); +CREATE INDEX `index_movies_on_studio_id` on `movies` (`studio_id`); + +-- remove checksum from studios +DROP INDEX `index_studios_on_checksum`; +DROP INDEX `index_studios_on_name`; +DROP INDEX `studios_checksum_unique`; + +CREATE TABLE `studios_new` ( + `id` INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + `name` VARCHAR(255) NOT NULL, + `url` VARCHAR(255), + `parent_id` INTEGER DEFAULT NULL CHECK (`id` IS NOT `parent_id`) REFERENCES `studios`(`id`) ON DELETE SET NULL, + `created_at` DATETIME NOT NULL, + `updated_at` DATETIME NOT NULL, + `details` TEXT, + `rating` TINYINT, + `ignore_auto_tag` BOOLEAN NOT NULL DEFAULT FALSE, + `image_blob` VARCHAR(255) REFERENCES `blobs`(`checksum`) +); +INSERT INTO `studios_new` SELECT `id`, `name`, `url`, `parent_id`, `created_at`, `updated_at`, `details`, `rating`, `ignore_auto_tag`, `image_blob` FROM `studios`; + +DROP TABLE `studios`; +ALTER TABLE `studios_new` RENAME TO `studios`; + +CREATE UNIQUE INDEX `index_studios_on_name_unique` ON `studios`(`name`); + +PRAGMA foreign_keys=ON; \ No newline at end of file diff --git a/pkg/sqlite/migrations/48_premigrate.go b/pkg/sqlite/migrations/48_premigrate.go new file mode 100644 index 000000000..b16c2258f --- /dev/null +++ b/pkg/sqlite/migrations/48_premigrate.go @@ -0,0 +1,150 @@ +package migrations + +import ( + "context" + "fmt" + + "github.com/jmoiron/sqlx" + "github.com/stashapp/stash/pkg/logger" + "github.com/stashapp/stash/pkg/sqlite" +) + +func pre48(ctx context.Context, db *sqlx.DB) error { + logger.Info("Running pre-migration for schema version 48") + + m := schema48PreMigrator{ + migrator: migrator{ + db: db, + }, + } + + if err := m.validateScrapedItems(ctx); err != nil { + return err + } + + if err := m.fixStudioNames(ctx); err != nil { + return err + } + + return nil +} + +type schema48PreMigrator struct { + migrator +} + +func (m *schema48PreMigrator) validateScrapedItems(ctx context.Context) error { + var count int + + row := m.db.QueryRowx("SELECT COUNT(*) FROM scraped_items") + err := row.Scan(&count) + if err != nil { + return err + } + + if count == 0 { + return nil + } + + return fmt.Errorf("found %d row(s) in scraped_items table, cannot migrate", count) +} + +func (m *schema48PreMigrator) fixStudioNames(ctx context.Context) error { + // First remove NULL names + if err := m.withTxn(ctx, func(tx *sqlx.Tx) error { + _, err := m.db.Exec("UPDATE studios SET name = 'NULL' WHERE name IS NULL") + return err + }); err != nil { + return err + } + + // Then remove duplicate names + + dupes := make(map[string][]int) + + // collect names + if err := m.withTxn(ctx, func(tx *sqlx.Tx) error { + rows, err := m.db.Query("SELECT id, name FROM studios ORDER BY name, id") + if err != nil { + return err + } + defer rows.Close() + + first := true + var lastName string + + for rows.Next() { + var ( + id int + name string + ) + + err := rows.Scan(&id, &name) + if err != nil { + return err + } + + if first { + first = false + lastName = name + continue + } + + if lastName == name { + dupes[name] = append(dupes[name], id) + } else { + lastName = name + } + } + + return rows.Err() + }); err != nil { + return err + } + + // rename them + if err := m.withTxn(ctx, func(tx *sqlx.Tx) error { + for name, ids := range dupes { + i := 0 + for _, id := range ids { + var newName string + for j := 0; ; j++ { + i++ + newName = fmt.Sprintf("%s (%d)", name, i) + + var count int + + row := m.db.QueryRowx("SELECT COUNT(*) FROM studios WHERE name = ?", newName) + err := row.Scan(&count) + if err != nil { + return err + } + + if count == 0 { + break + } + + // try up to 100 times to find a unique name + if j == 100 { + return fmt.Errorf("cannot make unique studio name for %s", name) + } + } + + logger.Info("Renaming duplicate studio id %d to %s", id, newName) + _, err := m.db.Exec("UPDATE studios SET name = ? WHERE id = ?", newName, id) + if err != nil { + return err + } + } + } + return nil + }); err != nil { + return err + } + + return nil +} + +func init() { + sqlite.RegisterPreMigration(48, pre48) +} diff --git a/pkg/sqlite/movies.go b/pkg/sqlite/movies.go index b899c2859..adc7fe29e 100644 --- a/pkg/sqlite/movies.go +++ b/pkg/sqlite/movies.go @@ -26,7 +26,6 @@ const ( type movieRow struct { ID int `db:"id" goqu:"skipinsert"` - Checksum string `db:"checksum"` Name zero.String `db:"name"` Aliases zero.String `db:"aliases"` Duration null.Int `db:"duration"` @@ -47,7 +46,6 @@ type movieRow struct { func (r *movieRow) fromMovie(o models.Movie) { r.ID = o.ID - r.Checksum = o.Checksum r.Name = zero.StringFrom(o.Name) r.Aliases = zero.StringFrom(o.Aliases) r.Duration = intFromPtr(o.Duration) @@ -64,7 +62,6 @@ func (r *movieRow) fromMovie(o models.Movie) { func (r *movieRow) resolve() *models.Movie { ret := &models.Movie{ ID: r.ID, - Checksum: r.Checksum, Name: r.Name.String, Aliases: r.Aliases.String, Duration: nullIntPtr(r.Duration), @@ -86,7 +83,6 @@ type movieRowRecord struct { } func (r *movieRowRecord) fromPartial(o models.MoviePartial) { - r.setString("checksum", o.Checksum) r.setNullString("name", o.Name) r.setNullString("aliases", o.Aliases) r.setNullInt("duration", o.Duration) diff --git a/pkg/sqlite/movies_test.go b/pkg/sqlite/movies_test.go index 050190625..ed0ef7242 100644 --- a/pkg/sqlite/movies_test.go +++ b/pkg/sqlite/movies_test.go @@ -12,7 +12,6 @@ import ( "github.com/stretchr/testify/assert" - "github.com/stashapp/stash/pkg/hash/md5" "github.com/stashapp/stash/pkg/models" ) @@ -293,7 +292,6 @@ func TestMovieUpdateFrontImage(t *testing.T) { const name = "TestMovieUpdateMovieImages" movie := models.Movie{ Name: name, - Checksum: md5.FromString(name), } err := qb.Create(ctx, &movie) if err != nil { @@ -314,7 +312,6 @@ func TestMovieUpdateBackImage(t *testing.T) { const name = "TestMovieUpdateMovieImages" movie := models.Movie{ Name: name, - Checksum: md5.FromString(name), } err := qb.Create(ctx, &movie) if err != nil { diff --git a/pkg/sqlite/performer.go b/pkg/sqlite/performer.go index dc2114298..cf6705e0c 100644 --- a/pkg/sqlite/performer.go +++ b/pkg/sqlite/performer.go @@ -766,7 +766,7 @@ func performerAgeFilterCriterionHandler(age *models.IntCriterionInput) criterion return func(ctx context.Context, f *filterBuilder) { if age != nil && age.Modifier.IsValid() { clause, args := getIntCriterionWhereClause( - "cast(strftime('%Y.%m%d',CASE WHEN performers.death_date IS NULL OR performers.death_date = '0001-01-01' OR performers.death_date = '' THEN 'now' ELSE performers.death_date END) - strftime('%Y.%m%d', performers.birthdate) as int)", + "cast(IFNULL(strftime('%Y.%m%d', performers.death_date), strftime('%Y.%m%d', 'now')) - strftime('%Y.%m%d', performers.birthdate) as int)", *age, ) f.addWhere(clause, args...) diff --git a/pkg/sqlite/performer_test.go b/pkg/sqlite/performer_test.go index 5abed4876..6752f199f 100644 --- a/pkg/sqlite/performer_test.go +++ b/pkg/sqlite/performer_test.go @@ -69,8 +69,8 @@ func Test_PerformerStore_Create(t *testing.T) { createdAt = time.Date(2001, 1, 1, 0, 0, 0, 0, time.UTC) updatedAt = time.Date(2001, 1, 1, 0, 0, 0, 0, time.UTC) - birthdate = models.NewDate("2003-02-01") - deathdate = models.NewDate("2023-02-01") + birthdate, _ = models.ParseDate("2003-02-01") + deathdate, _ = models.ParseDate("2023-02-01") ) tests := []struct { @@ -217,8 +217,8 @@ func Test_PerformerStore_Update(t *testing.T) { createdAt = time.Date(2001, 1, 1, 0, 0, 0, 0, time.UTC) updatedAt = time.Date(2001, 1, 1, 0, 0, 0, 0, time.UTC) - birthdate = models.NewDate("2003-02-01") - deathdate = models.NewDate("2023-02-01") + birthdate, _ = models.ParseDate("2003-02-01") + deathdate, _ = models.ParseDate("2023-02-01") ) tests := []struct { @@ -400,8 +400,8 @@ func Test_PerformerStore_UpdatePartial(t *testing.T) { createdAt = time.Date(2001, 1, 1, 0, 0, 0, 0, time.UTC) updatedAt = time.Date(2001, 1, 1, 0, 0, 0, 0, time.UTC) - birthdate = models.NewDate("2003-02-01") - deathdate = models.NewDate("2023-02-01") + birthdate, _ = models.ParseDate("2003-02-01") + deathdate, _ = models.ParseDate("2023-02-01") ) tests := []struct { diff --git a/pkg/sqlite/repository.go b/pkg/sqlite/repository.go index c3b1d74aa..15c9d4f7b 100644 --- a/pkg/sqlite/repository.go +++ b/pkg/sqlite/repository.go @@ -5,7 +5,6 @@ import ( "database/sql" "errors" "fmt" - "reflect" "strings" "github.com/jmoiron/sqlx" @@ -27,65 +26,11 @@ type repository struct { idColumn string } -func (r *repository) getByID(ctx context.Context, id int, dest interface{}) error { - stmt := fmt.Sprintf("SELECT * FROM %s WHERE %s = ? LIMIT 1", r.tableName, r.idColumn) - return r.tx.Get(ctx, dest, stmt, id) -} - func (r *repository) getAll(ctx context.Context, id int, f func(rows *sqlx.Rows) error) error { stmt := fmt.Sprintf("SELECT * FROM %s WHERE %s = ?", r.tableName, r.idColumn) return r.queryFunc(ctx, stmt, []interface{}{id}, false, f) } -func (r *repository) insert(ctx context.Context, obj interface{}) (sql.Result, error) { - stmt := fmt.Sprintf("INSERT INTO %s (%s) VALUES (%s)", r.tableName, listKeys(obj, false), listKeys(obj, true)) - return r.tx.NamedExec(ctx, stmt, obj) -} - -func (r *repository) insertObject(ctx context.Context, obj interface{}, out interface{}) error { - result, err := r.insert(ctx, obj) - if err != nil { - return err - } - id, err := result.LastInsertId() - if err != nil { - return err - } - return r.getByID(ctx, int(id), out) -} - -func (r *repository) update(ctx context.Context, id int, obj interface{}, partial bool) error { - exists, err := r.exists(ctx, id) - if err != nil { - return err - } - - if !exists { - return fmt.Errorf("%s %d does not exist in %s", r.idColumn, id, r.tableName) - } - - stmt := fmt.Sprintf("UPDATE %s SET %s WHERE %s.%s = :id", r.tableName, updateSet(obj, partial), r.tableName, r.idColumn) - _, err = r.tx.NamedExec(ctx, stmt, obj) - - return err -} - -// func (r *repository) updateMap(ctx context.Context, id int, m map[string]interface{}) error { -// exists, err := r.exists(ctx, id) -// if err != nil { -// return err -// } - -// if !exists { -// return fmt.Errorf("%s %d does not exist in %s", r.idColumn, id, r.tableName) -// } - -// stmt := fmt.Sprintf("UPDATE %s SET %s WHERE %s.%s = :id", r.tableName, updateSetMap(m), r.tableName, r.idColumn) -// _, err = r.tx.NamedExec(ctx, stmt, m) - -// return err -// } - func (r *repository) destroyExisting(ctx context.Context, ids []int) error { for _, id := range ids { exists, err := r.exists(ctx, id) @@ -597,53 +542,3 @@ func (r *filesRepository) get(ctx context.Context, id int) ([]file.ID, error) { return ret, nil } - -func listKeys(i interface{}, addPrefix bool) string { - var query []string - v := reflect.ValueOf(i) - for i := 0; i < v.NumField(); i++ { - // Get key for struct tag - rawKey := v.Type().Field(i).Tag.Get("db") - key := strings.Split(rawKey, ",")[0] - if key == "id" { - continue - } - if addPrefix { - key = ":" + key - } - query = append(query, key) - } - return strings.Join(query, ", ") -} - -func updateSet(i interface{}, partial bool) string { - var query []string - v := reflect.ValueOf(i) - for i := 0; i < v.NumField(); i++ { - // Get key for struct tag - rawKey := v.Type().Field(i).Tag.Get("db") - key := strings.Split(rawKey, ",")[0] - if key == "id" { - continue - } - - add := true - if partial { - reflectValue := reflect.ValueOf(v.Field(i).Interface()) - add = !reflectValue.IsNil() - } - - if add { - query = append(query, fmt.Sprintf("%s=:%s", key, key)) - } - } - return strings.Join(query, ", ") -} - -// func updateSetMap(m map[string]interface{}) string { -// var query []string -// for k := range m { -// query = append(query, fmt.Sprintf("%s=:%s", k, k)) -// } -// return strings.Join(query, ", ") -// } diff --git a/pkg/sqlite/scene.go b/pkg/sqlite/scene.go index ae8bca467..8fc37937b 100644 --- a/pkg/sqlite/scene.go +++ b/pkg/sqlite/scene.go @@ -1304,7 +1304,7 @@ func sceneIsMissingCriterionHandler(qb *SceneStore, isMissing *string) criterion qb.performersRepository().join(f, "performers_join", "scenes.id") f.addWhere("performers_join.scene_id IS NULL") case "date": - f.addWhere(`scenes.date IS NULL OR scenes.date IS "" OR scenes.date IS "0001-01-01"`) + f.addWhere(`scenes.date IS NULL OR scenes.date IS ""`) case "tags": qb.tagsRepository().join(f, "tags_join", "scenes.id") f.addWhere("tags_join.scene_id IS NULL") @@ -1441,7 +1441,6 @@ func scenePerformerAgeCriterionHandler(performerAge *models.IntCriterionInput) c f.addWhere("scenes.date != '' AND performers.birthdate != ''") f.addWhere("scenes.date IS NOT NULL AND performers.birthdate IS NOT NULL") - f.addWhere("scenes.date != '0001-01-01' AND performers.birthdate != '0001-01-01'") ageCalc := "cast(strftime('%Y.%m%d', scenes.date) - strftime('%Y.%m%d', performers.birthdate) as int)" whereClause, args := getIntWhereClause(ageCalc, performerAge.Modifier, performerAge.Value, performerAge.Value2) diff --git a/pkg/sqlite/scene_marker.go b/pkg/sqlite/scene_marker.go index ab0be7117..23ce50cfa 100644 --- a/pkg/sqlite/scene_marker.go +++ b/pkg/sqlite/scene_marker.go @@ -9,7 +9,6 @@ import ( "github.com/doug-martin/goqu/v9" "github.com/doug-martin/goqu/v9/exp" "github.com/jmoiron/sqlx" - "gopkg.in/guregu/null.v4/zero" "github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/sliceutil/intslice" @@ -29,7 +28,7 @@ type sceneMarkerRow struct { Title string `db:"title"` Seconds float64 `db:"seconds"` PrimaryTagID int `db:"primary_tag_id"` - SceneID zero.Int `db:"scene_id,omitempty"` // TODO: make schema non-nullable + SceneID int `db:"scene_id"` CreatedAt Timestamp `db:"created_at"` UpdatedAt Timestamp `db:"updated_at"` } @@ -39,7 +38,7 @@ func (r *sceneMarkerRow) fromSceneMarker(o models.SceneMarker) { r.Title = o.Title r.Seconds = o.Seconds r.PrimaryTagID = o.PrimaryTagID - r.SceneID = zero.IntFrom(int64(o.SceneID)) + r.SceneID = o.SceneID r.CreatedAt = Timestamp{Timestamp: o.CreatedAt} r.UpdatedAt = Timestamp{Timestamp: o.UpdatedAt} } @@ -50,7 +49,7 @@ func (r *sceneMarkerRow) resolve() *models.SceneMarker { Title: r.Title, Seconds: r.Seconds, PrimaryTagID: r.PrimaryTagID, - SceneID: int(r.SceneID.Int64), + SceneID: r.SceneID, CreatedAt: r.CreatedAt.Timestamp, UpdatedAt: r.UpdatedAt.Timestamp, } diff --git a/pkg/sqlite/scene_test.go b/pkg/sqlite/scene_test.go index 11085a0c1..8ab34a112 100644 --- a/pkg/sqlite/scene_test.go +++ b/pkg/sqlite/scene_test.go @@ -97,7 +97,7 @@ func Test_sceneQueryBuilder_Create(t *testing.T) { stashID1 = "stashid1" stashID2 = "stashid2" - date = models.NewDate("2003-02-01") + date, _ = models.ParseDate("2003-02-01") videoFile = makeFileWithID(fileIdxStartVideoFiles) ) @@ -336,7 +336,7 @@ func Test_sceneQueryBuilder_Update(t *testing.T) { stashID1 = "stashid1" stashID2 = "stashid2" - date = models.NewDate("2003-02-01") + date, _ = models.ParseDate("2003-02-01") ) tests := []struct { @@ -552,7 +552,7 @@ func Test_sceneQueryBuilder_UpdatePartial(t *testing.T) { stashID1 = "stashid1" stashID2 = "stashid2" - date = models.NewDate("2003-02-01") + date, _ = models.ParseDate("2003-02-01") ) tests := []struct { @@ -1460,10 +1460,6 @@ func makeSceneWithID(index int) *models.Scene { ret := makeScene(index) ret.ID = sceneIDs[index] - if ret.Date != nil && ret.Date.IsZero() { - ret.Date = nil - } - ret.Files = models.NewRelatedVideoFiles([]*file.VideoFile{makeSceneFile(index)}) return ret @@ -3243,12 +3239,12 @@ func TestSceneQueryIsMissingDate(t *testing.T) { scenes := queryScene(ctx, t, sqb, &sceneFilter, nil) - // three in four scenes have no date - assert.Len(t, scenes, int(math.Ceil(float64(totalScenes)/4*3))) + // one in four scenes have no date + assert.Len(t, scenes, int(math.Ceil(float64(totalScenes)/4))) - // ensure date is null, empty or "0001-01-01" + // ensure date is null for _, scene := range scenes { - assert.True(t, scene.Date == nil || scene.Date.Time == time.Time{}) + assert.Nil(t, scene.Date) } return nil @@ -3293,7 +3289,7 @@ func TestSceneQueryIsMissingRating(t *testing.T) { assert.True(t, len(scenes) > 0) - // ensure date is null, empty or "0001-01-01" + // ensure rating is null for _, scene := range scenes { assert.Nil(t, scene.Rating) } diff --git a/pkg/sqlite/scraped_item.go b/pkg/sqlite/scraped_item.go deleted file mode 100644 index 1b8216dab..000000000 --- a/pkg/sqlite/scraped_item.go +++ /dev/null @@ -1,81 +0,0 @@ -package sqlite - -import ( - "context" - "database/sql" - "errors" - - "github.com/stashapp/stash/pkg/models" -) - -const scrapedItemTable = "scraped_items" - -type scrapedItemQueryBuilder struct { - repository -} - -var ScrapedItemReaderWriter = &scrapedItemQueryBuilder{ - repository{ - tableName: scrapedItemTable, - idColumn: idColumn, - }, -} - -func (qb *scrapedItemQueryBuilder) Create(ctx context.Context, newObject models.ScrapedItem) (*models.ScrapedItem, error) { - var ret models.ScrapedItem - if err := qb.insertObject(ctx, newObject, &ret); err != nil { - return nil, err - } - - return &ret, nil -} - -func (qb *scrapedItemQueryBuilder) Update(ctx context.Context, updatedObject models.ScrapedItem) (*models.ScrapedItem, error) { - const partial = false - if err := qb.update(ctx, updatedObject.ID, updatedObject, partial); err != nil { - return nil, err - } - - return qb.find(ctx, updatedObject.ID) -} - -func (qb *scrapedItemQueryBuilder) Find(ctx context.Context, id int) (*models.ScrapedItem, error) { - return qb.find(ctx, id) -} - -func (qb *scrapedItemQueryBuilder) find(ctx context.Context, id int) (*models.ScrapedItem, error) { - var ret models.ScrapedItem - if err := qb.getByID(ctx, id, &ret); err != nil { - if errors.Is(err, sql.ErrNoRows) { - return nil, nil - } - return nil, err - } - return &ret, nil -} - -func (qb *scrapedItemQueryBuilder) All(ctx context.Context) ([]*models.ScrapedItem, error) { - return qb.queryScrapedItems(ctx, selectAll("scraped_items")+qb.getScrapedItemsSort(nil), nil) -} - -func (qb *scrapedItemQueryBuilder) getScrapedItemsSort(findFilter *models.FindFilterType) string { - var sort string - var direction string - if findFilter == nil { - sort = "id" // TODO studio_id and title - direction = "ASC" - } else { - sort = findFilter.GetSort("id") - direction = findFilter.GetDirection() - } - return getSort(sort, direction, "scraped_items") -} - -func (qb *scrapedItemQueryBuilder) queryScrapedItems(ctx context.Context, query string, args []interface{}) ([]*models.ScrapedItem, error) { - var ret models.ScrapedItems - if err := qb.query(ctx, query, args, &ret); err != nil { - return nil, err - } - - return []*models.ScrapedItem(ret), nil -} diff --git a/pkg/sqlite/setup_test.go b/pkg/sqlite/setup_test.go index d869a35ba..5a93ed2b5 100644 --- a/pkg/sqlite/setup_test.go +++ b/pkg/sqlite/setup_test.go @@ -15,7 +15,6 @@ import ( "time" "github.com/stashapp/stash/pkg/file" - "github.com/stashapp/stash/pkg/hash/md5" "github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/sliceutil/intslice" "github.com/stashapp/stash/pkg/sqlite" @@ -956,14 +955,14 @@ func getWidth(index int) int { } func getObjectDate(index int) *models.Date { - dates := []string{"null", "", "0001-01-01", "2001-02-03"} + dates := []string{"null", "2000-01-01", "0001-01-01", "2001-02-03"} date := dates[index%len(dates)] if date == "null" { return nil } - ret := models.NewDate(date) + ret, _ := models.ParseDate(date) return &ret } @@ -1313,7 +1312,6 @@ func createMovies(ctx context.Context, mqb models.MovieReaderWriter, n int, o in movie := models.Movie{ Name: name, URL: getMovieNullStringValue(index, urlField), - Checksum: md5.FromString(name), } err := mqb.Create(ctx, &movie) @@ -1578,7 +1576,6 @@ func getStudioNullStringValue(index int, field string) string { func createStudio(ctx context.Context, sqb models.StudioReaderWriter, name string, parentID *int) (*models.Studio, error) { studio := models.Studio{ Name: name, - Checksum: md5.FromString(name), } if parentID != nil { @@ -1621,7 +1618,6 @@ func createStudios(ctx context.Context, sqb models.StudioReaderWriter, n int, o name = getStudioStringValue(index, name) studio := models.Studio{ Name: name, - Checksum: md5.FromString(name), URL: getStudioNullStringValue(index, urlField), IgnoreAutoTag: getIgnoreAutoTag(i), } diff --git a/pkg/sqlite/sql.go b/pkg/sqlite/sql.go index 90b922520..599219141 100644 --- a/pkg/sqlite/sql.go +++ b/pkg/sqlite/sql.go @@ -249,9 +249,9 @@ func getDateWhereClause(column string, modifier models.CriterionModifier, value switch modifier { case models.CriterionModifierIsNull: - return fmt.Sprintf("(%s IS NULL OR %s = '' OR %s = '0001-01-01')", column, column, column), nil + return fmt.Sprintf("(%s IS NULL OR %s = '')", column, column), nil case models.CriterionModifierNotNull: - return fmt.Sprintf("(%s IS NOT NULL AND %s != '' AND %s != '0001-01-01')", column, column, column), nil + return fmt.Sprintf("(%s IS NOT NULL AND %s != '')", column, column), nil case models.CriterionModifierEquals: return fmt.Sprintf("%s = ?", column), args case models.CriterionModifierNotEquals: diff --git a/pkg/sqlite/studio.go b/pkg/sqlite/studio.go index 42bf42e03..cbb41ac63 100644 --- a/pkg/sqlite/studio.go +++ b/pkg/sqlite/studio.go @@ -28,7 +28,6 @@ const ( type studioRow struct { ID int `db:"id" goqu:"skipinsert"` - Checksum string `db:"checksum"` Name zero.String `db:"name"` URL zero.String `db:"url"` ParentID null.Int `db:"parent_id,omitempty"` @@ -45,7 +44,6 @@ type studioRow struct { func (r *studioRow) fromStudio(o models.Studio) { r.ID = o.ID - r.Checksum = o.Checksum r.Name = zero.StringFrom(o.Name) r.URL = zero.StringFrom(o.URL) r.ParentID = intFromPtr(o.ParentID) @@ -59,7 +57,6 @@ func (r *studioRow) fromStudio(o models.Studio) { func (r *studioRow) resolve() *models.Studio { ret := &models.Studio{ ID: r.ID, - Checksum: r.Checksum, Name: r.Name.String, URL: r.URL.String, ParentID: nullIntPtr(r.ParentID), @@ -78,7 +75,6 @@ type studioRowRecord struct { } func (r *studioRowRecord) fromPartial(o models.StudioPartial) { - r.setString("checksum", o.Checksum) r.setNullString("name", o.Name) r.setNullString("url", o.URL) r.setNullInt("parent_id", o.ParentID) @@ -173,13 +169,6 @@ func (qb *StudioStore) Destroy(ctx context.Context, id int) error { return err } - // TODO - set null on foreign key in scraped items - // remove studio from scraped items - _, err := qb.tx.Exec(ctx, "UPDATE scraped_items SET studio_id = null WHERE studio_id = ?", id) - if err != nil { - return err - } - return qb.destroyExisting(ctx, []int{id}) } diff --git a/pkg/sqlite/transaction.go b/pkg/sqlite/transaction.go index 797c26132..726b927e7 100644 --- a/pkg/sqlite/transaction.go +++ b/pkg/sqlite/transaction.go @@ -135,7 +135,6 @@ func (db *Database) TxnRepository() models.Repository { Performer: db.Performer, Scene: db.Scene, SceneMarker: db.SceneMarker, - ScrapedItem: ScrapedItemReaderWriter, Studio: db.Studio, Tag: db.Tag, SavedFilter: db.SavedFilter, diff --git a/pkg/studio/import.go b/pkg/studio/import.go index 0045f3ec5..bb29b6b2e 100644 --- a/pkg/studio/import.go +++ b/pkg/studio/import.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" - "github.com/stashapp/stash/pkg/hash/md5" "github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/models/jsonschema" "github.com/stashapp/stash/pkg/utils" @@ -31,10 +30,7 @@ type Importer struct { } func (i *Importer) PreImport(ctx context.Context) error { - checksum := md5.FromString(i.Input.Name) - i.studio = models.Studio{ - Checksum: checksum, Name: i.Input.Name, URL: i.Input.URL, Details: i.Input.Details, diff --git a/pkg/studio/import_test.go b/pkg/studio/import_test.go index a0ea0fddf..ef379529f 100644 --- a/pkg/studio/import_test.go +++ b/pkg/studio/import_test.go @@ -5,7 +5,6 @@ import ( "errors" "testing" - "github.com/stashapp/stash/pkg/hash/md5" "github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/models/jsonschema" "github.com/stashapp/stash/pkg/models/mocks" @@ -64,7 +63,6 @@ func TestImporterPreImport(t *testing.T) { assert.Nil(t, err) expectedStudio := createFullStudio(0, 0) expectedStudio.ParentID = nil - expectedStudio.Checksum = md5.FromString(studioName) assert.Equal(t, expectedStudio, i.studio) } diff --git a/pkg/utils/image.go b/pkg/utils/image.go index 20435e7eb..33edea19c 100644 --- a/pkg/utils/image.go +++ b/pkg/utils/image.go @@ -20,6 +20,10 @@ const base64RE = `^data:.+\/(.+);base64,(.*)$` // ProcessImageInput transforms an image string either from a base64 encoded // string, or from a URL, and returns the image as a byte slice func ProcessImageInput(ctx context.Context, imageInput string) ([]byte, error) { + if imageInput == "" { + return []byte{}, nil + } + regex := regexp.MustCompile(base64RE) if regex.MatchString(imageInput) { d, err := ProcessBase64Image(imageInput) diff --git a/ui/v2.5/src/docs/en/MigrationNotes/48.md b/ui/v2.5/src/docs/en/MigrationNotes/48.md new file mode 100644 index 000000000..d0ef6ac86 --- /dev/null +++ b/ui/v2.5/src/docs/en/MigrationNotes/48.md @@ -0,0 +1,3 @@ +This migration removes the unused `scraped_items` table from the database, which was only used in very old versions of Stash. For the vast majority of users, it should be empty, but if not, the migration will fail and restore the old database. If this happens, please manually edit the database and remove the table yourself, after making a copy of any contained data you'd like to keep. If you are not confident on how to do this, feel free to ask for assistance on the Discord server. + +This migration also enforces studio name uniqueness at the database level. Although no longer possible in recent versions, older versions of Stash allowed for different studios to have identical names. If your database has such duplicate names, the duplicates will have `" (1)"`, `" (2)"`, etc. appended to their names after this migration. diff --git a/ui/v2.5/src/docs/en/MigrationNotes/index.ts b/ui/v2.5/src/docs/en/MigrationNotes/index.ts index 6611333cf..f450e12a6 100644 --- a/ui/v2.5/src/docs/en/MigrationNotes/index.ts +++ b/ui/v2.5/src/docs/en/MigrationNotes/index.ts @@ -1,7 +1,9 @@ import migration32 from "./32.md"; import migration39 from "./39.md"; +import migration48 from "./48.md"; export const migrationNotes: Record = { 32: migration32, 39: migration39, + 48: migration48, };