Mirrors/stash
1
0
Fork 0
mirror of https://github.com/stashapp/stash.git synced 2025-12-17 04:14:39 +03:00
Code Issues Packages Projects Releases Wiki Activity

File storage rewrite (#2676)

Browse Source 
* Restructure data layer part 2 (#2599)
* Refactor and separate image model
* Refactor image query builder
* Handle relationships in image query builder
* Remove relationship management methods
* Refactor gallery model/query builder
* Add scenes to gallery model
* Convert scene model
* Refactor scene models
* Remove unused methods
* Add unit tests for gallery
* Add image tests
* Add scene tests
* Convert unnecessary scene value pointers to values
* Convert unnecessary pointer values to values
* Refactor scene partial
* Add scene partial tests
* Refactor ImagePartial
* Add image partial tests
* Refactor gallery partial update
* Add partial gallery update tests
* Use zero/null package for null values
* Add files and scan system
* Add sqlite implementation for files/folders
* Add unit tests for files/folders
* Image refactors
* Update image data layer
* Refactor gallery model and creation
* Refactor scene model
* Refactor scenes
* Don't set title from filename
* Allow galleries to freely add/remove images
* Add multiple scene file support to graphql and UI
* Add multiple file support for images in graphql/UI
* Add multiple file for galleries in graphql/UI
* Remove use of some deprecated fields
* Remove scene path usage
* Remove gallery path usage
* Remove path from image
* Move funscript to video file
* Refactor caption detection
* Migrate existing data
* Add post commit/rollback hook system
* Lint. Comment out import/export tests
* Add WithDatabase read only wrapper
* Prepend tasks to list
* Add 32 pre-migration
* Add warnings in release and migration notes
This commit is contained in:
WithoutPants
2022-07-13 16:30:54 +10:00
parent 30877c75fb
commit 5495d72849
359 changed files with 43690 additions and 16000 deletions
Download Patch File Download Diff File Expand all files Collapse all files

235
pkg/scene/import.go
View File

@@ -2,9 +2,7 @@ package scene
import (
"context"
"database/sql"
"fmt"
"strconv"
"strings"
"github.com/stashapp/stash/pkg/gallery"
@@ -21,8 +19,6 @@ import (
type FullCreatorUpdater interface {
CreatorUpdater
Updater
UpdateGalleries(ctx context.Context, sceneID int, galleryIDs []int) error
UpdateMovies(ctx context.Context, sceneID int, movies []models.MoviesScenes) error
}
type Importer struct {
@@ -39,10 +35,6 @@ type Importer struct {
ID int
scene models.Scene
galleries []*models.Gallery
performers []*models.Performer
movies []models.MoviesScenes
tags []*models.Tag
coverImageData []byte
}
@@ -82,68 +74,74 @@ func (i *Importer) PreImport(ctx context.Context) error {
func (i *Importer) sceneJSONToScene(sceneJSON jsonschema.Scene) models.Scene {
newScene := models.Scene{
Checksum: sql.NullString{String: sceneJSON.Checksum, Valid: sceneJSON.Checksum != ""},
OSHash: sql.NullString{String: sceneJSON.OSHash, Valid: sceneJSON.OSHash != ""},
Path: i.Path,
// Path: i.Path,
Title: sceneJSON.Title,
Details: sceneJSON.Details,
URL: sceneJSON.URL,
}
if sceneJSON.Phash != "" {
hash, err := strconv.ParseUint(sceneJSON.Phash, 16, 64)
newScene.Phash = sql.NullInt64{Int64: int64(hash), Valid: err == nil}
}
// if sceneJSON.Checksum != "" {
// newScene.Checksum = &sceneJSON.Checksum
// }
// if sceneJSON.OSHash != "" {
// newScene.OSHash = &sceneJSON.OSHash
// }
// if sceneJSON.Phash != "" {
// hash, err := strconv.ParseUint(sceneJSON.Phash, 16, 64)
// if err == nil {
// v := int64(hash)
// newScene.Phash = &v
// }
// }
if sceneJSON.Title != "" {
newScene.Title = sql.NullString{String: sceneJSON.Title, Valid: true}
}
if sceneJSON.Details != "" {
newScene.Details = sql.NullString{String: sceneJSON.Details, Valid: true}
}
if sceneJSON.URL != "" {
newScene.URL = sql.NullString{String: sceneJSON.URL, Valid: true}
}
if sceneJSON.Date != "" {
newScene.Date = models.SQLiteDate{String: sceneJSON.Date, Valid: true}
d := models.NewDate(sceneJSON.Date)
newScene.Date = &d
}
if sceneJSON.Rating != 0 {
newScene.Rating = sql.NullInt64{Int64: int64(sceneJSON.Rating), Valid: true}
newScene.Rating = &sceneJSON.Rating
}
newScene.Organized = sceneJSON.Organized
newScene.OCounter = sceneJSON.OCounter
newScene.CreatedAt = models.SQLiteTimestamp{Timestamp: sceneJSON.CreatedAt.GetTime()}
newScene.UpdatedAt = models.SQLiteTimestamp{Timestamp: sceneJSON.UpdatedAt.GetTime()}
newScene.CreatedAt = sceneJSON.CreatedAt.GetTime()
newScene.UpdatedAt = sceneJSON.UpdatedAt.GetTime()
if sceneJSON.File != nil {
if sceneJSON.File.Size != "" {
newScene.Size = sql.NullString{String: sceneJSON.File.Size, Valid: true}
}
if sceneJSON.File.Duration != "" {
duration, _ := strconv.ParseFloat(sceneJSON.File.Duration, 64)
newScene.Duration = sql.NullFloat64{Float64: duration, Valid: true}
}
if sceneJSON.File.VideoCodec != "" {
newScene.VideoCodec = sql.NullString{String: sceneJSON.File.VideoCodec, Valid: true}
}
if sceneJSON.File.AudioCodec != "" {
newScene.AudioCodec = sql.NullString{String: sceneJSON.File.AudioCodec, Valid: true}
}
if sceneJSON.File.Format != "" {
newScene.Format = sql.NullString{String: sceneJSON.File.Format, Valid: true}
}
if sceneJSON.File.Width != 0 {
newScene.Width = sql.NullInt64{Int64: int64(sceneJSON.File.Width), Valid: true}
}
if sceneJSON.File.Height != 0 {
newScene.Height = sql.NullInt64{Int64: int64(sceneJSON.File.Height), Valid: true}
}
if sceneJSON.File.Framerate != "" {
framerate, _ := strconv.ParseFloat(sceneJSON.File.Framerate, 64)
newScene.Framerate = sql.NullFloat64{Float64: framerate, Valid: true}
}
if sceneJSON.File.Bitrate != 0 {
newScene.Bitrate = sql.NullInt64{Int64: int64(sceneJSON.File.Bitrate), Valid: true}
}
}
// if sceneJSON.File != nil {
// if sceneJSON.File.Size != "" {
// newScene.Size = &sceneJSON.File.Size
// }
// if sceneJSON.File.Duration != "" {
// duration, _ := strconv.ParseFloat(sceneJSON.File.Duration, 64)
// newScene.Duration = &duration
// }
// if sceneJSON.File.VideoCodec != "" {
// newScene.VideoCodec = &sceneJSON.File.VideoCodec
// }
// if sceneJSON.File.AudioCodec != "" {
// newScene.AudioCodec = &sceneJSON.File.AudioCodec
// }
// if sceneJSON.File.Format != "" {
// newScene.Format = &sceneJSON.File.Format
// }
// if sceneJSON.File.Width != 0 {
// newScene.Width = &sceneJSON.File.Width
// }
// if sceneJSON.File.Height != 0 {
// newScene.Height = &sceneJSON.File.Height
// }
// if sceneJSON.File.Framerate != "" {
// framerate, _ := strconv.ParseFloat(sceneJSON.File.Framerate, 64)
// newScene.Framerate = &framerate
// }
// if sceneJSON.File.Bitrate != 0 {
// v := int64(sceneJSON.File.Bitrate)
// newScene.Bitrate = &v
// }
// }
newScene.StashIDs = append(newScene.StashIDs, i.Input.StashIDs...)
return newScene
}
@@ -169,13 +167,10 @@ func (i *Importer) populateStudio(ctx context.Context) error {
if err != nil {
return err
}
i.scene.StudioID = sql.NullInt64{
Int64: int64(studioID),
Valid: true,
}
i.scene.StudioID = &studioID
}
} else {
i.scene.StudioID = sql.NullInt64{Int64: int64(studio.ID), Valid: true}
i.scene.StudioID = &studio.ID
}
}
@@ -203,7 +198,7 @@ func (i *Importer) populateGalleries(ctx context.Context) error {
var pluckedChecksums []string
for _, gallery := range galleries {
pluckedChecksums = append(pluckedChecksums, gallery.Checksum)
pluckedChecksums = append(pluckedChecksums, gallery.Checksum())
}
missingGalleries := stringslice.StrFilter(checksums, func(checksum string) bool {
@@ -218,7 +213,9 @@ func (i *Importer) populateGalleries(ctx context.Context) error {
// we don't create galleries - just ignore
}
i.galleries = galleries
for _, o := range galleries {
i.scene.GalleryIDs = append(i.scene.GalleryIDs, o.ID)
}
}
return nil
@@ -261,7 +258,9 @@ func (i *Importer) populatePerformers(ctx context.Context) error {
// ignore if MissingRefBehaviour set to Ignore
}
i.performers = performers
for _, p := range performers {
i.scene.PerformerIDs = append(i.scene.PerformerIDs, p.ID)
}
}
return nil
@@ -314,13 +313,11 @@ func (i *Importer) populateMovies(ctx context.Context) error {
}
if inputMovie.SceneIndex != 0 {
toAdd.SceneIndex = sql.NullInt64{
Int64: int64(inputMovie.SceneIndex),
Valid: true,
}
index := inputMovie.SceneIndex
toAdd.SceneIndex = &index
}
i.movies = append(i.movies, toAdd)
i.scene.Movies = append(i.scene.Movies, toAdd)
}
}
@@ -346,7 +343,9 @@ func (i *Importer) populateTags(ctx context.Context) error {
return err
}
i.tags = tags
for _, p := range tags {
i.scene.TagIDs = append(i.scene.TagIDs, p.ID)
}
}
return nil
@@ -359,53 +358,6 @@ func (i *Importer) PostImport(ctx context.Context, id int) error {
}
}
if len(i.galleries) > 0 {
var galleryIDs []int
for _, gallery := range i.galleries {
galleryIDs = append(galleryIDs, gallery.ID)
}
if err := i.ReaderWriter.UpdateGalleries(ctx, id, galleryIDs); err != nil {
return fmt.Errorf("failed to associate galleries: %v", err)
}
}
if len(i.performers) > 0 {
var performerIDs []int
for _, performer := range i.performers {
performerIDs = append(performerIDs, performer.ID)
}
if err := i.ReaderWriter.UpdatePerformers(ctx, id, performerIDs); err != nil {
return fmt.Errorf("failed to associate performers: %v", err)
}
}
if len(i.movies) > 0 {
for index := range i.movies {
i.movies[index].SceneID = id
}
if err := i.ReaderWriter.UpdateMovies(ctx, id, i.movies); err != nil {
return fmt.Errorf("failed to associate movies: %v", err)
}
}
if len(i.tags) > 0 {
var tagIDs []int
for _, t := range i.tags {
tagIDs = append(tagIDs, t.ID)
}
if err := i.ReaderWriter.UpdateTags(ctx, id, tagIDs); err != nil {
return fmt.Errorf("failed to associate tags: %v", err)
}
}
if len(i.Input.StashIDs) > 0 {
if err := i.ReaderWriter.UpdateStashIDs(ctx, id, i.Input.StashIDs); err != nil {
return fmt.Errorf("error setting stash id: %v", err)
}
}
return nil
}
@@ -414,37 +366,37 @@ func (i *Importer) Name() string {
}
func (i *Importer) FindExistingID(ctx context.Context) (*int, error) {
var existing *models.Scene
var err error
// TODO
// var existing []*models.Scene
// var err error
switch i.FileNamingAlgorithm {
case models.HashAlgorithmMd5:
existing, err = i.ReaderWriter.FindByChecksum(ctx, i.Input.Checksum)
case models.HashAlgorithmOshash:
existing, err = i.ReaderWriter.FindByOSHash(ctx, i.Input.OSHash)
default:
panic("unknown file naming algorithm")
}
// switch i.FileNamingAlgorithm {
// case models.HashAlgorithmMd5:
// existing, err = i.ReaderWriter.FindByChecksum(ctx, i.Input.Checksum)
// case models.HashAlgorithmOshash:
// existing, err = i.ReaderWriter.FindByOSHash(ctx, i.Input.OSHash)
// default:
// panic("unknown file naming algorithm")
// }
if err != nil {
return nil, err
}
// if err != nil {
// return nil, err
// }
if existing != nil {
id := existing.ID
return &id, nil
}
// if len(existing) > 0 {
// id := existing[0].ID
// return &id, nil
// }
return nil, nil
}
func (i *Importer) Create(ctx context.Context) (*int, error) {
created, err := i.ReaderWriter.Create(ctx, i.scene)
if err != nil {
if err := i.ReaderWriter.Create(ctx, &i.scene, nil); err != nil {
return nil, fmt.Errorf("error creating scene: %v", err)
}
id := created.ID
id := i.scene.ID
i.ID = id
return &id, nil
}
@@ -453,8 +405,7 @@ func (i *Importer) Update(ctx context.Context, id int) error {
scene := i.scene
scene.ID = id
i.ID = id
_, err := i.ReaderWriter.UpdateFull(ctx, scene)
if err != nil {
if err := i.ReaderWriter.Update(ctx, &scene); err != nil {
return fmt.Errorf("error updating existing scene: %v", err)
}