Add updated_at field to stash_id's (#5259)

* Add updated_at field to stash_id's
* Only set updated at on stash ids when actually updating in identify

---------
Co-authored-by: WithoutPants <53250216+WithoutPants@users.noreply.github.com>

pkg/models/model_scene.go

@@ -192,9 +192,9 @@ func (s ScenePartial) UpdateInput(id int) SceneUpdateInput {
 		dateStr = &v
 	}
 
-	var stashIDs []StashID
+	var stashIDs StashIDs
 	if s.StashIDs != nil {
-		stashIDs = s.StashIDs.StashIDs
+		stashIDs = StashIDs(s.StashIDs.StashIDs)
 	}
 
 	ret := SceneUpdateInput{
@@ -212,7 +212,7 @@ func (s ScenePartial) UpdateInput(id int) SceneUpdateInput {
 		PerformerIds: s.PerformerIDs.IDStrings(),
 		Movies:       s.GroupIDs.SceneMovieInputs(),
 		TagIds:       s.TagIDs.IDStrings(),
-		StashIds:     stashIDs,
+		StashIds:     stashIDs.ToStashIDInputs(),
 	}
 
 	return ret

pkg/models/model_scraped_item.go

@@ -3,6 +3,7 @@ package models
 import (
 	"context"
 	"strconv"
+	"time"
 
 	"github.com/stashapp/stash/pkg/sliceutil/stringslice"
 	"github.com/stashapp/stash/pkg/utils"
@@ -29,8 +30,9 @@ func (s *ScrapedStudio) ToStudio(endpoint string, excluded map[string]bool) *Stu
 	if s.RemoteSiteID != nil && endpoint != "" {
 		ret.StashIDs = NewRelatedStashIDs([]StashID{
 			{
-				Endpoint: endpoint,
-				StashID:  *s.RemoteSiteID,
+				Endpoint:  endpoint,
+				StashID:   *s.RemoteSiteID,
+				UpdatedAt: time.Now(),
 			},
 		})
 	}
@@ -65,6 +67,7 @@ func (s *ScrapedStudio) GetImage(ctx context.Context, excluded map[string]bool)
 func (s *ScrapedStudio) ToPartial(id string, endpoint string, excluded map[string]bool, existingStashIDs []StashID) StudioPartial {
 	ret := NewStudioPartial()
 	ret.ID, _ = strconv.Atoi(id)
+	currentTime := time.Now()
 
 	if s.Name != "" && !excluded["name"] {
 		ret.Name = NewOptionalString(s.Name)
@@ -90,8 +93,9 @@ func (s *ScrapedStudio) ToPartial(id string, endpoint string, excluded map[strin
 			Mode:     RelationshipUpdateModeSet,
 		}
 		ret.StashIDs.Set(StashID{
-			Endpoint: endpoint,
-			StashID:  *s.RemoteSiteID,
+			Endpoint:  endpoint,
+			StashID:   *s.RemoteSiteID,
+			UpdatedAt: currentTime,
 		})
 	}
 
@@ -137,6 +141,7 @@ func (ScrapedPerformer) IsScrapedContent() {}
 
 func (p *ScrapedPerformer) ToPerformer(endpoint string, excluded map[string]bool) *Performer {
 	ret := NewPerformer()
+	currentTime := time.Now()
 	ret.Name = *p.Name
 
 	if p.Aliases != nil && !excluded["aliases"] {
@@ -244,8 +249,9 @@ func (p *ScrapedPerformer) ToPerformer(endpoint string, excluded map[string]bool
 	if p.RemoteSiteID != nil && endpoint != "" {
 		ret.StashIDs = NewRelatedStashIDs([]StashID{
 			{
-				Endpoint: endpoint,
-				StashID:  *p.RemoteSiteID,
+				Endpoint:  endpoint,
+				StashID:   *p.RemoteSiteID,
+				UpdatedAt: currentTime,
 			},
 		})
 	}
@@ -375,8 +381,9 @@ func (p *ScrapedPerformer) ToPartial(endpoint string, excluded map[string]bool,
 			Mode:     RelationshipUpdateModeSet,
 		}
 		ret.StashIDs.Set(StashID{
-			Endpoint: endpoint,
-			StashID:  *p.RemoteSiteID,
+			Endpoint:  endpoint,
+			StashID:   *p.RemoteSiteID,
+			UpdatedAt: time.Now(),
 		})
 	}
 

pkg/models/model_scraped_item_test.go

@@ -87,6 +87,11 @@ func Test_scrapedToStudioInput(t *testing.T) {
 
 			got.CreatedAt = time.Time{}
 			got.UpdatedAt = time.Time{}
+			if got.StashIDs.Loaded() && len(got.StashIDs.List()) > 0 {
+				for stid := range got.StashIDs.List() {
+					got.StashIDs.List()[stid].UpdatedAt = time.Time{}
+				}
+			}
 			assert.Equal(t, tt.want, got)
 		})
 	}
@@ -243,6 +248,12 @@ func Test_scrapedToPerformerInput(t *testing.T) {
 
 			got.CreatedAt = time.Time{}
 			got.UpdatedAt = time.Time{}
+
+			if got.StashIDs.Loaded() && len(got.StashIDs.List()) > 0 {
+				for stid := range got.StashIDs.List() {
+					got.StashIDs.List()[stid].UpdatedAt = time.Time{}
+				}
+			}
 			assert.Equal(t, tt.want, got)
 		})
 	}
@@ -263,7 +274,7 @@ func TestScrapedStudio_ToPartial(t *testing.T) {
 		images            = []string{image}
 
 		existingEndpoint = "existingEndpoint"
-		existingStashID  = StashID{"existingStashID", existingEndpoint}
+		existingStashID  = StashID{"existingStashID", existingEndpoint, time.Time{}}
 		existingStashIDs = []StashID{existingStashID}
 	)
 
@@ -362,6 +373,11 @@ func TestScrapedStudio_ToPartial(t *testing.T) {
 
 			// unset updatedAt - we don't need to compare it
 			got.UpdatedAt = OptionalTime{}
+			if got.StashIDs != nil && len(got.StashIDs.StashIDs) > 0 {
+				for stid := range got.StashIDs.StashIDs {
+					got.StashIDs.StashIDs[stid].UpdatedAt = time.Time{}
+				}
+			}
 
 			assert.Equal(t, tt.want, got)
 		})

pkg/models/performer.go

@@ -226,14 +226,14 @@ type PerformerCreateInput struct {
 	Favorite       *bool           `json:"favorite"`
 	TagIds         []string        `json:"tag_ids"`
 	// This should be a URL or a base64 encoded data URL
-	Image         *string   `json:"image"`
-	StashIds      []StashID `json:"stash_ids"`
-	Rating100     *int      `json:"rating100"`
-	Details       *string   `json:"details"`
-	DeathDate     *string   `json:"death_date"`
-	HairColor     *string   `json:"hair_color"`
-	Weight        *int      `json:"weight"`
-	IgnoreAutoTag *bool     `json:"ignore_auto_tag"`
+	Image         *string        `json:"image"`
+	StashIds      []StashIDInput `json:"stash_ids"`
+	Rating100     *int           `json:"rating100"`
+	Details       *string        `json:"details"`
+	DeathDate     *string        `json:"death_date"`
+	HairColor     *string        `json:"hair_color"`
+	Weight        *int           `json:"weight"`
+	IgnoreAutoTag *bool          `json:"ignore_auto_tag"`
 }
 
 type PerformerUpdateInput struct {
@@ -263,12 +263,12 @@ type PerformerUpdateInput struct {
 	Favorite       *bool           `json:"favorite"`
 	TagIds         []string        `json:"tag_ids"`
 	// This should be a URL or a base64 encoded data URL
-	Image         *string   `json:"image"`
-	StashIds      []StashID `json:"stash_ids"`
-	Rating100     *int      `json:"rating100"`
-	Details       *string   `json:"details"`
-	DeathDate     *string   `json:"death_date"`
-	HairColor     *string   `json:"hair_color"`
-	Weight        *int      `json:"weight"`
-	IgnoreAutoTag *bool     `json:"ignore_auto_tag"`
+	Image         *string        `json:"image"`
+	StashIds      []StashIDInput `json:"stash_ids"`
+	Rating100     *int           `json:"rating100"`
+	Details       *string        `json:"details"`
+	DeathDate     *string        `json:"death_date"`
+	HairColor     *string        `json:"hair_color"`
+	Weight        *int           `json:"weight"`
+	IgnoreAutoTag *bool          `json:"ignore_auto_tag"`
 }

pkg/models/scene.go

@@ -163,8 +163,8 @@ type SceneCreateInput struct {
 	Groups       []SceneGroupInput `json:"groups"`
 	TagIds       []string          `json:"tag_ids"`
 	// This should be a URL or a base64 encoded data URL
-	CoverImage *string   `json:"cover_image"`
-	StashIds   []StashID `json:"stash_ids"`
+	CoverImage *string        `json:"cover_image"`
+	StashIds   []StashIDInput `json:"stash_ids"`
 	// The first id will be assigned as primary.
 	// Files will be reassigned from existing scenes if applicable.
 	// Files must not already be primary for another scene.
@@ -191,12 +191,12 @@ type SceneUpdateInput struct {
 	Groups           []SceneGroupInput `json:"groups"`
 	TagIds           []string          `json:"tag_ids"`
 	// This should be a URL or a base64 encoded data URL
-	CoverImage    *string   `json:"cover_image"`
-	StashIds      []StashID `json:"stash_ids"`
-	ResumeTime    *float64  `json:"resume_time"`
-	PlayDuration  *float64  `json:"play_duration"`
-	PlayCount     *int      `json:"play_count"`
-	PrimaryFileID *string   `json:"primary_file_id"`
+	CoverImage    *string        `json:"cover_image"`
+	StashIds      []StashIDInput `json:"stash_ids"`
+	ResumeTime    *float64       `json:"resume_time"`
+	PlayDuration  *float64       `json:"play_duration"`
+	PlayCount     *int           `json:"play_count"`
+	PrimaryFileID *string        `json:"primary_file_id"`
 }
 
 type SceneDestroyInput struct {

pkg/models/stash_ids.go

@@ -1,8 +1,89 @@
 package models
 
+import (
+	"slices"
+	"time"
+)
+
 type StashID struct {
-	StashID  string `db:"stash_id" json:"stash_id"`
-	Endpoint string `db:"endpoint" json:"endpoint"`
+	StashID   string    `db:"stash_id" json:"stash_id"`
+	Endpoint  string    `db:"endpoint" json:"endpoint"`
+	UpdatedAt time.Time `db:"updated_at" json:"updated_at"`
+}
+
+func (s StashID) ToStashIDInput() StashIDInput {
+	t := s.UpdatedAt
+	return StashIDInput{
+		StashID:   s.StashID,
+		Endpoint:  s.Endpoint,
+		UpdatedAt: &t,
+	}
+}
+
+type StashIDs []StashID
+
+func (s StashIDs) ToStashIDInputs() StashIDInputs {
+	if s == nil {
+		return nil
+	}
+
+	ret := make(StashIDInputs, len(s))
+	for i, v := range s {
+		ret[i] = v.ToStashIDInput()
+	}
+	return ret
+}
+
+// HasSameStashIDs returns true if the two lists of StashIDs are the same, ignoring order and updated at time.
+func (s StashIDs) HasSameStashIDs(other StashIDs) bool {
+	if len(s) != len(other) {
+		return false
+	}
+
+	for _, v := range s {
+		if !slices.ContainsFunc(other, func(o StashID) bool {
+			return o.StashID == v.StashID && o.Endpoint == v.Endpoint
+		}) {
+			return false
+		}
+	}
+
+	return true
+}
+
+type StashIDInput struct {
+	StashID   string     `db:"stash_id" json:"stash_id"`
+	Endpoint  string     `db:"endpoint" json:"endpoint"`
+	UpdatedAt *time.Time `db:"updated_at" json:"updated_at"`
+}
+
+func (s StashIDInput) ToStashID() StashID {
+	ret := StashID{
+		StashID:  s.StashID,
+		Endpoint: s.Endpoint,
+	}
+	if s.UpdatedAt != nil {
+		ret.UpdatedAt = *s.UpdatedAt
+	} else {
+		// default to now if not provided
+		ret.UpdatedAt = time.Now()
+	}
+
+	return ret
+}
+
+type StashIDInputs []StashIDInput
+
+func (s StashIDInputs) ToStashIDs() StashIDs {
+	if s == nil {
+		return nil
+	}
+
+	ret := make(StashIDs, len(s))
+	for i, v := range s {
+		ret[i] = v.ToStashID()
+	}
+	return ret
 }
 
 type UpdateStashIDs struct {

pkg/models/studio.go

@@ -51,14 +51,14 @@ type StudioCreateInput struct {
 	URL      *string `json:"url"`
 	ParentID *string `json:"parent_id"`
 	// This should be a URL or a base64 encoded data URL
-	Image         *string   `json:"image"`
-	StashIds      []StashID `json:"stash_ids"`
-	Rating100     *int      `json:"rating100"`
-	Favorite      *bool     `json:"favorite"`
-	Details       *string   `json:"details"`
-	Aliases       []string  `json:"aliases"`
-	TagIds        []string  `json:"tag_ids"`
-	IgnoreAutoTag *bool     `json:"ignore_auto_tag"`
+	Image         *string        `json:"image"`
+	StashIds      []StashIDInput `json:"stash_ids"`
+	Rating100     *int           `json:"rating100"`
+	Favorite      *bool          `json:"favorite"`
+	Details       *string        `json:"details"`
+	Aliases       []string       `json:"aliases"`
+	TagIds        []string       `json:"tag_ids"`
+	IgnoreAutoTag *bool          `json:"ignore_auto_tag"`
 }
 
 type StudioUpdateInput struct {
@@ -67,12 +67,12 @@ type StudioUpdateInput struct {
 	URL      *string `json:"url"`
 	ParentID *string `json:"parent_id"`
 	// This should be a URL or a base64 encoded data URL
-	Image         *string   `json:"image"`
-	StashIds      []StashID `json:"stash_ids"`
-	Rating100     *int      `json:"rating100"`
-	Favorite      *bool     `json:"favorite"`
-	Details       *string   `json:"details"`
-	Aliases       []string  `json:"aliases"`
-	TagIds        []string  `json:"tag_ids"`
-	IgnoreAutoTag *bool     `json:"ignore_auto_tag"`
+	Image         *string        `json:"image"`
+	StashIds      []StashIDInput `json:"stash_ids"`
+	Rating100     *int           `json:"rating100"`
+	Favorite      *bool          `json:"favorite"`
+	Details       *string        `json:"details"`
+	Aliases       []string       `json:"aliases"`
+	TagIds        []string       `json:"tag_ids"`
+	IgnoreAutoTag *bool          `json:"ignore_auto_tag"`
 }

pkg/scene/update_test.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"strconv"
 	"testing"
+	"time"
 
 	"github.com/stashapp/stash/pkg/models"
 	"github.com/stashapp/stash/pkg/models/mocks"
@@ -236,16 +237,19 @@ func TestUpdateSet_UpdateInput(t *testing.T) {
 	tagIDStrs := intslice.IntSliceToStringSlice(tagIDs)
 	stashID := "stashID"
 	endpoint := "endpoint"
+	updatedAt := time.Now()
 	stashIDs := []models.StashID{
 		{
-			StashID:  stashID,
-			Endpoint: endpoint,
+			StashID:   stashID,
+			Endpoint:  endpoint,
+			UpdatedAt: updatedAt,
 		},
 	}
-	stashIDInputs := []models.StashID{
+	stashIDInputs := []models.StashIDInput{
 		{
-			StashID:  stashID,
-			Endpoint: endpoint,
+			StashID:   stashID,
+			Endpoint:  endpoint,
+			UpdatedAt: &updatedAt,
 		},
 	}
 

pkg/sqlite/database.go

@@ -34,7 +34,7 @@ const (
 	cacheSizeEnv = "STASH_SQLITE_CACHE_SIZE"
 )
 
-var appSchemaVersion uint = 68
+var appSchemaVersion uint = 69
 
 //go:embed migrations/*.sql
 var migrationsBox embed.FS

pkg/sqlite/migrations/69_stash_id_updated_at.up.sql

@@ -0,0 +1,3 @@
+ALTER TABLE `performer_stash_ids` ADD COLUMN `updated_at` datetime not null default '1970-01-01T00:00:00Z';
+ALTER TABLE `scene_stash_ids` ADD COLUMN `updated_at` datetime not null default '1970-01-01T00:00:00Z';
+ALTER TABLE `studio_stash_ids` ADD COLUMN `updated_at` datetime not null default '1970-01-01T00:00:00Z';

pkg/sqlite/repository.go

@@ -14,11 +14,6 @@ import (
 
 const idColumn = "id"
 
-type objectList interface {
-	Append(o interface{})
-	New() interface{}
-}
-
 type repository struct {
 	tableName string
 	idColumn  string
@@ -124,17 +119,6 @@ func (r *repository) queryFunc(ctx context.Context, query string, args []interfa
 	return nil
 }
 
-func (r *repository) query(ctx context.Context, query string, args []interface{}, out objectList) error {
-	return r.queryFunc(ctx, query, args, false, func(rows *sqlx.Rows) error {
-		object := out.New()
-		if err := rows.StructScan(object); err != nil {
-			return err
-		}
-		out.Append(object)
-		return nil
-	})
-}
-
 func (r *repository) queryStruct(ctx context.Context, query string, args []interface{}, out interface{}) error {
 	if err := r.queryFunc(ctx, query, args, true, func(rows *sqlx.Rows) error {
 		if err := rows.StructScan(out); err != nil {
@@ -421,7 +405,7 @@ type stashIDRepository struct {
 type stashIDs []models.StashID
 
 func (s *stashIDs) Append(o interface{}) {
-	*s = append(*s, *o.(*models.StashID))
+	*s = append(*s, o.(models.StashID))
 }
 
 func (s *stashIDs) New() interface{} {
@@ -429,10 +413,17 @@ func (s *stashIDs) New() interface{} {
 }
 
 func (r *stashIDRepository) get(ctx context.Context, id int) ([]models.StashID, error) {
-	query := fmt.Sprintf("SELECT stash_id, endpoint from %s WHERE %s = ?", r.tableName, r.idColumn)
+	query := fmt.Sprintf("SELECT stash_id, endpoint, updated_at from %s WHERE %s = ?", r.tableName, r.idColumn)
 	var ret stashIDs
-	err := r.query(ctx, query, []interface{}{id}, &ret)
-	return []models.StashID(ret), err
+	err := r.queryFunc(ctx, query, []interface{}{id}, false, func(rows *sqlx.Rows) error {
+		var v stashIDRow
+		if err := rows.StructScan(&v); err != nil {
+			return err
+		}
+		ret.Append(v.resolve())
+		return nil
+	})
+	return ret, err
 }
 
 type filesRepository struct {

pkg/sqlite/table.go

@@ -275,19 +275,21 @@ type stashIDTable struct {
 }
 
 type stashIDRow struct {
-	StashID  null.String `db:"stash_id"`
-	Endpoint null.String `db:"endpoint"`
+	StashID   null.String `db:"stash_id"`
+	Endpoint  null.String `db:"endpoint"`
+	UpdatedAt Timestamp   `db:"updated_at"`
 }
 
 func (r *stashIDRow) resolve() models.StashID {
 	return models.StashID{
-		StashID:  r.StashID.String,
-		Endpoint: r.Endpoint.String,
+		StashID:   r.StashID.String,
+		Endpoint:  r.Endpoint.String,
+		UpdatedAt: r.UpdatedAt.Timestamp,
 	}
 }
 
 func (t *stashIDTable) get(ctx context.Context, id int) ([]models.StashID, error) {
-	q := dialect.Select("endpoint", "stash_id").From(t.table.table).Where(t.idColumn.Eq(id))
+	q := dialect.Select("endpoint", "stash_id", "updated_at").From(t.table.table).Where(t.idColumn.Eq(id))
 
 	const single = false
 	var ret []models.StashID
@@ -308,8 +310,8 @@ func (t *stashIDTable) get(ctx context.Context, id int) ([]models.StashID, error
 }
 
 func (t *stashIDTable) insertJoin(ctx context.Context, id int, v models.StashID) (sql.Result, error) {
-	q := dialect.Insert(t.table.table).Cols(t.idColumn.GetCol(), "endpoint", "stash_id").Vals(
-		goqu.Vals{id, v.Endpoint, v.StashID},
+	var q = dialect.Insert(t.table.table).Cols(t.idColumn.GetCol(), "endpoint", "stash_id", "updated_at").Vals(
+		goqu.Vals{id, v.Endpoint, v.StashID, v.UpdatedAt},
 	)
 	ret, err := exec(ctx, q)
 	if err != nil {