Date precision (#6359)

* Remove month/year only formats from ParseDateStringAsTime
* Add precision field to Date and handle parsing year/month-only dates
* Add date precision columns for date columns
* Adjust UI to account for fuzzy dates
This commit is contained in:
WithoutPants
2025-12-08 09:11:40 +11:00
committed by GitHub
parent eb9d0705bc
commit 7db394bbea
23 changed files with 285 additions and 154 deletions

View File

@@ -1,31 +1,63 @@
package models
import (
"fmt"
"time"
"github.com/stashapp/stash/pkg/utils"
)
type DatePrecision int
const (
// default precision is day
DatePrecisionDay DatePrecision = iota
DatePrecisionMonth
DatePrecisionYear
)
// Date wraps a time.Time with a format of "YYYY-MM-DD"
type Date struct {
time.Time
Precision DatePrecision
}
const dateFormat = "2006-01-02"
var dateFormatPrecision = []string{
"2006-01-02",
"2006-01",
"2006",
}
func (d Date) String() string {
return d.Format(dateFormat)
return d.Format(dateFormatPrecision[d.Precision])
}
func (d Date) After(o Date) bool {
return d.Time.After(o.Time)
}
// ParseDate uses utils.ParseDateStringAsTime to parse a string into a date.
// ParseDate tries to parse the input string into a date using utils.ParseDateStringAsTime.
// If that fails, it attempts to parse the string with decreasing precision (month, then year).
// It returns a Date struct with the appropriate precision set, or an error if all parsing attempts fail.
func ParseDate(s string) (Date, error) {
var errs []error
// default parse to day precision
ret, err := utils.ParseDateStringAsTime(s)
if err != nil {
return Date{}, err
if err == nil {
return Date{Time: ret, Precision: DatePrecisionDay}, nil
}
return Date{Time: ret}, nil
errs = append(errs, err)
// try month and year precision
for i, format := range dateFormatPrecision[1:] {
ret, err := time.Parse(format, s)
if err == nil {
return Date{Time: ret, Precision: DatePrecision(i + 1)}, nil
}
errs = append(errs, err)
}
return Date{}, fmt.Errorf("failed to parse date %q: %v", s, errs)
}

50
pkg/models/date_test.go Normal file
View File

@@ -0,0 +1,50 @@
package models
import (
"testing"
"time"
)
func TestParseDateStringAsTime(t *testing.T) {
tests := []struct {
name string
input string
output Date
expectError bool
}{
// Full date formats (existing support)
{"RFC3339", "2014-01-02T15:04:05Z", Date{Time: time.Date(2014, 1, 2, 15, 4, 5, 0, time.UTC), Precision: DatePrecisionDay}, false},
{"Date only", "2014-01-02", Date{Time: time.Date(2014, 1, 2, 0, 0, 0, 0, time.UTC), Precision: DatePrecisionDay}, false},
{"Date with time", "2014-01-02 15:04:05", Date{Time: time.Date(2014, 1, 2, 15, 4, 5, 0, time.UTC), Precision: DatePrecisionDay}, false},
// Partial date formats (new support)
{"Year-Month", "2006-08", Date{Time: time.Date(2006, 8, 1, 0, 0, 0, 0, time.UTC), Precision: DatePrecisionMonth}, false},
{"Year only", "2014", Date{Time: time.Date(2014, 1, 1, 0, 0, 0, 0, time.UTC), Precision: DatePrecisionYear}, false},
// Invalid formats
{"Invalid format", "not-a-date", Date{}, true},
{"Empty string", "", Date{}, true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result, err := ParseDate(tt.input)
if tt.expectError {
if err == nil {
t.Errorf("Expected error for input %q, but got none", tt.input)
}
return
}
if err != nil {
t.Errorf("Unexpected error for input %q: %v", tt.input, err)
return
}
if !result.Time.Equal(tt.output.Time) || result.Precision != tt.output.Precision {
t.Errorf("For input %q, expected output %+v, got %+v", tt.input, tt.output, result)
}
})
}
}

View File

@@ -34,7 +34,7 @@ const (
cacheSizeEnv = "STASH_SQLITE_CACHE_SIZE"
)
var appSchemaVersion uint = 74
var appSchemaVersion uint = 75
//go:embed migrations/*.sql
var migrationsBox embed.FS

View File

@@ -5,6 +5,7 @@ import (
"time"
"github.com/stashapp/stash/pkg/models"
"gopkg.in/guregu/null.v4"
)
const sqliteDateLayout = "2006-01-02"
@@ -54,12 +55,12 @@ func (d NullDate) Value() (driver.Value, error) {
return d.Date.Format(sqliteDateLayout), nil
}
func (d *NullDate) DatePtr() *models.Date {
func (d *NullDate) DatePtr(precision null.Int) *models.Date {
if d == nil || !d.Valid {
return nil
}
return &models.Date{Time: d.Date}
return &models.Date{Time: d.Date, Precision: models.DatePrecision(precision.Int64)}
}
func NullDateFromDatePtr(d *models.Date) NullDate {
@@ -68,3 +69,11 @@ func NullDateFromDatePtr(d *models.Date) NullDate {
}
return NullDate{Date: d.Time, Valid: true}
}
func datePrecisionFromDatePtr(d *models.Date) null.Int {
if d == nil {
// default to day precision
return null.Int{}
}
return null.IntFrom(int64(d.Precision))
}

View File

@@ -30,12 +30,13 @@ const (
)
type galleryRow struct {
ID int `db:"id" goqu:"skipinsert"`
Title zero.String `db:"title"`
Code zero.String `db:"code"`
Date NullDate `db:"date"`
Details zero.String `db:"details"`
Photographer zero.String `db:"photographer"`
ID int `db:"id" goqu:"skipinsert"`
Title zero.String `db:"title"`
Code zero.String `db:"code"`
Date NullDate `db:"date"`
DatePrecision null.Int `db:"date_precision"`
Details zero.String `db:"details"`
Photographer zero.String `db:"photographer"`
// expressed as 1-100
Rating null.Int `db:"rating"`
Organized bool `db:"organized"`
@@ -50,6 +51,7 @@ func (r *galleryRow) fromGallery(o models.Gallery) {
r.Title = zero.StringFrom(o.Title)
r.Code = zero.StringFrom(o.Code)
r.Date = NullDateFromDatePtr(o.Date)
r.DatePrecision = datePrecisionFromDatePtr(o.Date)
r.Details = zero.StringFrom(o.Details)
r.Photographer = zero.StringFrom(o.Photographer)
r.Rating = intFromPtr(o.Rating)
@@ -74,7 +76,7 @@ func (r *galleryQueryRow) resolve() *models.Gallery {
ID: r.ID,
Title: r.Title.String,
Code: r.Code.String,
Date: r.Date.DatePtr(),
Date: r.Date.DatePtr(r.DatePrecision),
Details: r.Details.String,
Photographer: r.Photographer.String,
Rating: nullIntPtr(r.Rating),
@@ -102,7 +104,7 @@ type galleryRowRecord struct {
func (r *galleryRowRecord) fromPartial(o models.GalleryPartial) {
r.setNullString("title", o.Title)
r.setNullString("code", o.Code)
r.setNullDate("date", o.Date)
r.setNullDate("date", "date_precision", o.Date)
r.setNullString("details", o.Details)
r.setNullString("photographer", o.Photographer)
r.setNullInt("rating", o.Rating)

View File

@@ -32,11 +32,12 @@ const (
)
type groupRow struct {
ID int `db:"id" goqu:"skipinsert"`
Name zero.String `db:"name"`
Aliases zero.String `db:"aliases"`
Duration null.Int `db:"duration"`
Date NullDate `db:"date"`
ID int `db:"id" goqu:"skipinsert"`
Name zero.String `db:"name"`
Aliases zero.String `db:"aliases"`
Duration null.Int `db:"duration"`
Date NullDate `db:"date"`
DatePrecision null.Int `db:"date_precision"`
// expressed as 1-100
Rating null.Int `db:"rating"`
StudioID null.Int `db:"studio_id,omitempty"`
@@ -56,6 +57,7 @@ func (r *groupRow) fromGroup(o models.Group) {
r.Aliases = zero.StringFrom(o.Aliases)
r.Duration = intFromPtr(o.Duration)
r.Date = NullDateFromDatePtr(o.Date)
r.DatePrecision = datePrecisionFromDatePtr(o.Date)
r.Rating = intFromPtr(o.Rating)
r.StudioID = intFromPtr(o.StudioID)
r.Director = zero.StringFrom(o.Director)
@@ -70,7 +72,7 @@ func (r *groupRow) resolve() *models.Group {
Name: r.Name.String,
Aliases: r.Aliases.String,
Duration: nullIntPtr(r.Duration),
Date: r.Date.DatePtr(),
Date: r.Date.DatePtr(r.DatePrecision),
Rating: nullIntPtr(r.Rating),
StudioID: nullIntPtr(r.StudioID),
Director: r.Director.String,
@@ -90,7 +92,7 @@ func (r *groupRowRecord) fromPartial(o models.GroupPartial) {
r.setNullString("name", o.Name)
r.setNullString("aliases", o.Aliases)
r.setNullInt("duration", o.Duration)
r.setNullDate("date", o.Date)
r.setNullDate("date", "date_precision", o.Date)
r.setNullInt("rating", o.Rating)
r.setNullInt("studio_id", o.StudioID)
r.setNullString("director", o.Director)

View File

@@ -34,15 +34,16 @@ type imageRow struct {
Title zero.String `db:"title"`
Code zero.String `db:"code"`
// expressed as 1-100
Rating null.Int `db:"rating"`
Date NullDate `db:"date"`
Details zero.String `db:"details"`
Photographer zero.String `db:"photographer"`
Organized bool `db:"organized"`
OCounter int `db:"o_counter"`
StudioID null.Int `db:"studio_id,omitempty"`
CreatedAt Timestamp `db:"created_at"`
UpdatedAt Timestamp `db:"updated_at"`
Rating null.Int `db:"rating"`
Date NullDate `db:"date"`
DatePrecision null.Int `db:"date_precision"`
Details zero.String `db:"details"`
Photographer zero.String `db:"photographer"`
Organized bool `db:"organized"`
OCounter int `db:"o_counter"`
StudioID null.Int `db:"studio_id,omitempty"`
CreatedAt Timestamp `db:"created_at"`
UpdatedAt Timestamp `db:"updated_at"`
}
func (r *imageRow) fromImage(i models.Image) {
@@ -51,6 +52,7 @@ func (r *imageRow) fromImage(i models.Image) {
r.Code = zero.StringFrom(i.Code)
r.Rating = intFromPtr(i.Rating)
r.Date = NullDateFromDatePtr(i.Date)
r.DatePrecision = datePrecisionFromDatePtr(i.Date)
r.Details = zero.StringFrom(i.Details)
r.Photographer = zero.StringFrom(i.Photographer)
r.Organized = i.Organized
@@ -74,7 +76,7 @@ func (r *imageQueryRow) resolve() *models.Image {
Title: r.Title.String,
Code: r.Code.String,
Rating: nullIntPtr(r.Rating),
Date: r.Date.DatePtr(),
Date: r.Date.DatePtr(r.DatePrecision),
Details: r.Details.String,
Photographer: r.Photographer.String,
Organized: r.Organized,
@@ -103,7 +105,7 @@ func (r *imageRowRecord) fromPartial(i models.ImagePartial) {
r.setNullString("title", i.Title)
r.setNullString("code", i.Code)
r.setNullInt("rating", i.Rating)
r.setNullDate("date", i.Date)
r.setNullDate("date", "date_precision", i.Date)
r.setNullString("details", i.Details)
r.setNullString("photographer", i.Photographer)
r.setBool("organized", i.Organized)

View File

@@ -0,0 +1,13 @@
ALTER TABLE "scenes" ADD COLUMN "date_precision" TINYINT;
ALTER TABLE "images" ADD COLUMN "date_precision" TINYINT;
ALTER TABLE "galleries" ADD COLUMN "date_precision" TINYINT;
ALTER TABLE "groups" ADD COLUMN "date_precision" TINYINT;
ALTER TABLE "performers" ADD COLUMN "birthdate_precision" TINYINT;
ALTER TABLE "performers" ADD COLUMN "death_date_precision" TINYINT;
UPDATE "scenes" SET "date_precision" = 0 WHERE "date" IS NOT NULL;
UPDATE "images" SET "date_precision" = 0 WHERE "date" IS NOT NULL;
UPDATE "galleries" SET "date_precision" = 0 WHERE "date" IS NOT NULL;
UPDATE "groups" SET "date_precision" = 0 WHERE "date" IS NOT NULL;
UPDATE "performers" SET "birthdate_precision" = 0 WHERE "birthdate" IS NOT NULL;
UPDATE "performers" SET "death_date_precision" = 0 WHERE "death_date" IS NOT NULL;

View File

@@ -30,32 +30,34 @@ const (
)
type performerRow struct {
ID int `db:"id" goqu:"skipinsert"`
Name null.String `db:"name"` // TODO: make schema non-nullable
Disambigation zero.String `db:"disambiguation"`
Gender zero.String `db:"gender"`
Birthdate NullDate `db:"birthdate"`
Ethnicity zero.String `db:"ethnicity"`
Country zero.String `db:"country"`
EyeColor zero.String `db:"eye_color"`
Height null.Int `db:"height"`
Measurements zero.String `db:"measurements"`
FakeTits zero.String `db:"fake_tits"`
PenisLength null.Float `db:"penis_length"`
Circumcised zero.String `db:"circumcised"`
CareerLength zero.String `db:"career_length"`
Tattoos zero.String `db:"tattoos"`
Piercings zero.String `db:"piercings"`
Favorite bool `db:"favorite"`
CreatedAt Timestamp `db:"created_at"`
UpdatedAt Timestamp `db:"updated_at"`
ID int `db:"id" goqu:"skipinsert"`
Name null.String `db:"name"` // TODO: make schema non-nullable
Disambigation zero.String `db:"disambiguation"`
Gender zero.String `db:"gender"`
Birthdate NullDate `db:"birthdate"`
BirthdatePrecision null.Int `db:"birthdate_precision"`
Ethnicity zero.String `db:"ethnicity"`
Country zero.String `db:"country"`
EyeColor zero.String `db:"eye_color"`
Height null.Int `db:"height"`
Measurements zero.String `db:"measurements"`
FakeTits zero.String `db:"fake_tits"`
PenisLength null.Float `db:"penis_length"`
Circumcised zero.String `db:"circumcised"`
CareerLength zero.String `db:"career_length"`
Tattoos zero.String `db:"tattoos"`
Piercings zero.String `db:"piercings"`
Favorite bool `db:"favorite"`
CreatedAt Timestamp `db:"created_at"`
UpdatedAt Timestamp `db:"updated_at"`
// expressed as 1-100
Rating null.Int `db:"rating"`
Details zero.String `db:"details"`
DeathDate NullDate `db:"death_date"`
HairColor zero.String `db:"hair_color"`
Weight null.Int `db:"weight"`
IgnoreAutoTag bool `db:"ignore_auto_tag"`
Rating null.Int `db:"rating"`
Details zero.String `db:"details"`
DeathDate NullDate `db:"death_date"`
DeathDatePrecision null.Int `db:"death_date_precision"`
HairColor zero.String `db:"hair_color"`
Weight null.Int `db:"weight"`
IgnoreAutoTag bool `db:"ignore_auto_tag"`
// not used in resolution or updates
ImageBlob zero.String `db:"image_blob"`
@@ -69,6 +71,7 @@ func (r *performerRow) fromPerformer(o models.Performer) {
r.Gender = zero.StringFrom(o.Gender.String())
}
r.Birthdate = NullDateFromDatePtr(o.Birthdate)
r.BirthdatePrecision = datePrecisionFromDatePtr(o.Birthdate)
r.Ethnicity = zero.StringFrom(o.Ethnicity)
r.Country = zero.StringFrom(o.Country)
r.EyeColor = zero.StringFrom(o.EyeColor)
@@ -88,6 +91,7 @@ func (r *performerRow) fromPerformer(o models.Performer) {
r.Rating = intFromPtr(o.Rating)
r.Details = zero.StringFrom(o.Details)
r.DeathDate = NullDateFromDatePtr(o.DeathDate)
r.DeathDatePrecision = datePrecisionFromDatePtr(o.DeathDate)
r.HairColor = zero.StringFrom(o.HairColor)
r.Weight = intFromPtr(o.Weight)
r.IgnoreAutoTag = o.IgnoreAutoTag
@@ -98,7 +102,7 @@ func (r *performerRow) resolve() *models.Performer {
ID: r.ID,
Name: r.Name.String,
Disambiguation: r.Disambigation.String,
Birthdate: r.Birthdate.DatePtr(),
Birthdate: r.Birthdate.DatePtr(r.BirthdatePrecision),
Ethnicity: r.Ethnicity.String,
Country: r.Country.String,
EyeColor: r.EyeColor.String,
@@ -115,7 +119,7 @@ func (r *performerRow) resolve() *models.Performer {
// expressed as 1-100
Rating: nullIntPtr(r.Rating),
Details: r.Details.String,
DeathDate: r.DeathDate.DatePtr(),
DeathDate: r.DeathDate.DatePtr(r.DeathDatePrecision),
HairColor: r.HairColor.String,
Weight: nullIntPtr(r.Weight),
IgnoreAutoTag: r.IgnoreAutoTag,
@@ -142,7 +146,7 @@ func (r *performerRowRecord) fromPartial(o models.PerformerPartial) {
r.setString("name", o.Name)
r.setNullString("disambiguation", o.Disambiguation)
r.setNullString("gender", o.Gender)
r.setNullDate("birthdate", o.Birthdate)
r.setNullDate("birthdate", "birthdate_precision", o.Birthdate)
r.setNullString("ethnicity", o.Ethnicity)
r.setNullString("country", o.Country)
r.setNullString("eye_color", o.EyeColor)
@@ -159,7 +163,7 @@ func (r *performerRowRecord) fromPartial(o models.PerformerPartial) {
r.setTimestamp("updated_at", o.UpdatedAt)
r.setNullInt("rating", o.Rating)
r.setNullString("details", o.Details)
r.setNullDate("death_date", o.DeathDate)
r.setNullDate("death_date", "death_date_precision", o.DeathDate)
r.setNullString("hair_color", o.HairColor)
r.setNullInt("weight", o.Weight)
r.setBool("ignore_auto_tag", o.IgnoreAutoTag)

View File

@@ -100,8 +100,9 @@ func (r *updateRecord) setNullTimestamp(destField string, v models.OptionalTime)
}
}
func (r *updateRecord) setNullDate(destField string, v models.OptionalDate) {
func (r *updateRecord) setNullDate(destField string, precisionField string, v models.OptionalDate) {
if v.Set {
r.set(destField, NullDateFromDatePtr(v.Ptr()))
r.set(precisionField, datePrecisionFromDatePtr(v.Ptr()))
}
}

View File

@@ -76,12 +76,13 @@ ORDER BY files.size DESC;
`
type sceneRow struct {
ID int `db:"id" goqu:"skipinsert"`
Title zero.String `db:"title"`
Code zero.String `db:"code"`
Details zero.String `db:"details"`
Director zero.String `db:"director"`
Date NullDate `db:"date"`
ID int `db:"id" goqu:"skipinsert"`
Title zero.String `db:"title"`
Code zero.String `db:"code"`
Details zero.String `db:"details"`
Director zero.String `db:"director"`
Date NullDate `db:"date"`
DatePrecision null.Int `db:"date_precision"`
// expressed as 1-100
Rating null.Int `db:"rating"`
Organized bool `db:"organized"`
@@ -102,6 +103,7 @@ func (r *sceneRow) fromScene(o models.Scene) {
r.Details = zero.StringFrom(o.Details)
r.Director = zero.StringFrom(o.Director)
r.Date = NullDateFromDatePtr(o.Date)
r.DatePrecision = datePrecisionFromDatePtr(o.Date)
r.Rating = intFromPtr(o.Rating)
r.Organized = o.Organized
r.StudioID = intFromPtr(o.StudioID)
@@ -127,7 +129,7 @@ func (r *sceneQueryRow) resolve() *models.Scene {
Code: r.Code.String,
Details: r.Details.String,
Director: r.Director.String,
Date: r.Date.DatePtr(),
Date: r.Date.DatePtr(r.DatePrecision),
Rating: nullIntPtr(r.Rating),
Organized: r.Organized,
StudioID: nullIntPtr(r.StudioID),
@@ -159,7 +161,7 @@ func (r *sceneRowRecord) fromPartial(o models.ScenePartial) {
r.setNullString("code", o.Code)
r.setNullString("details", o.Details)
r.setNullString("director", o.Director)
r.setNullDate("date", o.Date)
r.setNullDate("date", "date_precision", o.Date)
r.setNullInt("rating", o.Rating)
r.setBool("organized", o.Organized)
r.setNullInt("studio_id", o.StudioID)

View File

@@ -23,17 +23,5 @@ func ParseDateStringAsTime(dateString string) (time.Time, error) {
return t, nil
}
// Support partial dates: year-month format
t, e = time.Parse("2006-01", dateString)
if e == nil {
return t, nil
}
// Support partial dates: year only format
t, e = time.Parse("2006", dateString)
if e == nil {
return t, nil
}
return time.Time{}, fmt.Errorf("ParseDateStringAsTime failed: dateString <%s>", dateString)
}

View File

@@ -2,7 +2,6 @@ package utils
import (
"testing"
"time"
)
func TestParseDateStringAsTime(t *testing.T) {
@@ -16,13 +15,11 @@ func TestParseDateStringAsTime(t *testing.T) {
{"Date only", "2014-01-02", false},
{"Date with time", "2014-01-02 15:04:05", false},
// Partial date formats (new support)
{"Year-Month", "2006-08", false},
{"Year only", "2014", false},
// Invalid formats
{"Invalid format", "not-a-date", true},
{"Empty string", "", true},
{"Year-Month", "2006-08", true},
{"Year only", "2014", true},
}
for _, tt := range tests {
@@ -44,37 +41,3 @@ func TestParseDateStringAsTime(t *testing.T) {
})
}
}
func TestParseDateStringAsTime_YearOnly(t *testing.T) {
result, err := ParseDateStringAsTime("2014")
if err != nil {
t.Fatalf("Failed to parse year-only date: %v", err)
}
if result.Year() != 2014 {
t.Errorf("Expected year 2014, got %d", result.Year())
}
if result.Month() != time.January {
t.Errorf("Expected month January, got %s", result.Month())
}
if result.Day() != 1 {
t.Errorf("Expected day 1, got %d", result.Day())
}
}
func TestParseDateStringAsTime_YearMonth(t *testing.T) {
result, err := ParseDateStringAsTime("2006-08")
if err != nil {
t.Fatalf("Failed to parse year-month date: %v", err)
}
if result.Year() != 2006 {
t.Errorf("Expected year 2006, got %d", result.Year())
}
if result.Month() != time.August {
t.Errorf("Expected month August, got %s", result.Month())
}
if result.Day() != 1 {
t.Errorf("Expected day 1, got %d", result.Day())
}
}