diff --git a/gqlgen.yml b/gqlgen.yml index 8a21df01b..5cd2c0915 100644 --- a/gqlgen.yml +++ b/gqlgen.yml @@ -14,6 +14,10 @@ resolver: struct_tag: gqlgen models: + # Scalars + Timestamp: + model: github.com/stashapp/stash/pkg/models.Timestamp + # Objects Gallery: model: github.com/stashapp/stash/pkg/models.Gallery Image: diff --git a/graphql/schema/types/metadata.graphql b/graphql/schema/types/metadata.graphql index 523be8c5e..0d6c84cec 100644 --- a/graphql/schema/types/metadata.graphql +++ b/graphql/schema/types/metadata.graphql @@ -58,8 +58,15 @@ type GeneratePreviewOptions { previewPreset: PreviewPreset } +"Filter options for meta data scannning" +input ScanMetaDataFilterInput { + "If set, files with a modification time before this time point are ignored by the scan" + minModTime: Timestamp +} + input ScanMetadataInput { paths: [String!] + """Set name, date, details from metadata (if present)""" useFileMetadata: Boolean """Strip file extension from title""" @@ -74,6 +81,9 @@ input ScanMetadataInput { scanGeneratePhashes: Boolean """Generate image thumbnails during scan""" scanGenerateThumbnails: Boolean + + "Filter options for the scan" + filter: ScanMetaDataFilterInput } type ScanMetadataOptions { @@ -122,11 +132,11 @@ enum IdentifyFieldStrategy { """Never sets the field value""" IGNORE """ - For multi-value fields, merge with existing. + For multi-value fields, merge with existing. For single-value fields, ignore if already set """ MERGE - """Always replaces the value if a value is found. + """Always replaces the value if a value is found. For multi-value fields, any existing values are removed and replaced with the scraped values. """ diff --git a/graphql/schema/types/scalars.graphql b/graphql/schema/types/scalars.graphql new file mode 100644 index 000000000..439f0d561 --- /dev/null +++ b/graphql/schema/types/scalars.graphql @@ -0,0 +1,7 @@ + +""" +Timestamp is a point in time. It is always output as RFC3339-compatible time points. +It can be input as a RFC3339 string, or as "<4h" for "4 hours in the past" or ">5m" +for "5 minutes in the future" +""" +scalar Timestamp \ No newline at end of file diff --git a/pkg/manager/task_scan.go b/pkg/manager/task_scan.go index 3d9af4ccd..80d874ac7 100644 --- a/pkg/manager/task_scan.go +++ b/pkg/manager/task_scan.go @@ -146,6 +146,11 @@ func (j *ScanJob) Execute(ctx context.Context, progress *job.Progress) { func (j *ScanJob) queueFiles(ctx context.Context, paths []*models.StashConfig, scanQueue chan<- scanFile, parallelTasks int) (total int, newFiles int) { defer close(scanQueue) + var minModTime time.Time + if j.input.Filter != nil && j.input.Filter.MinModTime != nil { + minModTime = *j.input.Filter.MinModTime + } + wg := sizedwaitgroup.New(parallelTasks) for _, sp := range paths { @@ -160,6 +165,11 @@ func (j *ScanJob) queueFiles(ctx context.Context, paths []*models.StashConfig, s return context.Canceled } + // exit early on cutoff + if info.Mode().IsRegular() && info.ModTime().Before(minModTime) { + return nil + } + wg.Add() go func() { diff --git a/pkg/models/timestamp.go b/pkg/models/timestamp.go new file mode 100644 index 000000000..478948da6 --- /dev/null +++ b/pkg/models/timestamp.go @@ -0,0 +1,57 @@ +package models + +import ( + "errors" + "fmt" + "io" + "strconv" + "time" + + "github.com/99designs/gqlgen/graphql" + "github.com/stashapp/stash/pkg/logger" + "github.com/stashapp/stash/pkg/utils" +) + +var ErrTimestamp = errors.New("cannot parse Timestamp") + +func MarshalTimestamp(t time.Time) graphql.Marshaler { + if t.IsZero() { + return graphql.Null + } + + return graphql.WriterFunc(func(w io.Writer) { + _, err := io.WriteString(w, strconv.Quote(t.Format(time.RFC3339Nano))) + if err != nil { + logger.Warnf("could not marshal timestamp: %v", err) + } + }) +} + +func UnmarshalTimestamp(v interface{}) (time.Time, error) { + if tmpStr, ok := v.(string); ok { + if len(tmpStr) == 0 { + return time.Time{}, fmt.Errorf("%w: empty string", ErrTimestamp) + } + + switch tmpStr[0] { + case '>', '<': + d, err := time.ParseDuration(tmpStr[1:]) + if err != nil { + return time.Time{}, fmt.Errorf("%w: cannot parse %v-duration: %v", ErrTimestamp, tmpStr[0], err) + } + t := time.Now() + // Compute point in time: + if tmpStr[0] == '<' { + t = t.Add(-d) + } else { + t = t.Add(d) + } + + return t, nil + } + + return utils.ParseDateStringAsTime(tmpStr) + } + + return time.Time{}, fmt.Errorf("%w: not a string", ErrTimestamp) +} diff --git a/pkg/models/timestamp_test.go b/pkg/models/timestamp_test.go new file mode 100644 index 000000000..392a1e8d8 --- /dev/null +++ b/pkg/models/timestamp_test.go @@ -0,0 +1,90 @@ +package models + +import ( + "bytes" + "strconv" + "testing" + "time" +) + +func TestTimestampSymmetry(t *testing.T) { + n := time.Now() + buf := bytes.NewBuffer([]byte{}) + MarshalTimestamp(n).MarshalGQL(buf) + + str, err := strconv.Unquote(buf.String()) + if err != nil { + t.Fatal("could not unquote string") + } + got, err := UnmarshalTimestamp(str) + if err != nil { + t.Fatalf("could not unmarshal time: %v", err) + } + + if !n.Equal(got) { + t.Fatalf("have %v, want %v", got, n) + } +} + +func TestTimestamp(t *testing.T) { + n := time.Now().In(time.UTC) + testCases := []struct { + name string + have string + want string + }{ + {"reflexivity", n.Format(time.RFC3339Nano), n.Format(time.RFC3339Nano)}, + {"rfc3339", "2021-11-04T01:02:03Z", "2021-11-04T01:02:03Z"}, + {"date", "2021-04-05", "2021-04-05T00:00:00Z"}, + {"datetime", "2021-04-05 14:45:36", "2021-04-05T14:45:36Z"}, + {"datetime-tz", "2021-04-05 14:45:36 PDT", "2021-04-05T14:45:36Z"}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + p, err := UnmarshalTimestamp(tc.have) + if err != nil { + t.Fatalf("could not unmarshal time: %v", err) + } + + buf := bytes.NewBuffer([]byte{}) + MarshalTimestamp(p).MarshalGQL(buf) + + got, err := strconv.Unquote(buf.String()) + if err != nil { + t.Fatalf("count not unquote string") + } + if got != tc.want { + t.Errorf("got %s; want %s", got, tc.want) + } + }) + } +} + +const epsilon = 10 * time.Second + +func TestTimestampRelative(t *testing.T) { + n := time.Now() + testCases := []struct { + name string + have string + want time.Time + }{ + {"past", "<4h", n.Add(-4 * time.Hour)}, + {"future", ">5m", n.Add(5 * time.Minute)}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + got, err := UnmarshalTimestamp(tc.have) + if err != nil { + t.Fatalf("could not unmarshal time: %v", err) + } + + if got.Sub(tc.want) > epsilon { + t.Errorf("not within bound of %v; got %s; want %s", epsilon, got, tc.want) + } + }) + } + +}