Performer custom fields (#5487)

* Backend changes
* Show custom field values
* Add custom fields table input
* Add custom field filtering
* Add unit tests
* Include custom fields in import/export
* Anonymise performer custom fields
* Move json.Number handler functions to api
* Handle json.Number conversion in api

internal/api/json.go

@@ -0,0 +1,36 @@
+package api
+
+import (
+	"encoding/json"
+	"strings"
+)
+
+// JSONNumberToNumber converts a JSON number to either a float64 or int64.
+func jsonNumberToNumber(n json.Number) interface{} {
+	if strings.Contains(string(n), ".") {
+		f, _ := n.Float64()
+		return f
+	}
+	ret, _ := n.Int64()
+	return ret
+}
+
+// ConvertMapJSONNumbers converts all JSON numbers in a map to either float64 or int64.
+func convertMapJSONNumbers(m map[string]interface{}) (ret map[string]interface{}) {
+	if m == nil {
+		return nil
+	}
+
+	ret = make(map[string]interface{})
+	for k, v := range m {
+		if n, ok := v.(json.Number); ok {
+			ret[k] = jsonNumberToNumber(n)
+		} else if mm, ok := v.(map[string]interface{}); ok {
+			ret[k] = convertMapJSONNumbers(mm)
+		} else {
+			ret[k] = v
+		}
+	}
+
+	return ret
+}

internal/api/json_test.go

@@ -0,0 +1,60 @@
+package api
+
+import (
+	"encoding/json"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestConvertMapJSONNumbers(t *testing.T) {
+	tests := []struct {
+		name     string
+		input    map[string]interface{}
+		expected map[string]interface{}
+	}{
+		{
+			name: "Convert JSON numbers to numbers",
+			input: map[string]interface{}{
+				"int":    json.Number("12"),
+				"float":  json.Number("12.34"),
+				"string": "foo",
+			},
+			expected: map[string]interface{}{
+				"int":    int64(12),
+				"float":  12.34,
+				"string": "foo",
+			},
+		},
+		{
+			name: "Convert JSON numbers to numbers in nested maps",
+			input: map[string]interface{}{
+				"foo": map[string]interface{}{
+					"int":           json.Number("56"),
+					"float":         json.Number("56.78"),
+					"nested-string": "bar",
+				},
+				"int":    json.Number("12"),
+				"float":  json.Number("12.34"),
+				"string": "foo",
+			},
+			expected: map[string]interface{}{
+				"foo": map[string]interface{}{
+					"int":           int64(56),
+					"float":         56.78,
+					"nested-string": "bar",
+				},
+				"int":    int64(12),
+				"float":  12.34,
+				"string": "foo",
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result := convertMapJSONNumbers(tt.input)
+			assert.Equal(t, tt.expected, result)
+		})
+	}
+}

internal/api/loaders/customfieldsloader_gen.go

@@ -0,0 +1,221 @@
+// Code generated by github.com/vektah/dataloaden, DO NOT EDIT.
+
+package loaders
+
+import (
+	"sync"
+	"time"
+
+	"github.com/stashapp/stash/pkg/models"
+)
+
+// CustomFieldsLoaderConfig captures the config to create a new CustomFieldsLoader
+type CustomFieldsLoaderConfig struct {
+	// Fetch is a method that provides the data for the loader
+	Fetch func(keys []int) ([]models.CustomFieldMap, []error)
+
+	// Wait is how long wait before sending a batch
+	Wait time.Duration
+
+	// MaxBatch will limit the maximum number of keys to send in one batch, 0 = not limit
+	MaxBatch int
+}
+
+// NewCustomFieldsLoader creates a new CustomFieldsLoader given a fetch, wait, and maxBatch
+func NewCustomFieldsLoader(config CustomFieldsLoaderConfig) *CustomFieldsLoader {
+	return &CustomFieldsLoader{
+		fetch:    config.Fetch,
+		wait:     config.Wait,
+		maxBatch: config.MaxBatch,
+	}
+}
+
+// CustomFieldsLoader batches and caches requests
+type CustomFieldsLoader struct {
+	// this method provides the data for the loader
+	fetch func(keys []int) ([]models.CustomFieldMap, []error)
+
+	// how long to done before sending a batch
+	wait time.Duration
+
+	// this will limit the maximum number of keys to send in one batch, 0 = no limit
+	maxBatch int
+
+	// INTERNAL
+
+	// lazily created cache
+	cache map[int]models.CustomFieldMap
+
+	// the current batch. keys will continue to be collected until timeout is hit,
+	// then everything will be sent to the fetch method and out to the listeners
+	batch *customFieldsLoaderBatch
+
+	// mutex to prevent races
+	mu sync.Mutex
+}
+
+type customFieldsLoaderBatch struct {
+	keys    []int
+	data    []models.CustomFieldMap
+	error   []error
+	closing bool
+	done    chan struct{}
+}
+
+// Load a CustomFieldMap by key, batching and caching will be applied automatically
+func (l *CustomFieldsLoader) Load(key int) (models.CustomFieldMap, error) {
+	return l.LoadThunk(key)()
+}
+
+// LoadThunk returns a function that when called will block waiting for a CustomFieldMap.
+// This method should be used if you want one goroutine to make requests to many
+// different data loaders without blocking until the thunk is called.
+func (l *CustomFieldsLoader) LoadThunk(key int) func() (models.CustomFieldMap, error) {
+	l.mu.Lock()
+	if it, ok := l.cache[key]; ok {
+		l.mu.Unlock()
+		return func() (models.CustomFieldMap, error) {
+			return it, nil
+		}
+	}
+	if l.batch == nil {
+		l.batch = &customFieldsLoaderBatch{done: make(chan struct{})}
+	}
+	batch := l.batch
+	pos := batch.keyIndex(l, key)
+	l.mu.Unlock()
+
+	return func() (models.CustomFieldMap, error) {
+		<-batch.done
+
+		var data models.CustomFieldMap
+		if pos < len(batch.data) {
+			data = batch.data[pos]
+		}
+
+		var err error
+		// its convenient to be able to return a single error for everything
+		if len(batch.error) == 1 {
+			err = batch.error[0]
+		} else if batch.error != nil {
+			err = batch.error[pos]
+		}
+
+		if err == nil {
+			l.mu.Lock()
+			l.unsafeSet(key, data)
+			l.mu.Unlock()
+		}
+
+		return data, err
+	}
+}
+
+// LoadAll fetches many keys at once. It will be broken into appropriate sized
+// sub batches depending on how the loader is configured
+func (l *CustomFieldsLoader) LoadAll(keys []int) ([]models.CustomFieldMap, []error) {
+	results := make([]func() (models.CustomFieldMap, error), len(keys))
+
+	for i, key := range keys {
+		results[i] = l.LoadThunk(key)
+	}
+
+	customFieldMaps := make([]models.CustomFieldMap, len(keys))
+	errors := make([]error, len(keys))
+	for i, thunk := range results {
+		customFieldMaps[i], errors[i] = thunk()
+	}
+	return customFieldMaps, errors
+}
+
+// LoadAllThunk returns a function that when called will block waiting for a CustomFieldMaps.
+// This method should be used if you want one goroutine to make requests to many
+// different data loaders without blocking until the thunk is called.
+func (l *CustomFieldsLoader) LoadAllThunk(keys []int) func() ([]models.CustomFieldMap, []error) {
+	results := make([]func() (models.CustomFieldMap, error), len(keys))
+	for i, key := range keys {
+		results[i] = l.LoadThunk(key)
+	}
+	return func() ([]models.CustomFieldMap, []error) {
+		customFieldMaps := make([]models.CustomFieldMap, len(keys))
+		errors := make([]error, len(keys))
+		for i, thunk := range results {
+			customFieldMaps[i], errors[i] = thunk()
+		}
+		return customFieldMaps, errors
+	}
+}
+
+// Prime the cache with the provided key and value. If the key already exists, no change is made
+// and false is returned.
+// (To forcefully prime the cache, clear the key first with loader.clear(key).prime(key, value).)
+func (l *CustomFieldsLoader) Prime(key int, value models.CustomFieldMap) bool {
+	l.mu.Lock()
+	var found bool
+	if _, found = l.cache[key]; !found {
+		l.unsafeSet(key, value)
+	}
+	l.mu.Unlock()
+	return !found
+}
+
+// Clear the value at key from the cache, if it exists
+func (l *CustomFieldsLoader) Clear(key int) {
+	l.mu.Lock()
+	delete(l.cache, key)
+	l.mu.Unlock()
+}
+
+func (l *CustomFieldsLoader) unsafeSet(key int, value models.CustomFieldMap) {
+	if l.cache == nil {
+		l.cache = map[int]models.CustomFieldMap{}
+	}
+	l.cache[key] = value
+}
+
+// keyIndex will return the location of the key in the batch, if its not found
+// it will add the key to the batch
+func (b *customFieldsLoaderBatch) keyIndex(l *CustomFieldsLoader, key int) int {
+	for i, existingKey := range b.keys {
+		if key == existingKey {
+			return i
+		}
+	}
+
+	pos := len(b.keys)
+	b.keys = append(b.keys, key)
+	if pos == 0 {
+		go b.startTimer(l)
+	}
+
+	if l.maxBatch != 0 && pos >= l.maxBatch-1 {
+		if !b.closing {
+			b.closing = true
+			l.batch = nil
+			go b.end(l)
+		}
+	}
+
+	return pos
+}
+
+func (b *customFieldsLoaderBatch) startTimer(l *CustomFieldsLoader) {
+	time.Sleep(l.wait)
+	l.mu.Lock()
+
+	// we must have hit a batch limit and are already finalizing this batch
+	if b.closing {
+		l.mu.Unlock()
+		return
+	}
+
+	l.batch = nil
+	l.mu.Unlock()
+
+	b.end(l)
+}
+
+func (b *customFieldsLoaderBatch) end(l *CustomFieldsLoader) {
+	b.data, b.error = l.fetch(b.keys)
+	close(b.done)
+}

internal/api/loaders/dataloaders.go

@@ -13,6 +13,7 @@
 //go:generate go run github.com/vektah/dataloaden SceneFileIDsLoader int []github.com/stashapp/stash/pkg/models.FileID
 //go:generate go run github.com/vektah/dataloaden ImageFileIDsLoader int []github.com/stashapp/stash/pkg/models.FileID
 //go:generate go run github.com/vektah/dataloaden GalleryFileIDsLoader int []github.com/stashapp/stash/pkg/models.FileID
+//go:generate go run github.com/vektah/dataloaden CustomFieldsLoader int github.com/stashapp/stash/pkg/models.CustomFieldMap
 //go:generate go run github.com/vektah/dataloaden SceneOCountLoader int int
 //go:generate go run github.com/vektah/dataloaden ScenePlayCountLoader int int
 //go:generate go run github.com/vektah/dataloaden SceneOHistoryLoader int []time.Time
@@ -51,13 +52,16 @@ type Loaders struct {
 	ImageFiles   *ImageFileIDsLoader
 	GalleryFiles *GalleryFileIDsLoader
 
-	GalleryByID   *GalleryLoader
-	ImageByID     *ImageLoader
-	PerformerByID *PerformerLoader
-	StudioByID    *StudioLoader
-	TagByID       *TagLoader
-	GroupByID     *GroupLoader
-	FileByID      *FileLoader
+	GalleryByID *GalleryLoader
+	ImageByID   *ImageLoader
+
+	PerformerByID         *PerformerLoader
+	PerformerCustomFields *CustomFieldsLoader
+
+	StudioByID *StudioLoader
+	TagByID    *TagLoader
+	GroupByID  *GroupLoader
+	FileByID   *FileLoader
 }
 
 type Middleware struct {
@@ -88,6 +92,11 @@ func (m Middleware) Middleware(next http.Handler) http.Handler {
 				maxBatch: maxBatch,
 				fetch:    m.fetchPerformers(ctx),
 			},
+			PerformerCustomFields: &CustomFieldsLoader{
+				wait:     wait,
+				maxBatch: maxBatch,
+				fetch:    m.fetchPerformerCustomFields(ctx),
+			},
 			StudioByID: &StudioLoader{
 				wait:     wait,
 				maxBatch: maxBatch,
@@ -214,6 +223,18 @@ func (m Middleware) fetchPerformers(ctx context.Context) func(keys []int) ([]*mo
 	}
 }
 
+func (m Middleware) fetchPerformerCustomFields(ctx context.Context) func(keys []int) ([]models.CustomFieldMap, []error) {
+	return func(keys []int) (ret []models.CustomFieldMap, errs []error) {
+		err := m.Repository.WithDB(ctx, func(ctx context.Context) error {
+			var err error
+			ret, err = m.Repository.Performer.GetCustomFieldsBulk(ctx, keys)
+			return err
+		})
+
+		return ret, toErrorSlice(err)
+	}
+}
+
 func (m Middleware) fetchStudios(ctx context.Context) func(keys []int) ([]*models.Studio, []error) {
 	return func(keys []int) (ret []*models.Studio, errs []error) {
 		err := m.Repository.WithDB(ctx, func(ctx context.Context) error {

internal/api/resolver_model_performer.go

@@ -268,6 +268,19 @@ func (r *performerResolver) Groups(ctx context.Context, obj *models.Performer) (
 	return ret, nil
 }
 
+func (r *performerResolver) CustomFields(ctx context.Context, obj *models.Performer) (map[string]interface{}, error) {
+	m, err := loaders.From(ctx).PerformerCustomFields.Load(obj.ID)
+	if err != nil {
+		return nil, err
+	}
+
+	if m == nil {
+		return make(map[string]interface{}), nil
+	}
+
+	return m, nil
+}
+
 // deprecated
 func (r *performerResolver) Movies(ctx context.Context, obj *models.Performer) (ret []*models.Group, err error) {
 	return r.Groups(ctx, obj)

internal/api/resolver_mutation_configure.go

@@ -645,13 +645,13 @@ func (r *mutationResolver) ConfigureUI(ctx context.Context, input map[string]int
 
 	if input != nil {
 		// #5483 - convert JSON numbers to float64 or int64
-		input = utils.ConvertMapJSONNumbers(input)
+		input = convertMapJSONNumbers(input)
 		c.SetUIConfiguration(input)
 	}
 
 	if partial != nil {
 		// #5483 - convert JSON numbers to float64 or int64
-		partial = utils.ConvertMapJSONNumbers(partial)
+		partial = convertMapJSONNumbers(partial)
 		// merge partial into existing config
 		existing := c.GetUIConfiguration()
 		utils.MergeMaps(existing, partial)
@@ -672,9 +672,9 @@ func (r *mutationResolver) ConfigureUISetting(ctx context.Context, key string, v
 
 	// #5483 - convert JSON numbers to float64 or int64
 	if m, ok := value.(map[string]interface{}); ok {
-		value = utils.ConvertMapJSONNumbers(m)
+		value = convertMapJSONNumbers(m)
 	} else if n, ok := value.(json.Number); ok {
-		value = utils.JSONNumberToNumber(n)
+		value = jsonNumberToNumber(n)
 	}
 
 	cfg.Set(key, value)
@@ -686,7 +686,7 @@ func (r *mutationResolver) ConfigurePlugin(ctx context.Context, pluginID string,
 	c := config.GetInstance()
 
 	// #5483 - convert JSON numbers to float64 or int64
-	input = utils.ConvertMapJSONNumbers(input)
+	input = convertMapJSONNumbers(input)
 	c.SetPluginConfiguration(pluginID, input)
 
 	if err := c.Write(); err != nil {

internal/api/resolver_mutation_performer.go

@@ -108,7 +108,13 @@ func (r *mutationResolver) PerformerCreate(ctx context.Context, input models.Per
 			return err
 		}
 
-		err = qb.Create(ctx, &newPerformer)
+		i := &models.CreatePerformerInput{
+			Performer: &newPerformer,
+			// convert json.Numbers to int/float
+			CustomFields: convertMapJSONNumbers(input.CustomFields),
+		}
+
+		err = qb.Create(ctx, i)
 		if err != nil {
 			return err
 		}
@@ -290,6 +296,11 @@ func (r *mutationResolver) PerformerUpdate(ctx context.Context, input models.Per
 		return nil, fmt.Errorf("converting tag ids: %w", err)
 	}
 
+	updatedPerformer.CustomFields = input.CustomFields
+	// convert json.Numbers to int/float
+	updatedPerformer.CustomFields.Full = convertMapJSONNumbers(updatedPerformer.CustomFields.Full)
+	updatedPerformer.CustomFields.Partial = convertMapJSONNumbers(updatedPerformer.CustomFields.Partial)
+
 	var imageData []byte
 	imageIncluded := translator.hasField("image")
 	if input.Image != nil {