Performer custom fields (#5487)

* Backend changes
* Show custom field values
* Add custom fields table input
* Add custom field filtering
* Add unit tests
* Include custom fields in import/export
* Anonymise performer custom fields
* Move json.Number handler functions to api
* Handle json.Number conversion in api

pkg/sqlite/performer_test.go

@@ -16,6 +16,12 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
+var testCustomFields = map[string]interface{}{
+	"string": "aaa",
+	"int":    int64(123), // int64 to match the type of the field in the database
+	"real":   1.23,
+}
+
 func loadPerformerRelationships(ctx context.Context, expected models.Performer, actual *models.Performer) error {
 	if expected.Aliases.Loaded() {
 		if err := actual.LoadAliases(ctx, db.Performer); err != nil {
@@ -81,57 +87,62 @@ func Test_PerformerStore_Create(t *testing.T) {
 
 	tests := []struct {
 		name      string
-		newObject models.Performer
+		newObject models.CreatePerformerInput
 		wantErr   bool
 	}{
 		{
 			"full",
-			models.Performer{
-				Name:           name,
-				Disambiguation: disambiguation,
-				Gender:         &gender,
-				URLs:           models.NewRelatedStrings(urls),
-				Birthdate:      &birthdate,
-				Ethnicity:      ethnicity,
-				Country:        country,
-				EyeColor:       eyeColor,
-				Height:         &height,
-				Measurements:   measurements,
-				FakeTits:       fakeTits,
-				PenisLength:    &penisLength,
-				Circumcised:    &circumcised,
-				CareerLength:   careerLength,
-				Tattoos:        tattoos,
-				Piercings:      piercings,
-				Favorite:       favorite,
-				Rating:         &rating,
-				Details:        details,
-				DeathDate:      &deathdate,
-				HairColor:      hairColor,
-				Weight:         &weight,
-				IgnoreAutoTag:  ignoreAutoTag,
-				TagIDs:         models.NewRelatedIDs([]int{tagIDs[tagIdx1WithPerformer], tagIDs[tagIdx1WithDupName]}),
-				Aliases:        models.NewRelatedStrings(aliases),
-				StashIDs: models.NewRelatedStashIDs([]models.StashID{
-					{
-						StashID:  stashID1,
-						Endpoint: endpoint1,
-					},
-					{
-						StashID:  stashID2,
-						Endpoint: endpoint2,
-					},
-				}),
-				CreatedAt: createdAt,
-				UpdatedAt: updatedAt,
+			models.CreatePerformerInput{
+				Performer: &models.Performer{
+					Name:           name,
+					Disambiguation: disambiguation,
+					Gender:         &gender,
+					URLs:           models.NewRelatedStrings(urls),
+					Birthdate:      &birthdate,
+					Ethnicity:      ethnicity,
+					Country:        country,
+					EyeColor:       eyeColor,
+					Height:         &height,
+					Measurements:   measurements,
+					FakeTits:       fakeTits,
+					PenisLength:    &penisLength,
+					Circumcised:    &circumcised,
+					CareerLength:   careerLength,
+					Tattoos:        tattoos,
+					Piercings:      piercings,
+					Favorite:       favorite,
+					Rating:         &rating,
+					Details:        details,
+					DeathDate:      &deathdate,
+					HairColor:      hairColor,
+					Weight:         &weight,
+					IgnoreAutoTag:  ignoreAutoTag,
+					TagIDs:         models.NewRelatedIDs([]int{tagIDs[tagIdx1WithPerformer], tagIDs[tagIdx1WithDupName]}),
+					Aliases:        models.NewRelatedStrings(aliases),
+					StashIDs: models.NewRelatedStashIDs([]models.StashID{
+						{
+							StashID:  stashID1,
+							Endpoint: endpoint1,
+						},
+						{
+							StashID:  stashID2,
+							Endpoint: endpoint2,
+						},
+					}),
+					CreatedAt: createdAt,
+					UpdatedAt: updatedAt,
+				},
+				CustomFields: testCustomFields,
 			},
 			false,
 		},
 		{
 			"invalid tag id",
-			models.Performer{
-				Name:   name,
-				TagIDs: models.NewRelatedIDs([]int{invalidID}),
+			models.CreatePerformerInput{
+				Performer: &models.Performer{
+					Name:   name,
+					TagIDs: models.NewRelatedIDs([]int{invalidID}),
+				},
 			},
 			true,
 		},
@@ -155,16 +166,16 @@ func Test_PerformerStore_Create(t *testing.T) {
 
 			assert.NotZero(p.ID)
 
-			copy := tt.newObject
+			copy := *tt.newObject.Performer
 			copy.ID = p.ID
 
 			// load relationships
-			if err := loadPerformerRelationships(ctx, copy, &p); err != nil {
+			if err := loadPerformerRelationships(ctx, copy, p.Performer); err != nil {
 				t.Errorf("loadPerformerRelationships() error = %v", err)
 				return
 			}
 
-			assert.Equal(copy, p)
+			assert.Equal(copy, *p.Performer)
 
 			// ensure can find the performer
 			found, err := qb.Find(ctx, p.ID)
@@ -183,6 +194,15 @@ func Test_PerformerStore_Create(t *testing.T) {
 			}
 			assert.Equal(copy, *found)
 
+			// ensure custom fields are set
+			cf, err := qb.GetCustomFields(ctx, p.ID)
+			if err != nil {
+				t.Errorf("PerformerStore.GetCustomFields() error = %v", err)
+				return
+			}
+
+			assert.Equal(tt.newObject.CustomFields, cf)
+
 			return
 		})
 	}
@@ -228,77 +248,109 @@ func Test_PerformerStore_Update(t *testing.T) {
 
 	tests := []struct {
 		name          string
-		updatedObject *models.Performer
+		updatedObject models.UpdatePerformerInput
 		wantErr       bool
 	}{
 		{
 			"full",
-			&models.Performer{
-				ID:             performerIDs[performerIdxWithGallery],
-				Name:           name,
-				Disambiguation: disambiguation,
-				Gender:         &gender,
-				URLs:           models.NewRelatedStrings(urls),
-				Birthdate:      &birthdate,
-				Ethnicity:      ethnicity,
-				Country:        country,
-				EyeColor:       eyeColor,
-				Height:         &height,
-				Measurements:   measurements,
-				FakeTits:       fakeTits,
-				PenisLength:    &penisLength,
-				Circumcised:    &circumcised,
-				CareerLength:   careerLength,
-				Tattoos:        tattoos,
-				Piercings:      piercings,
-				Favorite:       favorite,
-				Rating:         &rating,
-				Details:        details,
-				DeathDate:      &deathdate,
-				HairColor:      hairColor,
-				Weight:         &weight,
-				IgnoreAutoTag:  ignoreAutoTag,
-				Aliases:        models.NewRelatedStrings(aliases),
-				TagIDs:         models.NewRelatedIDs([]int{tagIDs[tagIdx1WithPerformer], tagIDs[tagIdx1WithDupName]}),
-				StashIDs: models.NewRelatedStashIDs([]models.StashID{
-					{
-						StashID:  stashID1,
-						Endpoint: endpoint1,
-					},
-					{
-						StashID:  stashID2,
-						Endpoint: endpoint2,
-					},
-				}),
-				CreatedAt: createdAt,
-				UpdatedAt: updatedAt,
+			models.UpdatePerformerInput{
+				Performer: &models.Performer{
+					ID:             performerIDs[performerIdxWithGallery],
+					Name:           name,
+					Disambiguation: disambiguation,
+					Gender:         &gender,
+					URLs:           models.NewRelatedStrings(urls),
+					Birthdate:      &birthdate,
+					Ethnicity:      ethnicity,
+					Country:        country,
+					EyeColor:       eyeColor,
+					Height:         &height,
+					Measurements:   measurements,
+					FakeTits:       fakeTits,
+					PenisLength:    &penisLength,
+					Circumcised:    &circumcised,
+					CareerLength:   careerLength,
+					Tattoos:        tattoos,
+					Piercings:      piercings,
+					Favorite:       favorite,
+					Rating:         &rating,
+					Details:        details,
+					DeathDate:      &deathdate,
+					HairColor:      hairColor,
+					Weight:         &weight,
+					IgnoreAutoTag:  ignoreAutoTag,
+					Aliases:        models.NewRelatedStrings(aliases),
+					TagIDs:         models.NewRelatedIDs([]int{tagIDs[tagIdx1WithPerformer], tagIDs[tagIdx1WithDupName]}),
+					StashIDs: models.NewRelatedStashIDs([]models.StashID{
+						{
+							StashID:  stashID1,
+							Endpoint: endpoint1,
+						},
+						{
+							StashID:  stashID2,
+							Endpoint: endpoint2,
+						},
+					}),
+					CreatedAt: createdAt,
+					UpdatedAt: updatedAt,
+				},
 			},
 			false,
 		},
 		{
 			"clear nullables",
-			&models.Performer{
-				ID:       performerIDs[performerIdxWithGallery],
-				Aliases:  models.NewRelatedStrings([]string{}),
-				URLs:     models.NewRelatedStrings([]string{}),
-				TagIDs:   models.NewRelatedIDs([]int{}),
-				StashIDs: models.NewRelatedStashIDs([]models.StashID{}),
+			models.UpdatePerformerInput{
+				Performer: &models.Performer{
+					ID:       performerIDs[performerIdxWithGallery],
+					Aliases:  models.NewRelatedStrings([]string{}),
+					URLs:     models.NewRelatedStrings([]string{}),
+					TagIDs:   models.NewRelatedIDs([]int{}),
+					StashIDs: models.NewRelatedStashIDs([]models.StashID{}),
+				},
 			},
 			false,
 		},
 		{
 			"clear tag ids",
-			&models.Performer{
-				ID:     performerIDs[sceneIdxWithTag],
-				TagIDs: models.NewRelatedIDs([]int{}),
+			models.UpdatePerformerInput{
+				Performer: &models.Performer{
+					ID:     performerIDs[sceneIdxWithTag],
+					TagIDs: models.NewRelatedIDs([]int{}),
+				},
+			},
+			false,
+		},
+		{
+			"set custom fields",
+			models.UpdatePerformerInput{
+				Performer: &models.Performer{
+					ID: performerIDs[performerIdxWithGallery],
+				},
+				CustomFields: models.CustomFieldsInput{
+					Full: testCustomFields,
+				},
+			},
+			false,
+		},
+		{
+			"clear custom fields",
+			models.UpdatePerformerInput{
+				Performer: &models.Performer{
+					ID: performerIDs[performerIdxWithGallery],
+				},
+				CustomFields: models.CustomFieldsInput{
+					Full: map[string]interface{}{},
+				},
 			},
 			false,
 		},
 		{
 			"invalid tag id",
-			&models.Performer{
-				ID:     performerIDs[sceneIdxWithGallery],
-				TagIDs: models.NewRelatedIDs([]int{invalidID}),
+			models.UpdatePerformerInput{
+				Performer: &models.Performer{
+					ID:     performerIDs[sceneIdxWithGallery],
+					TagIDs: models.NewRelatedIDs([]int{invalidID}),
+				},
 			},
 			true,
 		},
@@ -309,9 +361,9 @@ func Test_PerformerStore_Update(t *testing.T) {
 		runWithRollbackTxn(t, tt.name, func(t *testing.T, ctx context.Context) {
 			assert := assert.New(t)
 
-			copy := *tt.updatedObject
+			copy := *tt.updatedObject.Performer
 
-			if err := qb.Update(ctx, tt.updatedObject); (err != nil) != tt.wantErr {
+			if err := qb.Update(ctx, &tt.updatedObject); (err != nil) != tt.wantErr {
 				t.Errorf("PerformerStore.Update() error = %v, wantErr %v", err, tt.wantErr)
 			}
 
@@ -331,6 +383,17 @@ func Test_PerformerStore_Update(t *testing.T) {
 			}
 
 			assert.Equal(copy, *s)
+
+			// ensure custom fields are correct
+			if tt.updatedObject.CustomFields.Full != nil {
+				cf, err := qb.GetCustomFields(ctx, tt.updatedObject.ID)
+				if err != nil {
+					t.Errorf("PerformerStore.GetCustomFields() error = %v", err)
+					return
+				}
+
+				assert.Equal(tt.updatedObject.CustomFields.Full, cf)
+			}
 		})
 	}
 }
@@ -573,6 +636,79 @@ func Test_PerformerStore_UpdatePartial(t *testing.T) {
 	}
 }
 
+func Test_PerformerStore_UpdatePartialCustomFields(t *testing.T) {
+	tests := []struct {
+		name     string
+		id       int
+		partial  models.PerformerPartial
+		expected map[string]interface{} // nil to use the partial
+	}{
+		{
+			"set custom fields",
+			performerIDs[performerIdxWithGallery],
+			models.PerformerPartial{
+				CustomFields: models.CustomFieldsInput{
+					Full: testCustomFields,
+				},
+			},
+			nil,
+		},
+		{
+			"clear custom fields",
+			performerIDs[performerIdxWithGallery],
+			models.PerformerPartial{
+				CustomFields: models.CustomFieldsInput{
+					Full: map[string]interface{}{},
+				},
+			},
+			nil,
+		},
+		{
+			"partial custom fields",
+			performerIDs[performerIdxWithGallery],
+			models.PerformerPartial{
+				CustomFields: models.CustomFieldsInput{
+					Partial: map[string]interface{}{
+						"string":    "bbb",
+						"new_field": "new",
+					},
+				},
+			},
+			map[string]interface{}{
+				"int":       int64(3),
+				"real":      1.3,
+				"string":    "bbb",
+				"new_field": "new",
+			},
+		},
+	}
+	for _, tt := range tests {
+		qb := db.Performer
+
+		runWithRollbackTxn(t, tt.name, func(t *testing.T, ctx context.Context) {
+			assert := assert.New(t)
+
+			_, err := qb.UpdatePartial(ctx, tt.id, tt.partial)
+			if err != nil {
+				t.Errorf("PerformerStore.UpdatePartial() error = %v", err)
+				return
+			}
+
+			// ensure custom fields are correct
+			cf, err := qb.GetCustomFields(ctx, tt.id)
+			if err != nil {
+				t.Errorf("PerformerStore.GetCustomFields() error = %v", err)
+				return
+			}
+			if tt.expected == nil {
+				assert.Equal(tt.partial.CustomFields.Full, cf)
+			} else {
+				assert.Equal(tt.expected, cf)
+			}
+		})
+	}
+}
+
 func TestPerformerFindBySceneID(t *testing.T) {
 	withTxn(func(ctx context.Context) error {
 		pqb := db.Performer
@@ -1042,6 +1178,242 @@ func TestPerformerQuery(t *testing.T) {
 	}
 }
 
+func TestPerformerQueryCustomFields(t *testing.T) {
+	tests := []struct {
+		name        string
+		filter      *models.PerformerFilterType
+		includeIdxs []int
+		excludeIdxs []int
+		wantErr     bool
+	}{
+		{
+			"equals",
+			&models.PerformerFilterType{
+				CustomFields: []models.CustomFieldCriterionInput{
+					{
+						Field:    "string",
+						Modifier: models.CriterionModifierEquals,
+						Value:    []any{getPerformerStringValue(performerIdxWithGallery, "custom")},
+					},
+				},
+			},
+			[]int{performerIdxWithGallery},
+			nil,
+			false,
+		},
+		{
+			"not equals",
+			&models.PerformerFilterType{
+				Name: &models.StringCriterionInput{
+					Value:    getPerformerStringValue(performerIdxWithGallery, "Name"),
+					Modifier: models.CriterionModifierEquals,
+				},
+				CustomFields: []models.CustomFieldCriterionInput{
+					{
+						Field:    "string",
+						Modifier: models.CriterionModifierNotEquals,
+						Value:    []any{getPerformerStringValue(performerIdxWithGallery, "custom")},
+					},
+				},
+			},
+			nil,
+			[]int{performerIdxWithGallery},
+			false,
+		},
+		{
+			"includes",
+			&models.PerformerFilterType{
+				CustomFields: []models.CustomFieldCriterionInput{
+					{
+						Field:    "string",
+						Modifier: models.CriterionModifierIncludes,
+						Value:    []any{getPerformerStringValue(performerIdxWithGallery, "custom")[9:]},
+					},
+				},
+			},
+			[]int{performerIdxWithGallery},
+			nil,
+			false,
+		},
+		{
+			"excludes",
+			&models.PerformerFilterType{
+				Name: &models.StringCriterionInput{
+					Value:    getPerformerStringValue(performerIdxWithGallery, "Name"),
+					Modifier: models.CriterionModifierEquals,
+				},
+				CustomFields: []models.CustomFieldCriterionInput{
+					{
+						Field:    "string",
+						Modifier: models.CriterionModifierExcludes,
+						Value:    []any{getPerformerStringValue(performerIdxWithGallery, "custom")[9:]},
+					},
+				},
+			},
+			nil,
+			[]int{performerIdxWithGallery},
+			false,
+		},
+		{
+			"regex",
+			&models.PerformerFilterType{
+				CustomFields: []models.CustomFieldCriterionInput{
+					{
+						Field:    "string",
+						Modifier: models.CriterionModifierMatchesRegex,
+						Value:    []any{".*13_custom"},
+					},
+				},
+			},
+			[]int{performerIdxWithGallery},
+			nil,
+			false,
+		},
+		{
+			"invalid regex",
+			&models.PerformerFilterType{
+				CustomFields: []models.CustomFieldCriterionInput{
+					{
+						Field:    "string",
+						Modifier: models.CriterionModifierMatchesRegex,
+						Value:    []any{"["},
+					},
+				},
+			},
+			nil,
+			nil,
+			true,
+		},
+		{
+			"not matches regex",
+			&models.PerformerFilterType{
+				Name: &models.StringCriterionInput{
+					Value:    getPerformerStringValue(performerIdxWithGallery, "Name"),
+					Modifier: models.CriterionModifierEquals,
+				},
+				CustomFields: []models.CustomFieldCriterionInput{
+					{
+						Field:    "string",
+						Modifier: models.CriterionModifierNotMatchesRegex,
+						Value:    []any{".*13_custom"},
+					},
+				},
+			},
+			nil,
+			[]int{performerIdxWithGallery},
+			false,
+		},
+		{
+			"invalid not matches regex",
+			&models.PerformerFilterType{
+				CustomFields: []models.CustomFieldCriterionInput{
+					{
+						Field:    "string",
+						Modifier: models.CriterionModifierNotMatchesRegex,
+						Value:    []any{"["},
+					},
+				},
+			},
+			nil,
+			nil,
+			true,
+		},
+		{
+			"null",
+			&models.PerformerFilterType{
+				Name: &models.StringCriterionInput{
+					Value:    getPerformerStringValue(performerIdxWithGallery, "Name"),
+					Modifier: models.CriterionModifierEquals,
+				},
+				CustomFields: []models.CustomFieldCriterionInput{
+					{
+						Field:    "not existing",
+						Modifier: models.CriterionModifierIsNull,
+					},
+				},
+			},
+			[]int{performerIdxWithGallery},
+			nil,
+			false,
+		},
+		{
+			"null",
+			&models.PerformerFilterType{
+				Name: &models.StringCriterionInput{
+					Value:    getPerformerStringValue(performerIdxWithGallery, "Name"),
+					Modifier: models.CriterionModifierEquals,
+				},
+				CustomFields: []models.CustomFieldCriterionInput{
+					{
+						Field:    "string",
+						Modifier: models.CriterionModifierNotNull,
+					},
+				},
+			},
+			[]int{performerIdxWithGallery},
+			nil,
+			false,
+		},
+		{
+			"between",
+			&models.PerformerFilterType{
+				CustomFields: []models.CustomFieldCriterionInput{
+					{
+						Field:    "real",
+						Modifier: models.CriterionModifierBetween,
+						Value:    []any{0.05, 0.15},
+					},
+				},
+			},
+			[]int{performerIdx1WithScene},
+			nil,
+			false,
+		},
+		{
+			"not between",
+			&models.PerformerFilterType{
+				Name: &models.StringCriterionInput{
+					Value:    getPerformerStringValue(performerIdx1WithScene, "Name"),
+					Modifier: models.CriterionModifierEquals,
+				},
+				CustomFields: []models.CustomFieldCriterionInput{
+					{
+						Field:    "real",
+						Modifier: models.CriterionModifierNotBetween,
+						Value:    []any{0.05, 0.15},
+					},
+				},
+			},
+			nil,
+			[]int{performerIdx1WithScene},
+			false,
+		},
+	}
+
+	for _, tt := range tests {
+		runWithRollbackTxn(t, tt.name, func(t *testing.T, ctx context.Context) {
+			assert := assert.New(t)
+
+			performers, _, err := db.Performer.Query(ctx, tt.filter, nil)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("PerformerStore.Query() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+
+			ids := performersToIDs(performers)
+			include := indexesToIDs(performerIDs, tt.includeIdxs)
+			exclude := indexesToIDs(performerIDs, tt.excludeIdxs)
+
+			for _, i := range include {
+				assert.Contains(ids, i)
+			}
+			for _, e := range exclude {
+				assert.NotContains(ids, e)
+			}
+		})
+	}
+}
+
 func TestPerformerQueryPenisLength(t *testing.T) {
 	var upper = 4.0
 
@@ -1172,7 +1544,7 @@ func TestPerformerUpdatePerformerImage(t *testing.T) {
 		performer := models.Performer{
 			Name: name,
 		}
-		err := qb.Create(ctx, &performer)
+		err := qb.Create(ctx, &models.CreatePerformerInput{Performer: &performer})
 		if err != nil {
 			return fmt.Errorf("Error creating performer: %s", err.Error())
 		}
@@ -1680,7 +2052,7 @@ func TestPerformerStashIDs(t *testing.T) {
 		performer := &models.Performer{
 			Name: name,
 		}
-		if err := qb.Create(ctx, performer); err != nil {
+		if err := qb.Create(ctx, &models.CreatePerformerInput{Performer: performer}); err != nil {
 			return fmt.Errorf("Error creating performer: %s", err.Error())
 		}