Change performer height to be numeric (#3060)

* Make height an int. Add height_cm field
* Change UI to use height_cm
* Use number fields for height/weight
* Add migration note

pkg/models/filter.go

@@ -85,12 +85,30 @@ type StringCriterionInput struct {
 	Modifier CriterionModifier `json:"modifier"`
 }
 
+func (i StringCriterionInput) ValidModifier() bool {
+	switch i.Modifier {
+	case CriterionModifierEquals, CriterionModifierNotEquals, CriterionModifierIncludes, CriterionModifierExcludes, CriterionModifierMatchesRegex, CriterionModifierNotMatchesRegex,
+		CriterionModifierIsNull, CriterionModifierNotNull:
+		return true
+	}
+
+	return false
+}
+
 type IntCriterionInput struct {
 	Value    int               `json:"value"`
 	Value2   *int              `json:"value2"`
 	Modifier CriterionModifier `json:"modifier"`
 }
 
+func (i IntCriterionInput) ValidModifier() bool {
+	switch i.Modifier {
+	case CriterionModifierEquals, CriterionModifierNotEquals, CriterionModifierGreaterThan, CriterionModifierLessThan, CriterionModifierIsNull, CriterionModifierNotNull, CriterionModifierBetween, CriterionModifierNotBetween:
+		return true
+	}
+	return false
+}
+
 type ResolutionCriterionInput struct {
 	Value    ResolutionEnum    `json:"value"`
 	Modifier CriterionModifier `json:"modifier"`

pkg/models/jsonschema/performer.go

@@ -11,15 +11,16 @@ import (
 )
 
 type Performer struct {
-	Name          string           `json:"name,omitempty"`
-	Gender        string           `json:"gender,omitempty"`
-	URL           string           `json:"url,omitempty"`
-	Twitter       string           `json:"twitter,omitempty"`
-	Instagram     string           `json:"instagram,omitempty"`
-	Birthdate     string           `json:"birthdate,omitempty"`
-	Ethnicity     string           `json:"ethnicity,omitempty"`
-	Country       string           `json:"country,omitempty"`
-	EyeColor      string           `json:"eye_color,omitempty"`
+	Name      string `json:"name,omitempty"`
+	Gender    string `json:"gender,omitempty"`
+	URL       string `json:"url,omitempty"`
+	Twitter   string `json:"twitter,omitempty"`
+	Instagram string `json:"instagram,omitempty"`
+	Birthdate string `json:"birthdate,omitempty"`
+	Ethnicity string `json:"ethnicity,omitempty"`
+	Country   string `json:"country,omitempty"`
+	EyeColor  string `json:"eye_color,omitempty"`
+	// this should be int, but keeping string for backwards compatibility
 	Height        string           `json:"height,omitempty"`
 	Measurements  string           `json:"measurements,omitempty"`
 	FakeTits      string           `json:"fake_tits,omitempty"`

pkg/models/model_performer.go

@@ -18,7 +18,7 @@ type Performer struct {
 	Ethnicity     string     `json:"ethnicity"`
 	Country       string     `json:"country"`
 	EyeColor      string     `json:"eye_color"`
-	Height        string     `json:"height"`
+	Height        *int       `json:"height"`
 	Measurements  string     `json:"measurements"`
 	FakeTits      string     `json:"fake_tits"`
 	CareerLength  string     `json:"career_length"`
@@ -50,7 +50,7 @@ type PerformerPartial struct {
 	Ethnicity     OptionalString
 	Country       OptionalString
 	EyeColor      OptionalString
-	Height        OptionalString
+	Height        OptionalInt
 	Measurements  OptionalString
 	FakeTits      OptionalString
 	CareerLength  OptionalString

pkg/models/performer.go

@@ -79,8 +79,10 @@ type PerformerFilterType struct {
 	Country *StringCriterionInput `json:"country"`
 	// Filter by eye color
 	EyeColor *StringCriterionInput `json:"eye_color"`
-	// Filter by height
+	// Filter by height - deprecated: use height_cm instead
 	Height *StringCriterionInput `json:"height"`
+	// Filter by height in centimeters
+	HeightCm *IntCriterionInput `json:"height_cm"`
 	// Filter by measurements
 	Measurements *StringCriterionInput `json:"measurements"`
 	// Filter by fake tits value

pkg/performer/export.go

@@ -3,6 +3,7 @@ package performer
 import (
 	"context"
 	"fmt"
+	"strconv"
 
 	"github.com/stashapp/stash/pkg/models"
 	"github.com/stashapp/stash/pkg/models/json"
@@ -24,7 +25,6 @@ func ToJSON(ctx context.Context, reader ImageStashIDGetter, performer *models.Pe
 		Ethnicity:     performer.Ethnicity,
 		Country:       performer.Country,
 		EyeColor:      performer.EyeColor,
-		Height:        performer.Height,
 		Measurements:  performer.Measurements,
 		FakeTits:      performer.FakeTits,
 		CareerLength:  performer.CareerLength,
@@ -50,6 +50,11 @@ func ToJSON(ctx context.Context, reader ImageStashIDGetter, performer *models.Pe
 	if performer.DeathDate != nil {
 		newPerformerJSON.DeathDate = performer.DeathDate.String()
 	}
+
+	if performer.Height != nil {
+		newPerformerJSON.Height = strconv.Itoa(*performer.Height)
+	}
+
 	if performer.Weight != nil {
 		newPerformerJSON.Weight = *performer.Weight
 	}

pkg/performer/export_test.go

@@ -2,6 +2,7 @@ package performer
 
 import (
 	"errors"
+	"strconv"
 
 	"github.com/stashapp/stash/pkg/hash/md5"
 	"github.com/stashapp/stash/pkg/models"
@@ -30,7 +31,6 @@ const (
 	eyeColor      = "eyeColor"
 	fakeTits      = "fakeTits"
 	gender        = "gender"
-	height        = "height"
 	instagram     = "instagram"
 	measurements  = "measurements"
 	piercings     = "piercings"
@@ -44,6 +44,7 @@ const (
 
 var (
 	rating = 5
+	height = 123
 	weight = 60
 )
 
@@ -82,7 +83,7 @@ func createFullPerformer(id int, name string) *models.Performer {
 		FakeTits:      fakeTits,
 		Favorite:      true,
 		Gender:        gender,
-		Height:        height,
+		Height:        &height,
 		Instagram:     instagram,
 		Measurements:  measurements,
 		Piercings:     piercings,
@@ -120,7 +121,7 @@ func createFullJSONPerformer(name string, image string) *jsonschema.Performer {
 		FakeTits:     fakeTits,
 		Favorite:     true,
 		Gender:       gender,
-		Height:       height,
+		Height:       strconv.Itoa(height),
 		Instagram:    instagram,
 		Measurements: measurements,
 		Piercings:    piercings,

pkg/performer/import.go

@@ -3,9 +3,11 @@ package performer
 import (
 	"context"
 	"fmt"
+	"strconv"
 	"strings"
 
 	"github.com/stashapp/stash/pkg/hash/md5"
+	"github.com/stashapp/stash/pkg/logger"
 	"github.com/stashapp/stash/pkg/models"
 	"github.com/stashapp/stash/pkg/models/jsonschema"
 	"github.com/stashapp/stash/pkg/sliceutil/stringslice"
@@ -194,7 +196,6 @@ func performerJSONToPerformer(performerJSON jsonschema.Performer) models.Perform
 		Ethnicity:     performerJSON.Ethnicity,
 		Country:       performerJSON.Country,
 		EyeColor:      performerJSON.EyeColor,
-		Height:        performerJSON.Height,
 		Measurements:  performerJSON.Measurements,
 		FakeTits:      performerJSON.FakeTits,
 		CareerLength:  performerJSON.CareerLength,
@@ -235,5 +236,14 @@ func performerJSONToPerformer(performerJSON jsonschema.Performer) models.Perform
 		newPerformer.Weight = &performerJSON.Weight
 	}
 
+	if performerJSON.Height != "" {
+		h, err := strconv.Atoi(performerJSON.Height)
+		if err == nil {
+			newPerformer.Height = &h
+		} else {
+			logger.Warnf("error parsing height %q: %v", performerJSON.Height, err)
+		}
+	}
+
 	return newPerformer
 }

pkg/scraper/stashbox/stash_box.go

@@ -975,8 +975,9 @@ func (c Client) SubmitPerformerDraft(ctx context.Context, performer *models.Perf
 	if performer.HairColor != "" {
 		draft.HairColor = &performer.HairColor
 	}
-	if performer.Height != "" {
-		draft.Height = &performer.Height
+	if performer.Height != nil {
+		v := strconv.Itoa(*performer.Height)
+		draft.Height = &v
 	}
 	if performer.Measurements != "" {
 		draft.Measurements = &performer.Measurements

pkg/sqlite/database.go

@@ -22,7 +22,7 @@ import (
 	"github.com/stashapp/stash/pkg/logger"
 )
 
-var appSchemaVersion uint = 38
+var appSchemaVersion uint = 39
 
 //go:embed migrations/*.sql
 var migrationsBox embed.FS

pkg/sqlite/migrations/39_performer_height.up.sql

@@ -0,0 +1,103 @@
+-- add primary keys to association tables that are missing them
+PRAGMA foreign_keys=OFF;
+
+CREATE TABLE `performers_new` (
+  `id` integer not null primary key autoincrement,
+  `checksum` varchar(255) not null,
+  `name` varchar(255),
+  `gender` varchar(20),
+  `url` varchar(255),
+  `twitter` varchar(255),
+  `instagram` varchar(255),
+  `birthdate` date,
+  `ethnicity` varchar(255),
+  `country` varchar(255),
+  `eye_color` varchar(255),
+  -- changed from varchar(255)
+  `height` int,
+  `measurements` varchar(255),
+  `fake_tits` varchar(255),
+  `career_length` varchar(255),
+  `tattoos` varchar(255),
+  `piercings` varchar(255),
+  `aliases` varchar(255),
+  `favorite` boolean not null default '0',
+  `created_at` datetime not null,
+  `updated_at` datetime not null,
+  `details` text, 
+  `death_date` date, 
+  `hair_color` varchar(255), 
+  `weight` integer, 
+  `rating` tinyint, 
+  `ignore_auto_tag` boolean not null default '0'
+);
+
+INSERT INTO `performers_new`
+  (
+    `id`,
+    `checksum`,
+    `name`,
+    `gender`,
+    `url`,
+    `twitter`,
+    `instagram`,
+    `birthdate`,
+    `ethnicity`,
+    `country`,
+    `eye_color`,
+    `height`,
+    `measurements`,
+    `fake_tits`,
+    `career_length`,
+    `tattoos`,
+    `piercings`,
+    `aliases`,
+    `favorite`,
+    `created_at`,
+    `updated_at`,
+    `details`,
+    `death_date`,
+    `hair_color`,
+    `weight`,
+    `rating`,
+    `ignore_auto_tag`
+  )
+  SELECT 
+    `id`,
+    `checksum`,
+    `name`,
+    `gender`,
+    `url`,
+    `twitter`,
+    `instagram`,
+    `birthdate`,
+    `ethnicity`,
+    `country`,
+    `eye_color`,
+    CASE `height`
+      WHEN '' THEN NULL
+      WHEN NULL THEN NULL
+      ELSE CAST(`height` as int)
+    END,
+    `measurements`,
+    `fake_tits`,
+    `career_length`,
+    `tattoos`,
+    `piercings`,
+    `aliases`,
+    `favorite`,
+    `created_at`,
+    `updated_at`,
+    `details`,
+    `death_date`,
+    `hair_color`,
+    `weight`,
+    `rating`,
+    `ignore_auto_tag`
+  FROM `performers`;
+
+DROP TABLE `performers`;
+ALTER TABLE `performers_new` rename to `performers`;
+
+CREATE UNIQUE INDEX `performers_checksum_unique` on `performers` (`checksum`);
+CREATE INDEX `index_performers_on_name` on `performers` (`name`);

pkg/sqlite/performer.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"database/sql"
 	"fmt"
+	"strconv"
 	"strings"
 
 	"github.com/doug-martin/goqu/v9"
@@ -33,7 +34,7 @@ type performerRow struct {
 	Ethnicity     zero.String            `db:"ethnicity"`
 	Country       zero.String            `db:"country"`
 	EyeColor      zero.String            `db:"eye_color"`
-	Height        zero.String            `db:"height"`
+	Height        null.Int               `db:"height"`
 	Measurements  zero.String            `db:"measurements"`
 	FakeTits      zero.String            `db:"fake_tits"`
 	CareerLength  zero.String            `db:"career_length"`
@@ -67,7 +68,7 @@ func (r *performerRow) fromPerformer(o models.Performer) {
 	r.Ethnicity = zero.StringFrom(o.Ethnicity)
 	r.Country = zero.StringFrom(o.Country)
 	r.EyeColor = zero.StringFrom(o.EyeColor)
-	r.Height = zero.StringFrom(o.Height)
+	r.Height = intFromPtr(o.Height)
 	r.Measurements = zero.StringFrom(o.Measurements)
 	r.FakeTits = zero.StringFrom(o.FakeTits)
 	r.CareerLength = zero.StringFrom(o.CareerLength)
@@ -100,7 +101,7 @@ func (r *performerRow) resolve() *models.Performer {
 		Ethnicity:     r.Ethnicity.String,
 		Country:       r.Country.String,
 		EyeColor:      r.EyeColor.String,
-		Height:        r.Height.String,
+		Height:        nullIntPtr(r.Height),
 		Measurements:  r.Measurements.String,
 		FakeTits:      r.FakeTits.String,
 		CareerLength:  r.CareerLength.String,
@@ -136,7 +137,7 @@ func (r *performerRowRecord) fromPartial(o models.PerformerPartial) {
 	r.setNullString("ethnicity", o.Ethnicity)
 	r.setNullString("country", o.Country)
 	r.setNullString("eye_color", o.EyeColor)
-	r.setNullString("height", o.Height)
+	r.setNullInt("height", o.Height)
 	r.setNullString("measurements", o.Measurements)
 	r.setNullString("fake_tits", o.FakeTits)
 	r.setNullString("career_length", o.CareerLength)
@@ -445,6 +446,22 @@ func (qb *PerformerStore) validateFilter(filter *models.PerformerFilterType) err
 		return qb.validateFilter(filter.Not)
 	}
 
+	// if legacy height filter used, ensure only supported modifiers are used
+	if filter.Height != nil {
+		// treat as an int filter
+		intCrit := &models.IntCriterionInput{
+			Modifier: filter.Height.Modifier,
+		}
+		if !intCrit.ValidModifier() {
+			return fmt.Errorf("invalid height modifier: %s", filter.Height.Modifier)
+		}
+
+		// ensure value is a valid number
+		if _, err := strconv.Atoi(filter.Height.Value); err != nil {
+			return fmt.Errorf("invalid height value: %s", filter.Height.Value)
+		}
+	}
+
 	return nil
 }
 
@@ -483,7 +500,19 @@ func (qb *PerformerStore) makeFilter(ctx context.Context, filter *models.Perform
 	query.handleCriterion(ctx, stringCriterionHandler(filter.Ethnicity, tableName+".ethnicity"))
 	query.handleCriterion(ctx, stringCriterionHandler(filter.Country, tableName+".country"))
 	query.handleCriterion(ctx, stringCriterionHandler(filter.EyeColor, tableName+".eye_color"))
-	query.handleCriterion(ctx, stringCriterionHandler(filter.Height, tableName+".height"))
+
+	// special handler for legacy height filter
+	heightCmCrit := filter.HeightCm
+	if heightCmCrit == nil && filter.Height != nil {
+		heightCm, _ := strconv.Atoi(filter.Height.Value) // already validated
+		heightCmCrit = &models.IntCriterionInput{
+			Value:    heightCm,
+			Modifier: filter.Height.Modifier,
+		}
+	}
+
+	query.handleCriterion(ctx, intCriterionHandler(heightCmCrit, tableName+".height", nil))
+
 	query.handleCriterion(ctx, stringCriterionHandler(filter.Measurements, tableName+".measurements"))
 	query.handleCriterion(ctx, stringCriterionHandler(filter.FakeTits, tableName+".fake_tits"))
 	query.handleCriterion(ctx, stringCriterionHandler(filter.CareerLength, tableName+".career_length"))

pkg/sqlite/performer_test.go

@@ -30,7 +30,7 @@ func Test_PerformerStore_Update(t *testing.T) {
 		ethnicity     = "ethnicity"
 		country       = "country"
 		eyeColor      = "eyeColor"
-		height        = "height"
+		height        = 134
 		measurements  = "measurements"
 		fakeTits      = "fakeTits"
 		careerLength  = "careerLength"
@@ -67,7 +67,7 @@ func Test_PerformerStore_Update(t *testing.T) {
 				Ethnicity:     ethnicity,
 				Country:       country,
 				EyeColor:      eyeColor,
-				Height:        height,
+				Height:        &height,
 				Measurements:  measurements,
 				FakeTits:      fakeTits,
 				CareerLength:  careerLength,
@@ -133,7 +133,7 @@ func Test_PerformerStore_UpdatePartial(t *testing.T) {
 		ethnicity     = "ethnicity"
 		country       = "country"
 		eyeColor      = "eyeColor"
-		height        = "height"
+		height        = 143
 		measurements  = "measurements"
 		fakeTits      = "fakeTits"
 		careerLength  = "careerLength"
@@ -172,7 +172,7 @@ func Test_PerformerStore_UpdatePartial(t *testing.T) {
 				Ethnicity:     models.NewOptionalString(ethnicity),
 				Country:       models.NewOptionalString(country),
 				EyeColor:      models.NewOptionalString(eyeColor),
-				Height:        models.NewOptionalString(height),
+				Height:        models.NewOptionalInt(height),
 				Measurements:  models.NewOptionalString(measurements),
 				FakeTits:      models.NewOptionalString(fakeTits),
 				CareerLength:  models.NewOptionalString(careerLength),
@@ -201,7 +201,7 @@ func Test_PerformerStore_UpdatePartial(t *testing.T) {
 				Ethnicity:     ethnicity,
 				Country:       country,
 				EyeColor:      eyeColor,
-				Height:        height,
+				Height:        &height,
 				Measurements:  measurements,
 				FakeTits:      fakeTits,
 				CareerLength:  careerLength,
@@ -506,29 +506,62 @@ func TestPerformerIllegalQuery(t *testing.T) {
 		},
 	}
 
-	performerFilter := &models.PerformerFilterType{
-		And: &subFilter,
-		Or:  &subFilter,
+	tests := []struct {
+		name   string
+		filter models.PerformerFilterType
+	}{
+		{
+			// And and Or in the same filter
+			"AndOr",
+			models.PerformerFilterType{
+				And: &subFilter,
+				Or:  &subFilter,
+			},
+		},
+		{
+			// And and Not in the same filter
+			"AndNot",
+			models.PerformerFilterType{
+				And: &subFilter,
+				Not: &subFilter,
+			},
+		},
+		{
+			// Or and Not in the same filter
+			"OrNot",
+			models.PerformerFilterType{
+				Or:  &subFilter,
+				Not: &subFilter,
+			},
+		},
+		{
+			"invalid height modifier",
+			models.PerformerFilterType{
+				Height: &models.StringCriterionInput{
+					Modifier: models.CriterionModifierMatchesRegex,
+					Value:    "123",
+				},
+			},
+		},
+		{
+			"invalid height value",
+			models.PerformerFilterType{
+				Height: &models.StringCriterionInput{
+					Modifier: models.CriterionModifierEquals,
+					Value:    "foo",
+				},
+			},
+		},
 	}
 
-	withTxn(func(ctx context.Context) error {
-		sqb := db.Performer
+	sqb := db.Performer
 
-		_, _, err := sqb.Query(ctx, performerFilter, nil)
-		assert.NotNil(err)
-
-		performerFilter.Or = nil
-		performerFilter.Not = &subFilter
-		_, _, err = sqb.Query(ctx, performerFilter, nil)
-		assert.NotNil(err)
-
-		performerFilter.And = nil
-		performerFilter.Or = &subFilter
-		_, _, err = sqb.Query(ctx, performerFilter, nil)
-		assert.NotNil(err)
-
-		return nil
-	})
+	for _, tt := range tests {
+		runWithRollbackTxn(t, tt.name, func(t *testing.T, ctx context.Context) {
+			_, _, err := sqb.Query(ctx, &tt.filter, nil)
+			assert.NotNil(err)
+		})
+	}
 }
 
 func TestPerformerQueryIgnoreAutoTag(t *testing.T) {

pkg/sqlite/sql.go

@@ -178,7 +178,7 @@ func getIntWhereClause(column string, modifier models.CriterionModifier, value i
 		return fmt.Sprintf("%s > ?", column), args
 	}
 
-	panic("unsupported int modifier type")
+	panic("unsupported int modifier type " + modifier)
 }
 
 // returns where clause and having clause