diff --git a/pkg/scraper/mapped.go b/pkg/scraper/mapped.go index 5945f6570..87a040141 100644 --- a/pkg/scraper/mapped.go +++ b/pkg/scraper/mapped.go @@ -12,6 +12,7 @@ import ( "github.com/stashapp/stash/pkg/logger" "github.com/stashapp/stash/pkg/models" + "github.com/stashapp/stash/pkg/utils" "gopkg.in/yaml.v2" ) @@ -73,7 +74,9 @@ func (s mappedConfig) postProcess(q mappedQuery, attrConfig mappedScraperAttrCon result := attrConfig.concatenateResults(found) result = attrConfig.postProcess(result, q) if attrConfig.hasSplit() { - return attrConfig.splitString(result) + results := attrConfig.splitString(result) + results = attrConfig.distinctResults(results) + return results } ret = []string{result} @@ -86,6 +89,7 @@ func (s mappedConfig) postProcess(q mappedQuery, attrConfig mappedScraperAttrCon ret = append(ret, text) } + ret = attrConfig.distinctResults(ret) } return ret @@ -639,6 +643,10 @@ func (c mappedScraperAttrConfig) concatenateResults(nodes []string) string { return strings.Join(result, separator) } +func (c mappedScraperAttrConfig) distinctResults(nodes []string) []string { + return utils.StrUnique(nodes) +} + func (c mappedScraperAttrConfig) splitString(value string) []string { separator := c.Split var res []string diff --git a/pkg/scraper/xpath_test.go b/pkg/scraper/xpath_test.go index 4a2d94ed4..5983bd7a0 100644 --- a/pkg/scraper/xpath_test.go +++ b/pkg/scraper/xpath_test.go @@ -163,6 +163,8 @@ const htmlDoc1 = `