Get distinct values from scraper (#1338)

Co-authored-by: WithoutPants <53250216+WithoutPants@users.noreply.github.com>

pkg/utils/string_collections.go

@@ -35,6 +35,40 @@ func StrMap(vs []string, f func(string) string) []string {
 	return vsm
 }
 
+// StrAppendUnique appends toAdd to the vs string slice if toAdd does not already
+// exist in the slice. It returns the new or unchanged string slice.
+func StrAppendUnique(vs []string, toAdd string) []string {
+	if StrInclude(vs, toAdd) {
+		return vs
+	}
+
+	return append(vs, toAdd)
+}
+
+// StrAppendUniques appends a slice of string values to the vs string slice. It only
+// appends values that do not already exist in the slice. It returns the new or
+// unchanged string slice.
+func StrAppendUniques(vs []string, toAdd []string) []string {
+	for _, v := range toAdd {
+		vs = StrAppendUnique(vs, v)
+	}
+
+	return vs
+}
+
+// StrUnique returns the vs string slice with non-unique values removed.
+func StrUnique(vs []string) []string {
+	distinctValues := make(map[string]struct{})
+	var ret []string
+	for _, v := range vs {
+		if _, exists := distinctValues[v]; !exists {
+			distinctValues[v] = struct{}{}
+			ret = append(ret, v)
+		}
+	}
+	return ret
+}
+
 // StringSliceToIntSlice converts a slice of strings to a slice of ints.
 // Returns an error if any values cannot be parsed.
 func StringSliceToIntSlice(ss []string) ([]int, error) {