Add subtractDays pp action to scraper (#1399)

This commit is contained in:
bnkai
2021-05-21 05:20:12 +03:00
committed by GitHub
parent 76019af3e5
commit ab24d0f625
3 changed files with 44 additions and 6 deletions

View File

@@ -393,6 +393,22 @@ func (p *postProcessParseDate) Apply(value string, q mappedQuery) string {
return parsedValue.Format(internalDateFormat)
}
type postProcessSubtractDays bool
func (p *postProcessSubtractDays) Apply(value string, q mappedQuery) string {
const internalDateFormat = "2006-01-02"
i, err := strconv.Atoi(value)
if err != nil {
logger.Warnf("Error parsing day string %s: %s", value, err)
return value
}
dt := time.Now()
dt = dt.AddDate(0, 0, -i)
return dt.Format(internalDateFormat)
}
type postProcessReplace mappedRegexConfigs
func (c *postProcessReplace) Apply(value string, q mappedQuery) string {
@@ -479,12 +495,13 @@ func (p *postProcessLbToKg) Apply(value string, q mappedQuery) string {
}
type mappedPostProcessAction struct {
ParseDate string `yaml:"parseDate"`
Replace mappedRegexConfigs `yaml:"replace"`
SubScraper *mappedScraperAttrConfig `yaml:"subScraper"`
Map map[string]string `yaml:"map"`
FeetToCm bool `yaml:"feetToCm"`
LbToKg bool `yaml:"lbToKg"`
ParseDate string `yaml:"parseDate"`
SubtractDays bool `yaml:"subtractDays"`
Replace mappedRegexConfigs `yaml:"replace"`
SubScraper *mappedScraperAttrConfig `yaml:"subScraper"`
Map map[string]string `yaml:"map"`
FeetToCm bool `yaml:"feetToCm"`
LbToKg bool `yaml:"lbToKg"`
}
func (a mappedPostProcessAction) ToPostProcessAction() (postProcessAction, error) {
@@ -536,6 +553,14 @@ func (a mappedPostProcessAction) ToPostProcessAction() (postProcessAction, error
action := postProcessLbToKg(a.LbToKg)
ret = &action
}
if a.SubtractDays {
if found != "" {
return nil, fmt.Errorf("post-process actions must have a single field, found %s and %s", found, "subtractDays")
}
found = "subtractDays"
action := postProcessSubtractDays(a.SubtractDays)
ret = &action
}
if ret == nil {
return nil, errors.New("invalid post-process action")