Add unix timestamp parsing to scrapers parseDate (#2817)

* Add unix timestamp parsing to scrapers parseDate
* Add documentation
* Update ScraperDevelopment.md
* Add unit test

Co-authored-by: WithoutPants <53250216+WithoutPants@users.noreply.github.com>
This commit is contained in:
JackDawson94
2022-09-30 07:35:56 +02:00
committed by GitHub
parent 6c6e0b6236
commit 554448594c
4 changed files with 79 additions and 1 deletions

View File

@@ -394,6 +394,19 @@ func (p *postProcessParseDate) Apply(ctx context.Context, value string, q mapped
return value
}
if parseDate == "unix" {
// try to parse the date using unix timestamp format
// if it fails, then just fall back to the original value
timeAsInt, err := strconv.ParseInt(value, 10, 64)
if err != nil {
logger.Warnf("Error parsing date string '%s' using unix timestamp format : %s", value, err.Error())
return value
}
parsedValue := time.Unix(timeAsInt, 0)
return parsedValue.Format(internalDateFormat)
}
// try to parse the date using the pattern
// if it fails, then just fall back to the original value
parsedValue, err := time.Parse(parseDate, value)