Find scrapers in subdirectories (#554)

This commit is contained in:
WithoutPants
2020-05-19 08:44:33 +10:00
committed by GitHub
parent 5450fe8e9a
commit 05488d59c3

View File

@@ -2,6 +2,7 @@ package scraper
import ( import (
"errors" "errors"
"os"
"path/filepath" "path/filepath"
"strconv" "strconv"
@@ -21,7 +22,13 @@ func loadScrapers() ([]scraperConfig, error) {
scrapers = make([]scraperConfig, 0) scrapers = make([]scraperConfig, 0)
logger.Debugf("Reading scraper configs from %s", path) logger.Debugf("Reading scraper configs from %s", path)
scraperFiles, err := filepath.Glob(filepath.Join(path, "*.yml")) scraperFiles := []string{}
err := filepath.Walk(path, func(fp string, f os.FileInfo, err error) error {
if filepath.Ext(fp) == ".yml" {
scraperFiles = append(scraperFiles, fp)
}
return nil
})
if err != nil { if err != nil {
logger.Errorf("Error reading scraper configs: %s", err.Error()) logger.Errorf("Error reading scraper configs: %s", err.Error())