Set PYTHONPATH environment variable for Python script scrapers (#4372)

* Set PYTHONPATH environment variable for Python script scrapers
* Convert PYTHONPATH to absolute
* Generalise and apply to plugins

---------

Co-authored-by: WithoutPants <53250216+WithoutPants@users.noreply.github.com>

pkg/plugin/raw.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"io"
 	"os/exec"
+	"path/filepath"
 	"strings"
 	"sync"
 
@@ -52,6 +53,9 @@ func (t *rawPluginTask) Start() error {
 			logger.Warnf("%s", err)
 		} else {
 			cmd = p.Command(context.TODO(), command[1:])
+
+			envVariable, _ := filepath.Abs(filepath.Dir(filepath.Dir(t.plugin.path)))
+			python.AppendPythonPath(cmd, envVariable)
 		}
 	}
 

pkg/python/env.go

@@ -0,0 +1,15 @@
+package python
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+)
+
+func AppendPythonPath(cmd *exec.Cmd, path string) {
+	// Respect the users PYTHONPATH if set
+	if currentValue, set := os.LookupEnv("PYTHONPATH"); set {
+		path = fmt.Sprintf("%s%c%s", currentValue, os.PathListSeparator, path)
+	}
+	cmd.Env = append(os.Environ(), fmt.Sprintf("PYTHONPATH=%s", path))
+}

pkg/scraper/script.go

@@ -44,6 +44,8 @@ func (s *scriptScraper) runScraperScript(ctx context.Context, inString string, o
 			logger.Warnf("%s", err)
 		} else {
 			cmd = p.Command(context.TODO(), command[1:])
+			envVariable, _ := filepath.Abs(filepath.Dir(filepath.Dir(s.config.path)))
+			python.AppendPythonPath(cmd, envVariable)
 		}
 	}