Merge branch 'master' into version

pkg/api/resolver_mutation_configure.go

@@ -36,6 +36,20 @@ func (r *mutationResolver) ConfigureGeneral(ctx context.Context, input models.Co
 		config.Set(config.Generated, input.GeneratedPath)
 	}
 
+	if input.Username != nil {
+		config.Set(config.Username, input.Username)
+	}
+
+	if input.Password != nil {
+		// bit of a hack - check if the passed in password is the same as the stored hash
+		// and only set if they are different
+		currentPWHash := config.GetPasswordHash()
+
+		if *input.Password != currentPWHash {
+			config.SetPassword(*input.Password)
+		}
+	}
+
 	if err := config.Write(); err != nil {
 		return makeConfigGeneralResult(), err
 	}

pkg/api/resolver_query_configuration.go

@@ -32,6 +32,8 @@ func makeConfigGeneralResult() *models.ConfigGeneralResult {
 		Stashes:       config.GetStashPaths(),
 		DatabasePath:  config.GetDatabasePath(),
 		GeneratedPath: config.GetGeneratedPath(),
+		Username:      config.GetUsername(),
+		Password:      config.GetPasswordHash(),
 	}
 }
 

pkg/api/resolver_query_metadata.go

@@ -28,5 +28,6 @@ func (r *queryResolver) MetadataGenerate(ctx context.Context, input models.Gener
 }
 
 func (r *queryResolver) MetadataClean(ctx context.Context) (string, error) {
-	panic("not implemented")
+	manager.GetInstance().Clean()
+	return "todo", nil
 }

pkg/api/server.go

@@ -35,6 +35,32 @@ var uiBox *packr.Box
 //var legacyUiBox *packr.Box
 var setupUIBox *packr.Box
 
+func authenticateHandler() func(http.Handler) http.Handler {
+	return func(next http.Handler) http.Handler {
+		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			// only do this if credentials have been configured
+			if !config.HasCredentials() {
+				next.ServeHTTP(w, r)
+				return
+			}
+
+			authUser, authPW, ok := r.BasicAuth()
+
+			if !ok || !config.ValidateCredentials(authUser, authPW) {
+				unauthorized(w)
+				return
+			}
+
+			next.ServeHTTP(w, r)
+		})
+	}
+}
+
+func unauthorized(w http.ResponseWriter) {
+	w.Header().Add("WWW-Authenticate", `Basic realm=\"Stash\"`)
+	w.WriteHeader(http.StatusUnauthorized)
+}
+
 func Start() {
 	uiBox = packr.New("UI Box", "../../ui/v2/build")
 	//legacyUiBox = packr.New("UI Box", "../../ui/v1/dist/stash-frontend")
@@ -42,6 +68,7 @@ func Start() {
 
 	r := chi.NewRouter()
 
+	r.Use(authenticateHandler())
 	r.Use(middleware.Recoverer)
 	r.Use(middleware.Logger)
 	r.Use(middleware.DefaultCompress)

pkg/manager/config/config.go

@@ -1,6 +1,8 @@
 package config
 
 import (
+	"golang.org/x/crypto/bcrypt"
+
 	"io/ioutil"
 	"github.com/spf13/viper"
 
@@ -12,6 +14,8 @@ const Cache = "cache"
 const Generated = "generated"
 const Metadata = "metadata"
 const Downloads = "downloads"
+const Username = "username"
+const Password = "password"
 
 const Database = "database"
 
@@ -24,6 +28,15 @@ func Set(key string, value interface{}) {
 	viper.Set(key, value)
 }
 
+func SetPassword(value string) {
+	// if blank, don't bother hashing; we want it to be blank
+	if value == "" {
+		Set(Password, "")
+	} else {
+		Set(Password, hashPassword(value))
+	}
+}
+
 func Write() error {
 	return viper.WriteConfig()
 }
@@ -56,6 +69,52 @@ func GetPort() int {
 	return viper.GetInt(Port)
 }
 
+func GetUsername() string {
+	return viper.GetString(Username)
+}
+
+func GetPasswordHash() string {
+	return viper.GetString(Password)
+}
+
+func GetCredentials() (string, string) {
+	if HasCredentials() {
+		return viper.GetString(Username), viper.GetString(Password)
+	}
+
+	return "", ""
+}
+
+func HasCredentials() bool {
+	if !viper.IsSet(Username) || !viper.IsSet(Password) {
+		return false
+	}
+
+	username := GetUsername()
+	pwHash := GetPasswordHash()
+
+	return username != "" && pwHash != ""
+}
+
+func hashPassword(password string) string {
+	hash, _ := bcrypt.GenerateFromPassword([]byte(password), bcrypt.MinCost)
+
+	return string(hash)
+}
+
+func ValidateCredentials(username string, password string) bool {
+	if !HasCredentials() {
+		// don't need to authenticate if no credentials saved
+		return true
+	}
+
+	authUser, authPWHash := GetCredentials()
+
+	err := bcrypt.CompareHashAndPassword([]byte(authPWHash), []byte(password))
+
+	return username == authUser && err == nil
+}
+
 func GetCSSPath() string {
 	// search for custom.css in current directory, then $HOME/.stash
 	fn := "custom.css"
@@ -98,6 +157,7 @@ func GetCSSEnabled() bool {
 
 func IsValid() bool {
 	setPaths := viper.IsSet(Stash) && viper.IsSet(Cache) && viper.IsSet(Generated) && viper.IsSet(Metadata)
+
 	// TODO: check valid paths
 	return setPaths
 }

pkg/manager/manager_tasks.go

@@ -133,6 +133,41 @@ func (s *singleton) Generate(sprites bool, previews bool, markers bool, transcod
 	}()
 }
 
+func (s *singleton) Clean() {
+	if s.Status != Idle {
+		return
+	}
+	s.Status = Clean
+
+	qb := models.NewSceneQueryBuilder()
+	go func() {
+		defer s.returnToIdleState()
+
+		logger.Infof("Starting cleaning of tracked files")
+		scenes, err := qb.All()
+		if err != nil {
+			logger.Errorf("failed to fetch list of scenes for cleaning")
+			return
+		}
+
+		var wg sync.WaitGroup
+		for _, scene := range scenes {
+			if scene == nil {
+				logger.Errorf("nil scene, skipping generate")
+				continue
+			}
+
+			wg.Add(1)
+
+			task := CleanTask{Scene: *scene}
+			go task.Start(&wg)
+			wg.Wait()
+		}
+
+		logger.Info("Finished Cleaning")
+	}()
+}
+
 func (s *singleton) returnToIdleState() {
 	if r := recover(); r != nil {
 		logger.Info("recovered from ", r)

pkg/manager/task_clean.go

@@ -0,0 +1,149 @@
+package manager
+
+import (
+	"context"
+	"github.com/stashapp/stash/pkg/database"
+	"github.com/stashapp/stash/pkg/logger"
+	"github.com/stashapp/stash/pkg/models"
+	"github.com/stashapp/stash/pkg/utils"
+	"os"
+	"path/filepath"
+	"strconv"
+	"sync"
+)
+
+type CleanTask struct {
+	Scene models.Scene
+}
+
+func (t *CleanTask) Start(wg *sync.WaitGroup) {
+	defer wg.Done()
+
+	if t.fileExists(t.Scene.Path) {
+		logger.Debugf("File Found: %s", t.Scene.Path)
+	} else {
+		logger.Infof("File not found. Cleaning: %s", t.Scene.Path)
+		t.deleteScene(t.Scene.ID)
+	}
+}
+
+func (t *CleanTask) deleteScene(sceneID int) {
+	ctx := context.TODO()
+	qb := models.NewSceneQueryBuilder()
+	jqb := models.NewJoinsQueryBuilder()
+	tx := database.DB.MustBeginTx(ctx, nil)
+	strSceneID := strconv.Itoa(sceneID)
+	defer tx.Commit()
+
+	//check and make sure it still exists. scene is also used to delete generated files
+	scene, err := qb.Find(sceneID)
+	if err != nil {
+		_ = tx.Rollback()
+	}
+
+	if err := jqb.DestroyScenesTags(sceneID, tx); err != nil {
+		_ = tx.Rollback()
+	}
+
+	if err := jqb.DestroyPerformersScenes(sceneID, tx); err != nil {
+		_ = tx.Rollback()
+	}
+
+	if err := jqb.DestroyScenesMarkers(sceneID, tx); err != nil {
+		_ = tx.Rollback()
+	}
+
+	if err := jqb.DestroyScenesGalleries(sceneID, tx); err != nil {
+		_ = tx.Rollback()
+	}
+
+	if err := qb.Destroy(strSceneID, tx); err != nil {
+		_ = tx.Rollback()
+	}
+
+	t.deleteGeneratedSceneFiles(scene)
+}
+
+
+func (t *CleanTask)  deleteGeneratedSceneFiles(scene *models.Scene) {
+	markersFolder := filepath.Join(instance.Paths.Generated.Markers, scene.Checksum)
+
+	exists, _ := utils.FileExists(markersFolder)
+	if exists {
+		err := os.RemoveAll(markersFolder)
+		if err != nil {
+			logger.Warnf("Could not delete file %s: %s", scene.Path, err.Error())
+		}
+	}
+
+	thumbPath := instance.Paths.Scene.GetThumbnailScreenshotPath(scene.Checksum)
+	exists, _ = utils.FileExists(thumbPath)
+	if exists {
+		err := os.Remove(thumbPath)
+		if err != nil {
+			logger.Warnf("Could not delete file %s: %s", thumbPath, err.Error())
+		}
+	}
+
+	screenshotPath := instance.Paths.Scene.GetScreenshotPath(scene.Checksum)
+	exists, _ = utils.FileExists(screenshotPath)
+	if exists {
+		err := os.Remove(screenshotPath)
+		if err != nil {
+			logger.Warnf("Could not delete file %s: %s", screenshotPath, err.Error())
+		}
+	}
+
+	streamPreviewPath := instance.Paths.Scene.GetStreamPreviewPath(scene.Checksum)
+	exists, _ = utils.FileExists(streamPreviewPath)
+	if exists {
+		err := os.Remove(streamPreviewPath)
+		if err != nil {
+			logger.Warnf("Could not delete file %s: %s", streamPreviewPath, err.Error())
+		}
+	}
+
+	streamPreviewImagePath := instance.Paths.Scene.GetStreamPreviewImagePath(scene.Checksum)
+	exists, _ = utils.FileExists(streamPreviewImagePath)
+	if exists {
+		err := os.Remove(streamPreviewImagePath)
+		if err != nil {
+			logger.Warnf("Could not delete file %s: %s", streamPreviewImagePath, err.Error())
+		}
+	}
+
+	transcodePath := instance.Paths.Scene.GetTranscodePath(scene.Checksum)
+	exists, _ = utils.FileExists(transcodePath)
+	if exists {
+		err := os.Remove(transcodePath)
+		if err != nil {
+			logger.Warnf("Could not delete file %s: %s", transcodePath, err.Error())
+		}
+	}
+
+	spritePath := instance.Paths.Scene.GetSpriteImageFilePath(scene.Checksum)
+	exists, _ = utils.FileExists(spritePath)
+	if exists {
+		err := os.Remove(spritePath)
+		if err != nil {
+			logger.Warnf("Could not delete file %s: %s", spritePath, err.Error())
+		}
+	}
+
+	vttPath := instance.Paths.Scene.GetSpriteVttFilePath(scene.Checksum)
+	exists, _ = utils.FileExists(vttPath)
+	if exists {
+		err := os.Remove(vttPath)
+		if err != nil {
+			logger.Warnf("Could not delete file %s: %s", vttPath, err.Error())
+		}
+	}
+}
+
+func (t *CleanTask) fileExists(filename string) bool {
+	info, err := os.Stat(filename)
+	if os.IsNotExist(err) {
+		return false
+	}
+	return !info.IsDir()
+}

pkg/manager/task_scan.go

@@ -81,7 +81,8 @@ func (t *ScanTask) scanScene() {
 	qb := models.NewSceneQueryBuilder()
 	scene, _ := qb.FindByPath(t.FilePath)
 	if scene != nil {
-		// We already have this item in the database, keep going
+		// We already have this item in the database, check for thumbnails,screenshots
+		t.makeScreenshots(nil, scene.Checksum)
 		return
 	}
 
@@ -102,7 +103,7 @@ func (t *ScanTask) scanScene() {
 		return
 	}
 
-	t.makeScreenshots(*videoFile, checksum)
+	t.makeScreenshots(videoFile, checksum)
 
 	scene, _ = qb.FindByChecksum(checksum)
 	ctx := context.TODO()
@@ -150,19 +151,38 @@ func (t *ScanTask) scanScene() {
 	}
 }
 
-func (t *ScanTask) makeScreenshots(probeResult ffmpeg.VideoFile, checksum string) {
+func (t *ScanTask) makeScreenshots(probeResult *ffmpeg.VideoFile, checksum string) {
 	thumbPath := instance.Paths.Scene.GetThumbnailScreenshotPath(checksum)
 	normalPath := instance.Paths.Scene.GetScreenshotPath(checksum)
 
 	thumbExists, _ := utils.FileExists(thumbPath)
 	normalExists, _ := utils.FileExists(normalPath)
+
 	if thumbExists && normalExists {
 		logger.Debug("Screenshots already exist for this path... skipping")
 		return
 	}
 
-	t.makeScreenshot(probeResult, thumbPath, 5, 320)
-	t.makeScreenshot(probeResult, normalPath, 2, probeResult.Width)
+	if probeResult == nil {
+		var err error
+		probeResult, err = ffmpeg.NewVideoFile(instance.FFProbePath, t.FilePath)
+
+		if err != nil {
+			logger.Error(err.Error())
+			return
+		}
+		logger.Infof("Regenerating images for %s", t.FilePath)
+	}
+
+	if !thumbExists {
+		logger.Debugf("Creating thumbnail for %s", t.FilePath)
+		t.makeScreenshot(*probeResult, thumbPath, 5, 320)
+	}
+
+	if !normalExists {
+		logger.Debugf("Creating screenshot for %s", t.FilePath)
+		t.makeScreenshot(*probeResult, normalPath, 2, probeResult.Width)
+	}
 }
 
 func (t *ScanTask) makeScreenshot(probeResult ffmpeg.VideoFile, outputPath string, quality int, width int) {

pkg/models/querybuilder_joins.go

@@ -33,6 +33,14 @@ func (qb *JoinsQueryBuilder) UpdatePerformersScenes(sceneID int, updatedJoins []
 	return qb.CreatePerformersScenes(updatedJoins, tx)
 }
 
+func (qb *JoinsQueryBuilder) DestroyPerformersScenes(sceneID int, tx *sqlx.Tx) error {
+	ensureTx(tx)
+
+	// Delete the existing joins
+	_, err := tx.Exec("DELETE FROM performers_scenes WHERE scene_id = ?", sceneID)
+	return err
+}
+
 func (qb *JoinsQueryBuilder) CreateScenesTags(newJoins []ScenesTags, tx *sqlx.Tx) error {
 	ensureTx(tx)
 	for _, join := range newJoins {
@@ -47,6 +55,16 @@ func (qb *JoinsQueryBuilder) CreateScenesTags(newJoins []ScenesTags, tx *sqlx.Tx
 	return nil
 }
 
+func (qb *JoinsQueryBuilder) DestroyScenesTags(sceneID int, tx *sqlx.Tx) error {
+	ensureTx(tx)
+
+	// Delete the existing joins
+	_, err := tx.Exec("DELETE FROM scenes_tags WHERE scene_id = ?", sceneID)
+
+	return err
+}
+
+
 func (qb *JoinsQueryBuilder) UpdateScenesTags(sceneID int, updatedJoins []ScenesTags, tx *sqlx.Tx) error {
 	ensureTx(tx)
 
@@ -82,3 +100,32 @@ func (qb *JoinsQueryBuilder) UpdateSceneMarkersTags(sceneMarkerID int, updatedJo
 	}
 	return qb.CreateSceneMarkersTags(updatedJoins, tx)
 }
+
+func (qb *JoinsQueryBuilder) DestroySceneMarkersTags(sceneMarkerID int, updatedJoins []SceneMarkersTags, tx *sqlx.Tx) error {
+	ensureTx(tx)
+
+	// Delete the existing joins
+	_, err := tx.Exec("DELETE FROM scene_markers_tags WHERE scene_marker_id = ?", sceneMarkerID)
+	return err
+}
+
+func (qb *JoinsQueryBuilder) DestroyScenesGalleries(sceneID int, tx *sqlx.Tx) error {
+	ensureTx(tx)
+
+	// Unset the existing scene id from galleries
+	_, err := tx.Exec("UPDATE galleries SET scene_id = null WHERE scene_id = ?", sceneID)
+
+	return err
+}
+
+func (qb *JoinsQueryBuilder) DestroyScenesMarkers(sceneID int, tx *sqlx.Tx) error {
+	ensureTx(tx)
+
+	// Delete the scene marker tags
+	_, err := tx.Exec("DELETE t FROM scene_markers_tags t join scene_markers m on t.scene_marker_id = m.id WHERE m.scene_id = ?", sceneID)
+
+	// Delete the existing joins
+	_, err = tx.Exec("DELETE FROM scene_markers WHERE scene_id = ?", sceneID)
+
+	return err
+}

pkg/models/querybuilder_scene.go

@@ -74,6 +74,10 @@ func (qb *SceneQueryBuilder) Update(updatedScene ScenePartial, tx *sqlx.Tx) (*Sc
 	return qb.find(updatedScene.ID, tx)
 }
 
+func (qb *SceneQueryBuilder) Destroy(id string, tx *sqlx.Tx) error {
+	return executeDeleteQuery("scenes", id, tx)
+}
+
 func (qb *SceneQueryBuilder) Find(id int) (*Scene, error) {
 	return qb.find(id, nil)
 }
@@ -292,3 +296,4 @@ func (qb *SceneQueryBuilder) queryScenes(query string, args []interface{}, tx *s
 
 	return scenes, nil
 }
+

pkg/models/querybuilder_tag.go

@@ -1,6 +1,7 @@
 package models
 
 import (
+	"errors"
 	"database/sql"
 	"github.com/jmoiron/sqlx"
 	"github.com/stashapp/stash/pkg/database"
@@ -52,6 +53,29 @@ func (qb *TagQueryBuilder) Update(updatedTag Tag, tx *sqlx.Tx) (*Tag, error) {
 }
 
 func (qb *TagQueryBuilder) Destroy(id string, tx *sqlx.Tx) error {
+	// delete tag from scenes and markers first
+	_, err := tx.Exec("DELETE FROM scenes_tags WHERE tag_id = ?", id)
+	if err != nil {
+		return err
+	}
+
+	_, err = tx.Exec("DELETE FROM scene_markers_tags WHERE tag_id = ?", id)
+	if err != nil {
+		return err
+	}
+
+	// cannot unset primary_tag_id in scene_markers because it is not nullable
+	countQuery := "SELECT COUNT(*) as count FROM scene_markers where primary_tag_id = ?"
+	args := []interface{}{id}
+	primaryMarkers, err := runCountQuery(countQuery, args)
+	if err != nil {
+		return err
+	}
+
+	if primaryMarkers > 0 {
+		return errors.New("Cannot delete tag used as a primary tag in scene markers")
+	}
+
 	return executeDeleteQuery("tags", id, tx)
 }
 

pkg/scraper/freeones.go

@@ -66,7 +66,7 @@ func GetPerformer(performerName string) (*models.ScrapedPerformer, error) {
 			return true
 		}
 		alias := s.ParentsFiltered(".babeNameBlock").Find(".babeAlias").First();
-		if strings.EqualFold(alias.Text(), "aka " + performerName) {
+		if strings.Contains( strings.ToLower(alias.Text()), strings.ToLower(performerName) ) {
 			return true
 		}
 		return false
@@ -76,6 +76,10 @@ func GetPerformer(performerName string) (*models.ScrapedPerformer, error) {
 	href = strings.TrimSuffix(href, "/")
 	regex := regexp.MustCompile(`.+_links\/(.+)`)
 	matches := regex.FindStringSubmatch(href)
+	if len(matches) < 2 {
+		return nil, fmt.Errorf("No matches found in %s",href)
+	}
+
 	href = strings.Replace(href, matches[1], "bio_"+matches[1]+".php", -1)
 	href = "https://www.freeones.com" + href
 	
@@ -223,7 +227,7 @@ func getEthnicity(ethnicity string) string {
 
 func paramValue(params *goquery.Selection, paramIndex int) string {
 	i := paramIndex - 1
-	if paramIndex == 0 {
+	if paramIndex <= 0 {
 		return ""
 	}
 	node := params.Get(i).FirstChild