Restructure go project (#2356)

* Move main to cmd
* Move api to internal
* Move logger and manager to internal
* Move shell hiding code to separate package
* Decouple job from desktop and utils
* Decouple session from config
* Move static into internal
* Decouple config from dlna
* Move desktop to internal
* Move dlna to internal
* Decouple remaining packages from config
* Move config into internal
* Move jsonschema and paths to models
* Make ffmpeg functions private
* Move file utility methods into fsutil package
* Move symwalk into fsutil
* Move single-use util functions into client package
* Move slice functions to separate packages
* Add env var to suppress windowsgui arg
* Move hash functions into separate package
* Move identify to internal
* Move autotag to internal
* Touch UI when generating backend

pkg/hash/key.go

@@ -0,0 +1,27 @@
+// Package hash provides utility functions for generating hashes from strings and random keys.
+package hash
+
+import (
+	"crypto/rand"
+	"fmt"
+	"hash/fnv"
+)
+
+// GenerateRandomKey generates a random string of length l.
+// It returns an empty string and an error if an error occurs while generating a random number.
+func GenerateRandomKey(l int) (string, error) {
+	b := make([]byte, l)
+	if _, err := rand.Read(b); err != nil {
+		return "", err
+	}
+	return fmt.Sprintf("%x", b), nil
+}
+
+// IntFromString generates a uint64 from a string.
+// Values returned by this function are guaranteed to be the same for equal strings.
+// They are not guaranteed to be unique for different strings.
+func IntFromString(str string) uint64 {
+	h := fnv.New64a()
+	h.Write([]byte(str))
+	return h.Sum64()
+}

pkg/hash/md5/md5.go

@@ -0,0 +1,44 @@
+// Package md5 provides utility functions for generating MD5 hashes.
+package md5
+
+import (
+	"crypto/md5"
+	"fmt"
+	"io"
+	"os"
+)
+
+// FromBytes returns an MD5 checksum string from data.
+func FromBytes(data []byte) string {
+	result := md5.Sum(data)
+	return fmt.Sprintf("%x", result)
+}
+
+// FromString returns an MD5 checksum string from str.
+func FromString(str string) string {
+	data := []byte(str)
+	return FromBytes(data)
+}
+
+// FromFilePath returns an MD5 checksum string for the file at filePath.
+// It returns an empty string and an error if an error occurs opening the file.
+func FromFilePath(filePath string) (string, error) {
+	f, err := os.Open(filePath)
+	if err != nil {
+		return "", err
+	}
+	defer f.Close()
+
+	return FromReader(f)
+}
+
+// FromReader returns an MD5 checksum string from data read from src.
+// It returns an empty string and an error if an error occurs reading from src.
+func FromReader(src io.Reader) (string, error) {
+	h := md5.New()
+	if _, err := io.Copy(h, src); err != nil {
+		return "", err
+	}
+	checksum := h.Sum(nil)
+	return fmt.Sprintf("%x", checksum), nil
+}

pkg/hash/oshash/oshash.go

@@ -0,0 +1,100 @@
+// Package oshash implements the algorithm that OpenSubtitles.org uses to generate unique hashes.
+//
+// Calculation is as follows:
+// size + 64 bit checksum of the first and last 64k bytes of the file.
+package oshash
+
+import (
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+)
+
+const chunkSize int64 = 64 * 1024
+
+var ErrOsHashLen = errors.New("buffer is not a multiple of 8")
+
+func sumBytes(buf []byte) (uint64, error) {
+	if len(buf)%8 != 0 {
+		return 0, ErrOsHashLen
+	}
+
+	sz := len(buf) / 8
+	var sum uint64
+	for j := 0; j < sz; j++ {
+		sum += binary.LittleEndian.Uint64(buf[8*j : 8*(j+1)])
+	}
+
+	return sum, nil
+}
+
+func oshash(size int64, head []byte, tail []byte) (string, error) {
+	headSum, err := sumBytes(head)
+	if err != nil {
+		return "", fmt.Errorf("oshash head: %w", err)
+	}
+	tailSum, err := sumBytes(tail)
+	if err != nil {
+		return "", fmt.Errorf("oshash tail: %w", err)
+	}
+
+	// Compute the sum of the head, tail and file size
+	result := headSum + tailSum + uint64(size)
+	// output as hex
+	return fmt.Sprintf("%016x", result), nil
+}
+
+// FromFilePath calculates the hash reading from src.
+func FromReader(src io.ReadSeeker, fileSize int64) (string, error) {
+	if fileSize == 0 {
+		return "", nil
+	}
+
+	fileChunkSize := chunkSize
+	if fileSize < fileChunkSize {
+		fileChunkSize = fileSize
+	}
+
+	head := make([]byte, fileChunkSize)
+	tail := make([]byte, fileChunkSize)
+
+	// read the head of the file into the start of the buffer
+	_, err := src.Read(head)
+	if err != nil {
+		return "", err
+	}
+
+	// seek to the end of the file - the chunk size
+	_, err = src.Seek(-fileChunkSize, 2)
+	if err != nil {
+		return "", err
+	}
+
+	// read the tail of the file
+	_, err = src.Read(tail)
+	if err != nil {
+		return "", err
+	}
+
+	return oshash(fileSize, head, tail)
+}
+
+// Is the equivalent of opening filePath, and calling FromReader with the data and file size.
+func FromFilePath(filePath string) (string, error) {
+	f, err := os.Open(filePath)
+	if err != nil {
+		return "", err
+	}
+	defer f.Close()
+
+	fi, err := f.Stat()
+	if err != nil {
+		return "", err
+	}
+
+	fileSize := fi.Size()
+
+	return FromReader(f, fileSize)
+}

pkg/hash/oshash/oshash_internal_test.go

@@ -0,0 +1,75 @@
+package oshash
+
+import (
+	"math/rand"
+	"testing"
+)
+
+// Note that the public API returns "" instead.
+func TestOshashEmpty(t *testing.T) {
+	var size int64
+	head := make([]byte, chunkSize)
+	tail := make([]byte, chunkSize)
+	want := "0000000000000000"
+	got, err := oshash(size, head, tail)
+	if err != nil {
+		t.Errorf("TestOshashEmpty: Error from oshash: %v", err)
+	}
+	if got != want {
+		t.Errorf("TestOshashEmpty: oshash(0, 0, 0) = %q; want %q", got, want)
+	}
+}
+
+// As oshash sums byte values, causing collisions is trivial.
+func TestOshashCollisions(t *testing.T) {
+	buf1 := []byte("this is dumb")
+	buf2 := []byte("dumb is this")
+	size := int64(len(buf1))
+	head := make([]byte, chunkSize)
+
+	tail1 := make([]byte, chunkSize)
+	copy(tail1[len(tail1)-len(buf1):], buf1)
+	hash1, err := oshash(size, head, tail1)
+	if err != nil {
+		t.Errorf("TestOshashCollisions: Error from oshash: %v", err)
+	}
+
+	tail2 := make([]byte, chunkSize)
+	copy(tail2[len(tail2)-len(buf2):], buf2)
+	hash2, err := oshash(size, head, tail2)
+	if err != nil {
+		t.Errorf("TestOshashCollisions: Error from oshash: %v", err)
+	}
+
+	if hash1 != hash2 {
+		t.Errorf("TestOshashCollisions: oshash(n, k, ... %v) =! oshash(n, k, ... %v)", buf1, buf2)
+	}
+}
+
+func BenchmarkOsHash(b *testing.B) {
+	src := rand.NewSource(9999)
+	r := rand.New(src)
+
+	size := int64(1234567890)
+
+	head := make([]byte, 1024*64)
+	_, err := r.Read(head)
+	if err != nil {
+		b.Errorf("unable to generate head array: %v", err)
+	}
+
+	tail := make([]byte, 1024*64)
+	_, err = r.Read(tail)
+	if err != nil {
+		b.Errorf("unable to generate tail array: %v", err)
+	}
+
+	b.ResetTimer()
+
+	for n := 0; n < b.N; n++ {
+		_, err := oshash(size, head, tail)
+		if err != nil {
+			b.Errorf("unexpected error: %v", err)
+		}
+	}
+}