diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 765705989..3927d9438 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -68,7 +68,7 @@ jobs: - name: Validate UI # skip UI validation for pull requests if UI is unchanged if: ${{ github.event_name != 'pull_request' || steps.cache-ui.outputs.cache-hit != 'true' }} - run: docker exec -t build /bin/bash -c "make validate-frontend" + run: docker exec -t build /bin/bash -c "make validate-ui" # Static validation happens in the linter workflow in parallel to this workflow # Run Dynamic validation here, to make sure we pass all the projects integration tests diff --git a/Makefile b/Makefile index 3559803ba..701542585 100644 --- a/Makefile +++ b/Makefile @@ -18,13 +18,21 @@ endif LDFLAGS := $(LDFLAGS) # set OUTPUT environment variable to generate a specific binary name +# this will apply to both `stash` and `phasher`, so build them separately +# alternatively use STASH_OUTPUT or PHASHER_OUTPUT to set the value individually ifdef OUTPUT - OUTPUT := -o $(OUTPUT) + STASH_OUTPUT := $(OUTPUT) + PHASHER_OUTPUT := $(OUTPUT) +endif +ifdef STASH_OUTPUT + STASH_OUTPUT := -o $(STASH_OUTPUT) +endif +ifdef PHASHER_OUTPUT + PHASHER_OUTPUT := -o $(PHASHER_OUTPUT) endif # set GO_BUILD_FLAGS environment variable to any extra build flags required GO_BUILD_FLAGS := $(GO_BUILD_FLAGS) -GO_BUILD_FLAGS += -buildmode=pie # set GO_BUILD_TAGS environment variable to any extra build tags required GO_BUILD_TAGS := $(GO_BUILD_TAGS) @@ -41,26 +49,51 @@ export CGO_ENABLED := 1 .PHONY: release release: pre-ui generate ui build-release -.PHONY: pre-build -pre-build: +# targets to set various build flags + +.PHONY: flags-release +flags-release: + $(eval LDFLAGS += -s -w) + $(eval GO_BUILD_FLAGS += -trimpath) + +.PHONY: flags-pie +flags-pie: + $(eval GO_BUILD_FLAGS += -buildmode=pie) + +.PHONY: flags-static +flags-static: + $(eval LDFLAGS += -extldflags=-static) + $(eval GO_BUILD_TAGS += sqlite_omit_load_extension osusergo netgo) + +.PHONY: flags-static-pie +flags-static-pie: + $(eval LDFLAGS += -extldflags=-static-pie) + $(eval GO_BUILD_FLAGS += -buildmode=pie) + $(eval GO_BUILD_TAGS += sqlite_omit_load_extension osusergo netgo) + +.PHONY: flags-static-windows +flags-static-windows: + $(eval LDFLAGS += -extldflags=-static-pie) + $(eval GO_BUILD_FLAGS += -buildmode=pie) + $(eval GO_BUILD_TAGS += sqlite_omit_load_extension osusergo) + +.PHONY: build-info +build-info: ifndef BUILD_DATE $(eval BUILD_DATE := $(shell go run scripts/getDate.go)) endif - ifndef GITHASH $(eval GITHASH := $(shell git rev-parse --short HEAD)) endif - ifndef STASH_VERSION $(eval STASH_VERSION := $(shell git describe --tags --exclude latest_develop)) endif - ifndef OFFICIAL_BUILD $(eval OFFICIAL_BUILD := false) endif .PHONY: build-flags -build-flags: pre-build +build-flags: build-info $(eval BUILD_LDFLAGS := $(LDFLAGS)) $(eval BUILD_LDFLAGS += -X 'github.com/stashapp/stash/internal/build.buildstamp=$(BUILD_DATE)') $(eval BUILD_LDFLAGS += -X 'github.com/stashapp/stash/internal/build.githash=$(GITHASH)') @@ -68,35 +101,59 @@ build-flags: pre-build $(eval BUILD_LDFLAGS += -X 'github.com/stashapp/stash/internal/build.officialBuild=$(OFFICIAL_BUILD)') $(eval BUILD_FLAGS := -v -tags "$(GO_BUILD_TAGS)" $(GO_BUILD_FLAGS) -ldflags "$(BUILD_LDFLAGS)") -# builds a dynamically-linked debug binary -.PHONY: build -build: build-flags -build: - go build $(OUTPUT) $(BUILD_FLAGS) ./cmd/stash +.PHONY: stash +stash: build-flags + go build $(STASH_OUTPUT) $(BUILD_FLAGS) ./cmd/stash + +.PHONY: stash-release +stash-release: flags-release +stash-release: flags-pie +stash-release: stash + +.PHONY: stash-release-static +stash-release-static: flags-release +stash-release-static: flags-static-pie +stash-release-static: stash + +.PHONY: stash-release-static-windows +stash-release-static-windows: flags-release +stash-release-static-windows: flags-static-windows +stash-release-static-windows: stash -# TODO: Integrate the phasher target with the rest of the Makefile, -# TODO: so it can be built as part of normal releases. .PHONY: phasher -phasher: - go build -o $@ -trimpath -buildmode=pie -ldflags '-s -w' ./cmd/phasher +phasher: build-flags + go build $(PHASHER_OUTPUT) $(BUILD_FLAGS) ./cmd/phasher -# builds a dynamically-linked release binary +.PHONY: phasher-release +phasher-release: flags-release +phasher-release: flags-pie +phasher-release: phasher + +.PHONY: phasher-release-static +phasher-release-static: flags-release +phasher-release-static: flags-static-pie +phasher-release-static: phasher + +.PHONY: phasher-release-static-windows +phasher-release-static-windows: flags-release +phasher-release-static-windows: flags-static-windows +phasher-release-static-windows: phasher + +# builds dynamically-linked debug binaries +.PHONY: build +build: stash phasher + +# builds dynamically-linked release binaries .PHONY: build-release -build-release: LDFLAGS += -s -w -build-release: GO_BUILD_FLAGS += -trimpath -build-release: build +build-release: stash-release phasher-release -# builds a statically-linked release binary +# builds statically-linked release binaries .PHONY: build-release-static -build-release-static: GO_BUILD_TAGS += netgo -build-release-static: build-release-static-windows +build-release-static: stash-release-static phasher-release-static # build-release-static, but excluding netgo, which is not needed on windows .PHONY: build-release-static-windows -build-release-static-windows: LDFLAGS += -extldflags=-static -s -w -build-release-static-windows: GO_BUILD_FLAGS += -trimpath -build-release-static-windows: GO_BUILD_TAGS += sqlite_omit_load_extension osusergo -build-release-static-windows: build +build-release-static-windows: stash-release-static-windows phasher-release-static-windows # cross-compile- targets should be run within the compiler docker container .PHONY: cross-compile-windows @@ -104,26 +161,35 @@ cross-compile-windows: export GOOS := windows cross-compile-windows: export GOARCH := amd64 cross-compile-windows: export CC := x86_64-w64-mingw32-gcc cross-compile-windows: export CXX := x86_64-w64-mingw32-g++ -cross-compile-windows: OUTPUT := -o dist/stash-win.exe -cross-compile-windows: build-release-static-windows +cross-compile-windows: STASH_OUTPUT := -o dist/stash-win.exe +cross-compile-windows: PHASHER_OUTPUT := -o dist/phasher-win.exe +cross-compile-windows: flags-release +cross-compile-windows: flags-static-windows +cross-compile-windows: build .PHONY: cross-compile-macos-intel cross-compile-macos-intel: export GOOS := darwin cross-compile-macos-intel: export GOARCH := amd64 cross-compile-macos-intel: export CC := o64-clang cross-compile-macos-intel: export CXX := o64-clang++ -cross-compile-macos-intel: OUTPUT := -o dist/stash-macos-intel +cross-compile-macos-intel: STASH_OUTPUT := -o dist/stash-macos-intel +cross-compile-macos-intel: PHASHER_OUTPUT := -o dist/phasher-macos-intel +cross-compile-macos-intel: flags-release # can't use static build for OSX -cross-compile-macos-intel: build-release +cross-compile-macos-intel: flags-pie +cross-compile-macos-intel: build .PHONY: cross-compile-macos-applesilicon cross-compile-macos-applesilicon: export GOOS := darwin cross-compile-macos-applesilicon: export GOARCH := arm64 cross-compile-macos-applesilicon: export CC := oa64e-clang cross-compile-macos-applesilicon: export CXX := oa64e-clang++ -cross-compile-macos-applesilicon: OUTPUT := -o dist/stash-macos-applesilicon +cross-compile-macos-applesilicon: STASH_OUTPUT := -o dist/stash-macos-applesilicon +cross-compile-macos-applesilicon: PHASHER_OUTPUT := -o dist/phasher-macos-applesilicon +cross-compile-macos-applesilicon: flags-release # can't use static build for OSX -cross-compile-macos-applesilicon: build-release +cross-compile-macos-applesilicon: flags-pie +cross-compile-macos-applesilicon: build .PHONY: cross-compile-macos cross-compile-macos: @@ -143,37 +209,52 @@ cross-compile-macos: .PHONY: cross-compile-freebsd cross-compile-freebsd: export GOOS := freebsd cross-compile-freebsd: export GOARCH := amd64 -cross-compile-freebsd: OUTPUT := -o dist/stash-freebsd -cross-compile-freebsd: build-release-static +cross-compile-freebsd: STASH_OUTPUT := -o dist/stash-freebsd +cross-compile-freebsd: PHASHER_OUTPUT := -o dist/phasher-freebsd +cross-compile-freebsd: flags-release +cross-compile-freebsd: flags-static-pie +cross-compile-freebsd: build .PHONY: cross-compile-linux cross-compile-linux: export GOOS := linux cross-compile-linux: export GOARCH := amd64 -cross-compile-linux: OUTPUT := -o dist/stash-linux -cross-compile-linux: build-release-static +cross-compile-linux: STASH_OUTPUT := -o dist/stash-linux +cross-compile-linux: PHASHER_OUTPUT := -o dist/phasher-linux +cross-compile-linux: flags-release +cross-compile-linux: flags-static-pie +cross-compile-linux: build .PHONY: cross-compile-linux-arm64v8 cross-compile-linux-arm64v8: export GOOS := linux cross-compile-linux-arm64v8: export GOARCH := arm64 cross-compile-linux-arm64v8: export CC := aarch64-linux-gnu-gcc -cross-compile-linux-arm64v8: OUTPUT := -o dist/stash-linux-arm64v8 -cross-compile-linux-arm64v8: build-release-static +cross-compile-linux-arm64v8: STASH_OUTPUT := -o dist/stash-linux-arm64v8 +cross-compile-linux-arm64v8: PHASHER_OUTPUT := -o dist/phasher-linux-arm64v8 +cross-compile-linux-arm64v8: flags-release +cross-compile-linux-arm64v8: flags-static-pie +cross-compile-linux-arm64v8: build .PHONY: cross-compile-linux-arm32v7 cross-compile-linux-arm32v7: export GOOS := linux cross-compile-linux-arm32v7: export GOARCH := arm cross-compile-linux-arm32v7: export GOARM := 7 cross-compile-linux-arm32v7: export CC := arm-linux-gnueabihf-gcc -cross-compile-linux-arm32v7: OUTPUT := -o dist/stash-linux-arm32v7 -cross-compile-linux-arm32v7: build-release-static +cross-compile-linux-arm32v7: STASH_OUTPUT := -o dist/stash-linux-arm32v7 +cross-compile-linux-arm32v7: PHASHER_OUTPUT := -o dist/phasher-linux-arm32v7 +cross-compile-linux-arm32v7: flags-release +cross-compile-linux-arm32v7: flags-static +cross-compile-linux-arm32v7: build .PHONY: cross-compile-linux-arm32v6 cross-compile-linux-arm32v6: export GOOS := linux cross-compile-linux-arm32v6: export GOARCH := arm cross-compile-linux-arm32v6: export GOARM := 6 cross-compile-linux-arm32v6: export CC := arm-linux-gnueabi-gcc -cross-compile-linux-arm32v6: OUTPUT := -o dist/stash-linux-arm32v6 -cross-compile-linux-arm32v6: build-release-static +cross-compile-linux-arm32v6: STASH_OUTPUT := -o dist/stash-linux-arm32v6 +cross-compile-linux-arm32v6: PHASHER_OUTPUT := -o dist/phasher-linux-arm32v6 +cross-compile-linux-arm32v6: flags-release +cross-compile-linux-arm32v6: flags-static +cross-compile-linux-arm32v6: build .PHONY: cross-compile-all cross-compile-all: @@ -197,10 +278,10 @@ endif # Regenerates GraphQL files .PHONY: generate -generate: generate-backend generate-frontend +generate: generate-backend generate-ui -.PHONY: generate-frontend -generate-frontend: +.PHONY: generate-ui +generate-ui: cd ui/v2.5 && yarn run gqlgen .PHONY: generate-backend @@ -264,7 +345,7 @@ pre-ui: cd ui/v2.5 && yarn install --frozen-lockfile .PHONY: ui-env -ui-env: pre-build +ui-env: build-info $(eval export VITE_APP_DATE := $(BUILD_DATE)) $(eval export VITE_APP_GITHASH := $(GITHASH)) $(eval export VITE_APP_STASH_VERSION := $(STASH_VERSION)) @@ -295,29 +376,25 @@ ui-start: ui-env fmt-ui: cd ui/v2.5 && yarn format -# runs tests and checks on the UI and builds it -.PHONY: ui-validate -ui-validate: - cd ui/v2.5 && yarn run validate - -# runs all of the tests and checks required for a PR to be accepted -.PHONY: validate -validate: validate-frontend validate-backend - # runs all of the frontend PR-acceptance steps -.PHONY: validate-frontend -validate-frontend: ui-validate +.PHONY: validate-ui +validate-ui: + cd ui/v2.5 && yarn run validate # runs all of the backend PR-acceptance steps .PHONY: validate-backend validate-backend: lint it +# runs all of the tests and checks required for a PR to be accepted +.PHONY: validate +validate: validate-ui validate-backend + # locally builds and tags a 'stash/build' docker image .PHONY: docker-build -docker-build: pre-build +docker-build: build-info docker build --build-arg GITHASH=$(GITHASH) --build-arg STASH_VERSION=$(STASH_VERSION) -t stash/build -f docker/build/x86_64/Dockerfile . # locally builds and tags a 'stash/cuda-build' docker image .PHONY: docker-cuda-build -docker-cuda-build: pre-build +docker-cuda-build: build-info docker build --build-arg GITHASH=$(GITHASH) --build-arg STASH_VERSION=$(STASH_VERSION) -t stash/cuda-build -f docker/build/x86_64/Dockerfile-CUDA . diff --git a/docker/build/x86_64/Dockerfile b/docker/build/x86_64/Dockerfile index 5133eba36..3e8b70d5b 100644 --- a/docker/build/x86_64/Dockerfile +++ b/docker/build/x86_64/Dockerfile @@ -6,11 +6,11 @@ RUN apk add --no-cache make ## cache node_modules separately COPY ./ui/v2.5/package.json ./ui/v2.5/yarn.lock /stash/ui/v2.5/ WORKDIR /stash -RUN yarn --cwd ui/v2.5 install --frozen-lockfile. +RUN make pre-ui COPY Makefile /stash/ COPY ./graphql /stash/graphql/ COPY ./ui /stash/ui/ -RUN make generate-frontend +RUN make generate-ui ARG GITHASH ARG STASH_VERSION RUN BUILD_DATE=$(date +"%Y-%m-%d %H:%M:%S") make ui @@ -29,7 +29,7 @@ COPY --from=frontend /stash /stash/ RUN make generate-backend ARG GITHASH ARG STASH_VERSION -RUN make build +RUN make stash-release # Final Runnable Image FROM alpine:latest diff --git a/docker/build/x86_64/Dockerfile-CUDA b/docker/build/x86_64/Dockerfile-CUDA index 676e2ead9..6524ab251 100644 --- a/docker/build/x86_64/Dockerfile-CUDA +++ b/docker/build/x86_64/Dockerfile-CUDA @@ -6,7 +6,7 @@ RUN apk add --no-cache make ## cache node_modules separately COPY ./ui/v2.5/package.json ./ui/v2.5/yarn.lock /stash/ui/v2.5/ WORKDIR /stash -RUN yarn --cwd ui/v2.5 install --frozen-lockfile. +RUN make pre-ui COPY Makefile /stash/ COPY ./graphql /stash/graphql/ COPY ./ui /stash/ui/ @@ -29,7 +29,7 @@ COPY --from=frontend /stash /stash/ RUN make generate-backend ARG GITHASH ARG STASH_VERSION -RUN make build +RUN make stash-release # Final Runnable Image FROM nvidia/cuda:12.0.1-base-ubuntu22.04 diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index 94882efa5..a19b9bc2a 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -6,21 +6,18 @@ * [GolangCI](https://golangci-lint.run/) - A meta-linter which runs several linters in parallel * To install, follow the [local installation instructions](https://golangci-lint.run/usage/install/#local-installation) * [Yarn](https://yarnpkg.com/en/docs/install) - Yarn package manager - * Run `yarn install --frozen-lockfile` in the `stash/ui/v2.5` folder (before running make generate for first time). - -NOTE: You may need to run the `go get` commands outside the project directory to avoid modifying the projects module file. ## Environment ### Windows 1. Download and install [Go for Windows](https://golang.org/dl/) -2. Download and extract [MingW64](https://sourceforge.net/projects/mingw-w64/files/) (scroll down and select x86_64-posix-seh, dont use the autoinstaller it doesnt work) -3. Search for "advanced system settings" and open the system properties dialog. +2. Download and extract [MinGW64](https://sourceforge.net/projects/mingw-w64/files/) (scroll down and select x86_64-posix-seh, don't use the autoinstaller, it doesn't work) +3. Search for "Advanced System Settings" and open the System Properties dialog. 1. Click the `Environment Variables` button - 2. Under system variables find the `Path`. Edit and add `C:\MinGW\bin` (replace with the correct path to where you extracted MingW64). + 2. Under System Variables find `Path`. Edit and add `C:\MinGW\bin` (replace with the correct path to where you extracted MingW64). -NOTE: The `make` command in Windows will be `mingw32-make` with MingW. For example `make pre-ui` will be `mingw32-make pre-ui` +NOTE: The `make` command in Windows will be `mingw32-make` with MinGW. For example, `make pre-ui` will be `mingw32-make pre-ui`. ### macOS @@ -30,28 +27,36 @@ NOTE: The `make` command in Windows will be `mingw32-make` with MingW. For examp ### Linux #### Arch Linux + 1. Install dependencies: `sudo pacman -S go git yarn gcc make nodejs ffmpeg --needed` #### Ubuntu + 1. Install dependencies: `sudo apt-get install golang git gcc nodejs ffmpeg -y` 2. Enable corepack in Node.js: `corepack enable` 3. Install yarn: `corepack prepare yarn@stable --activate` ## Commands -* `make pre-ui` - Installs the UI dependencies. Only needs to be run once before building the UI for the first time, or if the dependencies are updated -* `make generate` - Generate Go and UI GraphQL files -* `make fmt-ui` - Formats the UI source code -* `make ui` - Builds the frontend -* `make build` - Builds the binary (make sure to build the UI as well... see below) +* `make pre-ui` - Installs the UI dependencies. This only needs to be run once after cloning the repository, or if the dependencies are updated. +* `make generate` - Generates Go and UI GraphQL files. Requires `make pre-ui` to have been run. +* `make ui` - Builds the UI. Requires `make pre-ui` to have been run. +* `make stash` - Builds the `stash` binary (make sure to build the UI as well... see below) +* `make stash-release` - Builds a release version the `stash` binary, with debug information removed +* `make phasher` - Builds the `phasher` binary +* `make phasher-release` - Builds a release version the `phasher` binary, with debug information removed +* `make build` - Builds both the `stash` and `phasher` binaries +* `make build-release` - Builds release versions of both the `stash` and `phasher` binaries * `make docker-build` - Locally builds and tags a complete 'stash/build' docker image -* `make lint` - Run the linter on the backend -* `make fmt` - Run `go fmt` -* `make it` - Run the unit and integration tests -* `make validate` - Run all of the tests and checks required to submit a PR -* `make server-start` - Runs an instance of the server in the `.local` directory. -* `make server-clean` - Removes the `.local` directory and all of its contents. -* `make ui-start` - Runs the UI in development mode. Requires a running stash server to connect to. Stash server port can be changed from the default of `9999` using environment variable `VITE_APP_PLATFORM_PORT`. UI runs on port `3000` or the next available port. +* `make docker-cuda-build` - Locally builds and tags a complete 'stash/cuda-build' docker image +* `make validate` - Runs all of the tests and checks required to submit a PR +* `make lint` - Runs `golangci-lint` on the backend +* `make it` - Runs all unit and integration tests +* `make fmt` - Formats the Go source code +* `make fmt-ui` - Formats the UI source code +* `make server-start` - Runs a development stash server in the `.local` directory +* `make server-clean` - Removes the `.local` directory and all of its contents +* `make ui-start` - Runs the UI in development mode. Requires a running Stash server to connect to. The server port can be changed from the default of `9999` using the environment variable `VITE_APP_PLATFORM_PORT`. The UI runs on port `3000` or the next available port. ## Local development quickstart @@ -59,13 +64,14 @@ NOTE: The `make` command in Windows will be `mingw32-make` with MingW. For examp 2. Run `make generate` to create generated files 3. In one terminal, run `make server-start` to run the server code 4. In a separate terminal, run `make ui-start` to run the UI in development mode -5. Open the UI in a browser `http://localhost:3000/` +5. Open the UI in a browser: `http://localhost:3000/` Changes to the UI code can be seen by reloading the browser page. -Changes to the server code requires a restart (`CTRL-C` in the server terminal). +Changes to the backend code require a server restart (`CTRL-C` in the server terminal, followed by `make server-start` again) to be seen. On first launch: + 1. On the "Stash Setup Wizard" screen, choose a directory with some files to test with 2. Press "Next" to use the default locations for the database and generated content 3. Press the "Confirm" and "Finish" buttons to get into the UI @@ -73,17 +79,20 @@ On first launch: 5. You're all set! Set any other configurations you'd like and test your code changes. To start fresh with new configuration: + 1. Stop the server (`CTRL-C` in the server terminal) -2. Run `make server-clean` to clear all config, database, and generated files (under `.local/`) +2. Run `make server-clean` to clear all config, database, and generated files (under `.local`) 3. Run `make server-start` to restart the server 4. Follow the "On first launch" steps above ## Building a release +Simply run `make` or `make release`, or equivalently: + 1. Run `make pre-ui` to install UI dependencies 2. Run `make generate` to create generated files -3. Run `make ui` to compile the frontend -4. Run `make build` to build the executable for your current platform +3. Run `make ui` to build the frontend +4. Run `make build-release` to build a release executable for your current platform ## Cross compiling