Replace packr with go embed (#1751)

* Embed performer images
* Embed schema migrations
* Update dependencies
* Embed UI
* Remove remaining packr references
This commit is contained in:
WithoutPants
2021-09-22 13:08:34 +10:00
committed by GitHub
parent f292238e7f
commit 56111433a1
429 changed files with 39923 additions and 23061 deletions

View File

@@ -2,7 +2,6 @@
FAQ.md
README.md
LICENSE
Makefile
.gitignore
.travis.yml
CONTRIBUTING.md

View File

@@ -4,4 +4,7 @@ cli/cli
cli/migrate
.coverage
.godoc.pid
vendor/
vendor/
.vscode/
.idea
dist/

View File

@@ -1,6 +1,6 @@
run:
# timeout for analysis, e.g. 30s, 5m, default is 1m
deadline: 2m
timeout: 5m
linters:
enable:
#- golint

View File

@@ -0,0 +1,98 @@
project_name: migrate
before:
hooks:
- go mod tidy
builds:
- env:
- CGO_ENABLED=0
goos:
- linux
- windows
- darwin
goarch:
- amd64
- arm
- arm64
- 386
goarm:
- 7
main: ./cmd/migrate
ldflags:
- '-w -s -X main.Version={{ .Version }} -extldflags "static"'
flags:
- "-tags={{ .Env.DATABASE }} {{ .Env.SOURCE }}"
- "-trimpath"
nfpms:
- homepage: "https://github.com/golang-migrate/migrate"
maintainer: "dhui@users.noreply.github.com"
license: MIT
description: "Database migrations"
formats:
- deb
file_name_template: "{{ .ProjectName }}.{{ .Os }}-{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}"
dockers:
- goos: linux
goarch: amd64
dockerfile: Dockerfile.github-actions
use: buildx
ids:
- migrate
image_templates:
- 'migrate/migrate:{{ .Tag }}-amd64'
build_flag_templates:
- '--label=org.opencontainers.image.created={{ .Date }}'
- '--label=org.opencontainers.image.title={{ .ProjectName }}'
- '--label=org.opencontainers.image.revision={{ .FullCommit }}'
- '--label=org.opencontainers.image.version={{ .Version }}'
- "--label=org.opencontainers.image.source={{ .GitURL }}"
- "--platform=linux/amd64"
- goos: linux
goarch: arm64
dockerfile: Dockerfile.github-actions
use: buildx
ids:
- migrate
image_templates:
- 'migrate/migrate:{{ .Tag }}-arm64'
build_flag_templates:
- '--label=org.opencontainers.image.created={{ .Date }}'
- '--label=org.opencontainers.image.title={{ .ProjectName }}'
- '--label=org.opencontainers.image.revision={{ .FullCommit }}'
- '--label=org.opencontainers.image.version={{ .Version }}'
- "--label=org.opencontainers.image.source={{ .GitURL }}"
- "--platform=linux/arm64"
docker_manifests:
- name_template: 'migrate/migrate:{{ .Tag }}'
image_templates:
- 'migrate/migrate:{{ .Tag }}-amd64'
- 'migrate/migrate:{{ .Tag }}-arm64'
- name_template: 'migrate/migrate:{{ .Major }}'
image_templates:
- 'migrate/migrate:{{ .Tag }}-amd64'
- 'migrate/migrate:{{ .Tag }}-arm64'
- name_template: 'migrate/migrate:latest'
image_templates:
- 'migrate/migrate:{{ .Tag }}-amd64'
- 'migrate/migrate:{{ .Tag }}-arm64'
archives:
- name_template: "{{ .ProjectName }}.{{ .Os }}-{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}"
format_overrides:
- goos: windows
format: zip
checksum:
name_template: 'sha256sum.txt'
release:
draft: true
changelog:
skip: false
sort: asc
filters:
exclude:
- '^docs:'
- '^test:'
- Merge pull request
- Merge branch
- go mod tidy
snapshot:
name_template: "{{ .Tag }}-next"

View File

@@ -6,8 +6,8 @@ matrix:
- go: master
include:
# Supported versions of Go: https://golang.org/dl/
- go: "1.11.x"
- go: "1.12.x"
- go: "1.14.x"
- go: "1.15.x"
- go: master
go_import_path: github.com/golang-migrate/migrate
@@ -26,8 +26,15 @@ cache:
directories:
- $GOPATH/pkg
before_install:
- curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s -- -b $(go env GOPATH)/bin v1.16.0
# Update docker to latest version: https://docs.travis-ci.com/user/docker/#installing-a-newer-docker-version
- curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
- sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
- sudo apt-get update
- sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce
# Install golangci-lint
- curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s -- -b $(go env GOPATH)/bin v1.30.0
- echo "TRAVIS_GO_VERSION=${TRAVIS_GO_VERSION}"
install:
@@ -50,13 +57,16 @@ deploy:
secure: hWH1HLPpzpfA8pXQ93T1qKQVFSpQp0as/JLQ7D91jHuJ8p+RxVeqblDrR6HQY/95R/nyiE9GJmvUolSuw5h449LSrGxPtVWhdh6EnkxlQHlen5XeMhVjRjFV0sE9qGe8v7uAkiTfRO61ktTWHrEAvw5qpyqnNISodmZS78XIasPODQbNlzwINhWhDTHIjXGb4FpizYaL3OGCanrxfR9fQyCaqKGGBjRq3Mfq8U6Yd4mApmsE+uJxgaZV8K5zBqpkSzQRWhcVGNL5DuLsU3gfSJOo7kZeA2G71SHffH577dBoqtCZ4VFv169CoUZehLWCb+7XKJZmHXVujCURATSySLGUOPc6EoLFAn3YtsCA04mS4bZVo5FZPWVwfhjmkhtDR4f6wscKp7r1HsFHSOgm59QfETQdrn4MnZ44H2Jd39axqndn5DvK9EcZVjPHynOPnueXP2u6mTuUgh2VyyWBCDO3CNo0fGlo7VJI69IkIWNSD87K9cHZWYMClyKZkUzS+PmRAhHRYbVd+9ZjKOmnU36kUHNDG/ft1D4ogsY+rhVtXB4lgWDM5adri+EIScYdYnB1/pQexLBigcJY9uE7nQTR0U6QgVNYvun7uRNs40E0c4voSfmPdFO0FlOD2y1oQhnaXfWLbu9nMcTcs4RFGrcC7NzkUN4/WjG8s285V6w=
skip_cleanup: true
on:
go: "1.12.x"
go: "1.15.x"
repo: golang-migrate/migrate
tags: true
file:
- cli/build/migrate.linux-amd64.tar.gz
- cli/build/migrate.linux-armv7.tar.gz
- cli/build/migrate.linux-arm64.tar.gz
- cli/build/migrate.darwin-amd64.tar.gz
- cli/build/migrate.windows-amd64.exe.tar.gz
- cli/build/migrate.windows-386.exe.tar.gz
- cli/build/sha256sum.txt
- dependency_tree.txt
- provider: packagecloud
@@ -68,7 +78,7 @@ deploy:
package_glob: '*.deb'
skip_cleanup: true
on:
go: "1.12.x"
go: "1.15.x"
repo: golang-migrate/migrate
tags: true
- provider: packagecloud
@@ -80,7 +90,7 @@ deploy:
package_glob: '*.deb'
skip_cleanup: true
on:
go: "1.12.x"
go: "1.15.x"
repo: golang-migrate/migrate
tags: true
- provider: packagecloud
@@ -88,11 +98,11 @@ deploy:
username: golang-migrate
token:
secure: aICwu3gJ1sJ1QVCD3elpg+Jxzt4P+Zj1uoh5f0sOwnjDNIZ4FwUT1cMrWloP8P2KD0iyCOawuZER27o/kQ21oX2OxHvQbYPReA2znLm7lHzCmypAAOHPxpgnQ4rMGHHJXd+OsxtdclGs67c+EbdBfoRRbK400Qz/vjPJEDeH4mh02ZHC2nw4Nk/wV4jjBIkIt9dGEx6NgOA17FCMa3MaPHlHeFIzU7IfTlDHbS0mCCYbg/wafWBWcbGqtZLWAYtJDmfjrAStmDLdAX5J5PsB7taGSGPZHmPmpGoVgrKt/tb9Xz1rFBGslTpGROOiO4CiMAvkEKFn8mxrBGjfSBqp7Dp3eeSalKXB1DJAbEXx2sEbMcvmnoR9o43meaAn+ZRts8lRL8S/skBloe6Nk8bx3NlJCGB9WPK1G56b7c/fZnJxQbrCw6hxDfbZwm8S2YPviFTo/z1BfZDhRsL74reKsN2kgnGo2W/k38vvzIpsssQ9DHN1b0TLCxolCNPtQ7oHcQ1ohcjP2UgYXk0FhqDoL+9LQva/DU4N9sKH0UbAaqsMVSErLeG8A4aauuFcVrWRBaDYyTag4dQqzTulEy7iru2kDDIBgSQ1gMW/yoBOIPK4oi6MtbTf1X39fzXFLS1cDd3LW61yAu3YrbjAetpfx2frIvrRAiL9TxWA1gnrs5o=
dist: ubuntu/cosmic
dist: ubuntu/focal
package_glob: '*.deb'
skip_cleanup: true
on:
go: "1.12.x"
go: "1.15.x"
repo: golang-migrate/migrate
tags: true
- provider: packagecloud
@@ -104,7 +114,7 @@ deploy:
package_glob: '*.deb'
skip_cleanup: true
on:
go: "1.12.x"
go: "1.15.x"
repo: golang-migrate/migrate
tags: true
- provider: packagecloud
@@ -116,13 +126,13 @@ deploy:
package_glob: '*.deb'
skip_cleanup: true
on:
go: "1.12.x"
go: "1.15.x"
repo: golang-migrate/migrate
tags: true
- provider: script
script: ./docker-deploy.sh
skip_cleanup: true
on:
go: "1.12.x"
go: "1.15.x"
repo: golang-migrate/migrate
tags: true

View File

@@ -1,23 +1,26 @@
FROM golang:1.12-alpine3.9 AS downloader
FROM golang:1.16-alpine3.13 AS builder
ARG VERSION
RUN apk add --no-cache git gcc musl-dev
RUN apk add --no-cache git gcc musl-dev make
WORKDIR /go/src/github.com/golang-migrate/migrate
ENV GO111MODULE=on
COPY go.mod go.sum ./
RUN go mod download
COPY . ./
ENV GO111MODULE=on
ENV DATABASES="postgres mysql redshift cassandra spanner cockroachdb clickhouse mongodb"
ENV SOURCES="file go_bindata github aws_s3 google_cloud_storage godoc_vfs gitlab"
RUN make build-docker
RUN go build -a -o build/migrate.linux-386 -ldflags="-X main.Version=${VERSION}" -tags "$DATABASES $SOURCES" ./cmd/migrate
FROM alpine:3.9
FROM alpine:3.13
RUN apk add --no-cache ca-certificates
COPY --from=downloader /go/src/github.com/golang-migrate/migrate/build/migrate.linux-386 /migrate
COPY --from=builder /go/src/github.com/golang-migrate/migrate/build/migrate.linux-386 /usr/local/bin/migrate
RUN ln -s /usr/local/bin/migrate /migrate
ENTRYPOINT ["/migrate"]
ENTRYPOINT ["migrate"]
CMD ["--help"]

View File

@@ -0,0 +1,17 @@
ARG DOCKER_IMAGE
FROM $DOCKER_IMAGE
RUN apk add --no-cache git gcc musl-dev make
WORKDIR /go/src/github.com/golang-migrate/migrate
ENV GO111MODULE=on
ENV COVERAGE_DIR=/tmp/coverage
COPY go.mod go.sum ./
RUN go mod download
COPY . ./
CMD ["make", "test"]

View File

@@ -0,0 +1,8 @@
FROM alpine:3.13
RUN apk add --no-cache ca-certificates
ENTRYPOINT ["/usr/bin/migrate"]
CMD ["--help"]
COPY migrate /usr/bin/migrate

View File

@@ -16,7 +16,7 @@
NilMigration defines a migration without a body. NilVersion is defined as const -1.
#### What is the difference between uint(version) and int(targetVersion)?
version refers to an existing migration version coming from a source and therefor can never be negative.
version refers to an existing migration version coming from a source and therefore can never be negative.
targetVersion can either be a version OR represent a NilVersion, which equals -1.
#### What's the difference between Next/Previous and Up/Down?
@@ -53,7 +53,7 @@
Yes, technically thats possible. We want to encourage you to contribute your driver to this respository though.
The driver's functionality is dictated by migrate's interfaces. That means there should really
just be one driver for a database/ source. We want to prevent a future where several drivers doing the exact same thing,
just implemented a bit differently, co-exist somewhere on Github. If users have to do research first to find the
just implemented a bit differently, co-exist somewhere on GitHub. If users have to do research first to find the
"best" available driver for a database in order to get started, we would have failed as an open source community.
#### Can I mix multiple sources during a batch of migrations?
@@ -68,3 +68,12 @@
Database-specific locking features are used by *some* database drivers to prevent multiple instances of migrate from running migrations at the same time
the same database at the same time. For example, the MySQL driver uses the `GET_LOCK` function, while the Postgres driver uses
the `pg_advisory_lock` function.
#### Do I need to create a table for tracking migration version used?
No, it is done automatically.
#### Can I use migrate with a non-Go project?
Yes, you can use the migrate CLI in a non-Go project, but there are probably other libraries/frameworks available that offer better test and deploy integrations in that language/framework.
#### I have got an error `Dirty database version 1. Fix and force version`. What should I do?
Keep calm and refer to [the getting started docs](GETTING_STARTED.md#forcing-your-database-version).

View File

@@ -0,0 +1,53 @@
# Getting started
Before you start, you should understand the concept of forward/up and reverse/down database migrations.
Configure a database for your application. Make sure that your database driver is supported [here](README.md#databases)
## Create migrations
Create some migrations using migrate CLI. Here is an example:
```
migrate create -ext sql -dir db/migrations -seq create_users_table
```
Once you create your files, you should fill them.
**IMPORTANT:** In a project developed by more than one person there is a chance of migrations inconsistency - e.g. two developers can create conflicting migrations, and the developer that created his migration later gets it merged to the repository first.
Developers and Teams should keep an eye on such cases (especially during code review).
[Here](https://github.com/golang-migrate/migrate/issues/179#issuecomment-475821264) is the issue summary if you would like to read more.
Consider making your migrations idempotent - we can run the same sql code twice in a row with the same result. This makes our migrations more robust. On the other hand, it causes slightly less control over database schema - e.g. let's say you forgot to drop the table in down migration. You run down migration - the table is still there. When you run up migration again - `CREATE TABLE` would return an error, helping you find an issue in down migration, while `CREATE TABLE IF NOT EXISTS` would not. Use those conditions wisely.
In case you would like to run several commands/queries in one migration, you should wrap them in a transaction (if your database supports it).
This way if one of commands fails, our database will remain unchanged.
## Run migrations
Run your migrations through the CLI or your app and check if they applied expected changes.
Just to give you an idea:
```
migrate -database YOUR_DATABASE_URL -path PATH_TO_YOUR_MIGRATIONS up
```
Just add the code to your app and you're ready to go!
Before commiting your migrations you should run your migrations up, down, and then up again to see if migrations are working properly both ways.
(e.g. if you created a table in a migration but reverse migration did not delete it, you will encounter an error when running the forward migration again)
It's also worth checking your migrations in a separate, containerized environment. You can find some tools in the end of this document.
**IMPORTANT:** If you would like to run multiple instances of your app on different machines be sure to use a database that supports locking when running migrations. Otherwise you may encounter issues.
## Forcing your database version
In case you run a migration that contained an error, migrate will not let you run other migrations on the same database. You will see an error like `Dirty database version 1. Fix and force version`, even when you fix the erred migration. This means your database was marked as 'dirty'.
You need to investigate the migration error - was your migration applied partially, or was it not applied at all? Once you know, you should force your database to a version reflecting it's real state. You can do so with `force` command:
```
migrate -path PATH_TO_YOUR_MIGRATIONS -database YOUR_DATABASE_URL force VERSION
```
Once you force the version and your migration was fixed, your database is 'clean' again and you can proceed with your migrations.
For details and example of usage see [this comment](https://github.com/golang-migrate/migrate/issues/282#issuecomment-530743258).
## Further reading:
- [PostgreSQL tutorial](database/postgres/TUTORIAL.md)
- [Best practices](MIGRATIONS.md)
- [FAQ](FAQ.md)
- Tools for testing your migrations in a container:
- https://github.com/dhui/dktest
- https://github.com/ory/dockertest

View File

@@ -44,9 +44,14 @@ It is suggested that the version number of corresponding `up` and `down` migrati
files be equivalent for clarity, but they are allowed to differ so long as the
relative ordering of the migrations is preserved.
The migration files are permitted to be empty, so in the event that a migration
is a no-op or is irreversible, it is recommended to still include both migration
files, and either leaving them empty or adding a comment as appropriate.
The migration files are permitted to be "empty", in the event that a migration
is a no-op or is irreversible. It is recommended to still include both migration
files by making the whole migration file consist of a comment.
If your database does not support comments, then deleting the migration file will also work.
Note, an actual empty file (e.g. a 0 byte file) may cause issues with your database since migrate
will attempt to run an empty query. In this case, deleting the migration file will also work.
For the rational of this behavior see:
[#244 (comment)](https://github.com/golang-migrate/migrate/issues/244#issuecomment-510758270)
## Migration Content Format

View File

@@ -1,15 +1,30 @@
SOURCE ?= file go_bindata github aws_s3 google_cloud_storage godoc_vfs gitlab
DATABASE ?= postgres mysql redshift cassandra spanner cockroachdb clickhouse mongodb
SOURCE ?= file go_bindata github github_ee bitbucket aws_s3 google_cloud_storage godoc_vfs gitlab
DATABASE ?= postgres mysql redshift cassandra spanner cockroachdb clickhouse mongodb sqlserver firebird neo4j pgx
DATABASE_TEST ?= $(DATABASE) sqlite sqlite3 sqlcipher
VERSION ?= $(shell git describe --tags 2>/dev/null | cut -c 2-)
TEST_FLAGS ?=
REPO_OWNER ?= $(shell cd .. && basename "$$(pwd)")
COVERAGE_DIR ?= .coverage
echo-source:
@echo "$(SOURCE)"
echo-database:
@echo "$(DATABASE)"
build:
CGO_ENABLED=0 go build -ldflags='-X main.Version=$(VERSION)' -tags '$(DATABASE) $(SOURCE)' ./cmd/migrate
build-docker:
CGO_ENABLED=0 go build -a -o build/migrate.linux-386 -ldflags="-s -w -X main.Version=${VERSION}" -tags "$(DATABASE) $(SOURCE)" ./cmd/migrate
build-cli: clean
-mkdir ./cli/build
cd ./cmd/migrate && CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o ../../cli/build/migrate.linux-amd64 -ldflags='-X main.Version=$(VERSION) -extldflags "-static"' -tags '$(DATABASE) $(SOURCE)' .
cd ./cmd/migrate && CGO_ENABLED=0 GOOS=linux GOARCH=arm GOARM=7 go build -a -o ../../cli/build/migrate.linux-armv7 -ldflags='-X main.Version=$(VERSION) -extldflags "-static"' -tags '$(DATABASE) $(SOURCE)' .
cd ./cmd/migrate && CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -a -o ../../cli/build/migrate.linux-arm64 -ldflags='-X main.Version=$(VERSION) -extldflags "-static"' -tags '$(DATABASE) $(SOURCE)' .
cd ./cmd/migrate && CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 go build -a -o ../../cli/build/migrate.darwin-amd64 -ldflags='-X main.Version=$(VERSION) -extldflags "-static"' -tags '$(DATABASE) $(SOURCE)' .
cd ./cmd/migrate && CGO_ENABLED=0 GOOS=windows GOARCH=386 go build -a -o ../../cli/build/migrate.windows-386.exe -ldflags='-X main.Version=$(VERSION) -extldflags "-static"' -tags '$(DATABASE) $(SOURCE)' .
cd ./cmd/migrate && CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build -a -o ../../cli/build/migrate.windows-amd64.exe -ldflags='-X main.Version=$(VERSION) -extldflags "-static"' -tags '$(DATABASE) $(SOURCE)' .
cd ./cli/build && find . -name 'migrate*' | xargs -I{} tar czf {}.tar.gz {}
cd ./cli/build && shasum -a 256 * > sha256sum.txt
@@ -27,27 +42,14 @@ test-short:
test:
@-rm -r $(COVERAGE_DIR)
@mkdir $(COVERAGE_DIR)
make test-with-flags TEST_FLAGS='-v -race -covermode atomic -coverprofile $$(COVERAGE_DIR)/_$$(RAND).txt -bench=. -benchmem -timeout 20m'
@echo 'mode: atomic' > $(COVERAGE_DIR)/combined.txt
@cat $(COVERAGE_DIR)/_*.txt | grep -v 'mode: atomic' >> $(COVERAGE_DIR)/combined.txt
make test-with-flags TEST_FLAGS='-v -race -covermode atomic -coverprofile $$(COVERAGE_DIR)/combined.txt -bench=. -benchmem -timeout 20m'
test-with-flags:
@echo SOURCE: $(SOURCE)
@echo DATABASE: $(DATABASE)
@echo SOURCE: $(SOURCE)
@echo DATABASE_TEST: $(DATABASE_TEST)
@go test $(TEST_FLAGS) .
@go test $(TEST_FLAGS) ./cli/...
@go test $(TEST_FLAGS) ./database
@go test $(TEST_FLAGS) ./testing/...
@echo -n '$(SOURCE)' | tr -s ' ' '\n' | xargs -I{} go test $(TEST_FLAGS) ./source/{}
@go test $(TEST_FLAGS) ./source/testing/...
@go test $(TEST_FLAGS) ./source/stub/...
@echo -n '$(DATABASE)' | tr -s ' ' '\n' | xargs -I{} go test $(TEST_FLAGS) ./database/{}
@go test $(TEST_FLAGS) ./database/testing/...
@go test $(TEST_FLAGS) ./database/stub/...
@go test $(TEST_FLAGS) ./...
kill-orphaned-docker-containers:
@@ -84,7 +86,7 @@ rewrite-import-paths:
docs:
-make kill-docs
nohup godoc -play -http=127.0.0.1:6064 </dev/null >/dev/null 2>&1 & echo $$! > .godoc.pid
cat .godoc.pid
cat .godoc.pid
kill-docs:
@@ -109,10 +111,10 @@ define external_deps
endef
.PHONY: build-cli clean test-short test test-with-flags html-coverage \
.PHONY: build build-docker build-cli clean test-short test test-with-flags html-coverage \
restore-import-paths rewrite-import-paths list-external-deps release \
docs kill-docs open-docs kill-orphaned-docker-containers
SHELL = /bin/bash
SHELL = /bin/sh
RAND = $(shell echo $$RANDOM)

View File

@@ -1,21 +1,21 @@
[![Build Status](https://img.shields.io/travis/com/golang-migrate/migrate/master.svg)](https://travis-ci.com/golang-migrate/migrate)
[![GoDoc](https://godoc.org/github.com/golang-migrate/migrate?status.svg)](https://godoc.org/github.com/golang-migrate/migrate)
[![CircleCI - Build Status](https://img.shields.io/circleci/build/github/golang-migrate/migrate/master)](https://circleci.com/gh/golang-migrate/migrate)
[![GoDoc](https://pkg.go.dev/badge/github.com/golang-migrate/migrate)](https://pkg.go.dev/github.com/golang-migrate/migrate/v4)
[![Coverage Status](https://img.shields.io/coveralls/github/golang-migrate/migrate/master.svg)](https://coveralls.io/github/golang-migrate/migrate?branch=master)
[![packagecloud.io](https://img.shields.io/badge/deb-packagecloud.io-844fec.svg)](https://packagecloud.io/golang-migrate/migrate?filter=debs)
[![Docker Pulls](https://img.shields.io/docker/pulls/migrate/migrate.svg)](https://hub.docker.com/r/migrate/migrate/)
![Supported Go Versions](https://img.shields.io/badge/Go-1.11%2C%201.12-lightgrey.svg)
![Supported Go Versions](https://img.shields.io/badge/Go-1.15%2C%201.16-lightgrey.svg)
[![GitHub Release](https://img.shields.io/github/release/golang-migrate/migrate.svg)](https://github.com/golang-migrate/migrate/releases)
[![Go Report Card](https://goreportcard.com/badge/github.com/golang-migrate/migrate)](https://goreportcard.com/report/github.com/golang-migrate/migrate)
# migrate
__Database migrations written in Go. Use as [CLI](#cli-usage) or import as [library](#use-in-your-go-project).__
* Migrate reads migrations from [sources](#migration-sources)
* Migrate reads migrations from [sources](#migration-sources)
and applies them in correct order to a [database](#databases).
* Drivers are "dumb", migrate glues everything together and makes sure the logic is bulletproof.
* Drivers are "dumb", migrate glues everything together and makes sure the logic is bulletproof.
(Keeps the drivers lightweight, too.)
* Database drivers don't assume things or try to correct user input. When in doubt, fail.
* Database drivers don't assume things or try to correct user input. When in doubt, fail.
Forked from [mattes/migrate](https://github.com/mattes/migrate)
@@ -23,24 +23,28 @@ Forked from [mattes/migrate](https://github.com/mattes/migrate)
Database drivers run migrations. [Add a new database?](database/driver.go)
* [PostgreSQL](database/postgres)
* [Redshift](database/redshift)
* [Ql](database/ql)
* [Cassandra](database/cassandra)
* [SQLite](database/sqlite3) ([todo #165](https://github.com/mattes/migrate/issues/165))
* [MySQL/ MariaDB](database/mysql)
* [Neo4j](database/neo4j) ([todo #167](https://github.com/mattes/migrate/issues/167))
* [MongoDB](database/mongodb)
* [CrateDB](database/crate) ([todo #170](https://github.com/mattes/migrate/issues/170))
* [Shell](database/shell) ([todo #171](https://github.com/mattes/migrate/issues/171))
* [Google Cloud Spanner](database/spanner)
* [CockroachDB](database/cockroachdb)
* [ClickHouse](database/clickhouse)
* [Firebird](database/firebird)
* [PostgreSQL](database/postgres)
* [PGX](database/pgx)
* [Redshift](database/redshift)
* [Ql](database/ql)
* [Cassandra](database/cassandra)
* [SQLite](database/sqlite)
* [SQLite3](database/sqlite3) ([todo #165](https://github.com/mattes/migrate/issues/165))
* [SQLCipher](database/sqlcipher)
* [MySQL/ MariaDB](database/mysql)
* [Neo4j](database/neo4j)
* [MongoDB](database/mongodb)
* [CrateDB](database/crate) ([todo #170](https://github.com/mattes/migrate/issues/170))
* [Shell](database/shell) ([todo #171](https://github.com/mattes/migrate/issues/171))
* [Google Cloud Spanner](database/spanner)
* [CockroachDB](database/cockroachdb)
* [ClickHouse](database/clickhouse)
* [Firebird](database/firebird)
* [MS SQL Server](database/sqlserver)
### Database URLs
Database connection strings are specified via URLs. The URL format is driver dependent but generally has the form: `dbdriver://username:password@host:port/dbname?option1=true&option2=false`
Database connection strings are specified via URLs. The URL format is driver dependent but generally has the form: `dbdriver://username:password@host:port/dbname?param1=true&param2=false`
Any [reserved URL characters](https://en.wikipedia.org/wiki/Percent-encoding#Percent-encoding_reserved_characters) need to be escaped. Note, the `%` character also [needs to be escaped](https://en.wikipedia.org/wiki/Percent-encoding#Percent-encoding_the_percent_character)
@@ -48,6 +52,7 @@ Explicitly, the following characters need to be escaped:
`!`, `#`, `$`, `%`, `&`, `'`, `(`, `)`, `*`, `+`, `,`, `/`, `:`, `;`, `=`, `?`, `@`, `[`, `]`
It's easiest to always run the URL parts of your DB connection URL (e.g. username, password, etc) through an URL encoder. See the example Python snippets below:
```bash
$ python3 -c 'import urllib.parse; print(urllib.parse.quote(input("String to encode: "), ""))'
String to encode: FAKEpassword!#$%&'()*+,/:;=?@[]
@@ -62,44 +67,44 @@ $
Source drivers read migrations from local or remote sources. [Add a new source?](source/driver.go)
* [Filesystem](source/file) - read from fileystem
* [Go-Bindata](source/go_bindata) - read from embedded binary data ([jteeuwen/go-bindata](https://github.com/jteeuwen/go-bindata))
* [Github](source/github) - read from remote Github repositories
* [Gitlab](source/gitlab) - read from remote Gitlab repositories
* [AWS S3](source/aws_s3) - read from Amazon Web Services S3
* [Google Cloud Storage](source/google_cloud_storage) - read from Google Cloud Platform Storage
* [Filesystem](source/file) - read from filesystem
* [Go-Bindata](source/go_bindata) - read from embedded binary data ([jteeuwen/go-bindata](https://github.com/jteeuwen/go-bindata))
* [GitHub](source/github) - read from remote GitHub repositories
* [GitHub Enterprise](source/github_ee) - read from remote GitHub Enterprise repositories
* [Bitbucket](source/bitbucket) - read from remote Bitbucket repositories
* [Gitlab](source/gitlab) - read from remote Gitlab repositories
* [AWS S3](source/aws_s3) - read from Amazon Web Services S3
* [Google Cloud Storage](source/google_cloud_storage) - read from Google Cloud Platform Storage
## CLI usage
* Simple wrapper around this library.
* Handles ctrl+c (SIGINT) gracefully.
* No config search paths, no config files, no magic ENV var injections.
* Simple wrapper around this library.
* Handles ctrl+c (SIGINT) gracefully.
* No config search paths, no config files, no magic ENV var injections.
__[CLI Documentation](cli)__
__[CLI Documentation](cmd/migrate)__
### Basic usage:
### Basic usage
```
```bash
$ migrate -source file://path/to/migrations -database postgres://localhost:5432/database up 2
```
### Docker usage
```
```bash
$ docker run -v {{ migration dir }}:/migrations --network host migrate/migrate
-path=/migrations/ -database postgres://localhost:5432/database up 2
```
## Use in your Go project
* API is stable and frozen for this release (v3 & v4).
* Uses [Go modules](https://golang.org/cmd/go/#hdr-Modules__module_versions__and_more) to manage dependencies.
* To help prevent database corruptions, it supports graceful stops via `GracefulStop chan bool`.
* Bring your own logger.
* Uses `io.Reader` streams internally for low memory overhead.
* Thread-safe and no goroutine leaks.
* API is stable and frozen for this release (v3 & v4).
* Uses [Go modules](https://golang.org/cmd/go/#hdr-Modules__module_versions__and_more) to manage dependencies.
* To help prevent database corruptions, it supports graceful stops via `GracefulStop chan bool`.
* Bring your own logger.
* Uses `io.Reader` streams internally for low memory overhead.
* Thread-safe and no goroutine leaks.
__[Go Documentation](https://godoc.org/github.com/golang-migrate/migrate)__
@@ -139,11 +144,22 @@ func main() {
}
```
## Getting started
Go to [getting started](GETTING_STARTED.md)
## Tutorials
* [CockroachDB](database/cockroachdb/TUTORIAL.md)
* [PostgreSQL](database/postgres/TUTORIAL.md)
(more tutorials to come)
## Migration files
Each migration has an up and down migration. [Why?](FAQ.md#why-two-separate-files-up-and-down-for-a-migration)
```
```bash
1481574547_create_users_table.up.sql
1481574547_create_users_table.down.sql
```
@@ -165,8 +181,6 @@ read the [development guide](CONTRIBUTING.md).
Also have a look at the [FAQ](FAQ.md).
---
Looking for alternatives? [https://awesome-go.com/#database](https://awesome-go.com/#database).

View File

@@ -7,12 +7,14 @@ package database
import (
"fmt"
"io"
nurl "net/url"
"sync"
iurl "github.com/golang-migrate/migrate/v4/internal/url"
)
var (
ErrLocked = fmt.Errorf("can't acquire lock")
ErrLocked = fmt.Errorf("can't acquire lock")
ErrNotLocked = fmt.Errorf("can't unlock, as not currently locked")
)
const NilVersion int = -1
@@ -32,7 +34,7 @@ var drivers = make(map[string]Driver)
// All other functions are tested by tests in database/testing.
// Saves you some time and makes sure all database drivers behave the same way.
// 5. Call Register in init().
// 6. Create a migrate/cli/build_<driver-name>.go file
// 6. Create a internal/cli/build_<driver-name>.go file
// 7. Add driver name in 'DATABASE' variable in Makefile
//
// Guidelines:
@@ -60,7 +62,7 @@ type Driver interface {
// all migrations have been run.
Unlock() error
// Run applies a migration to the database. migration is garantueed to be not nil.
// Run applies a migration to the database. migration is guaranteed to be not nil.
Run(migration io.Reader) error
// SetVersion saves version and dirty state.
@@ -81,21 +83,16 @@ type Driver interface {
// Open returns a new driver instance.
func Open(url string) (Driver, error) {
u, err := nurl.Parse(url)
scheme, err := iurl.SchemeFromURL(url)
if err != nil {
return nil, fmt.Errorf("Unable to parse URL. Did you escape all reserved URL characters? "+
"See: https://github.com/golang-migrate/migrate#database-urls Error: %v", err)
}
if u.Scheme == "" {
return nil, fmt.Errorf("database driver: invalid URL scheme")
return nil, err
}
driversMu.RLock()
d, ok := drivers[u.Scheme]
d, ok := drivers[scheme]
driversMu.RUnlock()
if !ok {
return nil, fmt.Errorf("database driver: unknown driver %v (forgotten import?)", u.Scheme)
return nil, fmt.Errorf("database driver: unknown driver %v (forgotten import?)", scheme)
}
return d.Open(url)

View File

@@ -0,0 +1,16 @@
# sqlite3
`sqlite3://path/to/database?query`
Unlike other migrate database drivers, the sqlite3 driver will automatically wrap each migration in an implicit transaction by default. Migrations must not contain explicit `BEGIN` or `COMMIT` statements. This behavior may change in a future major release. (See below for a workaround.)
Refer to [upstream documentation](https://github.com/mattn/go-sqlite3/blob/master/README.md#connection-string) for a complete list of query parameters supported by the sqlite3 database driver. The auxiliary query parameters listed below may be supplied to tailor migrate behavior. All auxiliary query parameters are optional.
| URL Query | WithInstance Config | Description |
|------------|---------------------|-------------|
| `x-migrations-table` | `MigrationsTable` | Name of the migrations table. Defaults to `schema_migrations`. |
| `x-no-tx-wrap` | `NoTxWrap` | Disable implicit transactions when `true`. Migrations may, and should, contain explicit `BEGIN` and `COMMIT` statements. |
## Notes
* Uses the `github.com/mattn/go-sqlite3` sqlite db driver (cgo)

View File

@@ -3,9 +3,11 @@ package sqlite3
import (
"database/sql"
"fmt"
"go.uber.org/atomic"
"io"
"io/ioutil"
nurl "net/url"
"strconv"
"strings"
"github.com/golang-migrate/migrate/v4"
@@ -28,11 +30,12 @@ var (
type Config struct {
MigrationsTable string
DatabaseName string
NoTxWrap bool
}
type Sqlite struct {
db *sql.DB
isLocked bool
isLocked atomic.Bool
config *Config
}
@@ -100,13 +103,25 @@ func (m *Sqlite) Open(url string) (database.Driver, error) {
return nil, err
}
migrationsTable := purl.Query().Get("x-migrations-table")
qv := purl.Query()
migrationsTable := qv.Get("x-migrations-table")
if len(migrationsTable) == 0 {
migrationsTable = DefaultMigrationsTable
}
noTxWrap := false
if v := qv.Get("x-no-tx-wrap"); v != "" {
noTxWrap, err = strconv.ParseBool(v)
if err != nil {
return nil, fmt.Errorf("x-no-tx-wrap: %s", err)
}
}
mx, err := WithInstance(db, &Config{
DatabaseName: purl.Path,
MigrationsTable: migrationsTable,
NoTxWrap: noTxWrap,
})
if err != nil {
return nil, err
@@ -129,6 +144,7 @@ func (m *Sqlite) Drop() (err error) {
err = multierror.Append(err, errClose)
}
}()
tableNames := make([]string, 0)
for tables.Next() {
var tableName string
@@ -139,6 +155,10 @@ func (m *Sqlite) Drop() (err error) {
tableNames = append(tableNames, tableName)
}
}
if err := tables.Err(); err != nil {
return &database.Error{OrigErr: err, Query: []byte(query)}
}
if len(tableNames) > 0 {
for _, t := range tableNames {
query := "DROP TABLE " + t
@@ -158,18 +178,16 @@ func (m *Sqlite) Drop() (err error) {
}
func (m *Sqlite) Lock() error {
if m.isLocked {
if !m.isLocked.CAS(false, true) {
return database.ErrLocked
}
m.isLocked = true
return nil
}
func (m *Sqlite) Unlock() error {
if !m.isLocked {
return nil
if !m.isLocked.CAS(true, false) {
return database.ErrNotLocked
}
m.isLocked = false
return nil
}
@@ -180,6 +198,9 @@ func (m *Sqlite) Run(migration io.Reader) error {
}
query := string(migr[:])
if m.config.NoTxWrap {
return m.executeQueryNoTx(query)
}
return m.executeQuery(query)
}
@@ -200,6 +221,13 @@ func (m *Sqlite) executeQuery(query string) error {
return nil
}
func (m *Sqlite) executeQueryNoTx(query string) error {
if _, err := m.db.Exec(query); err != nil {
return &database.Error{OrigErr: err, Query: []byte(query)}
}
return nil
}
func (m *Sqlite) SetVersion(version int, dirty bool) error {
tx, err := m.db.Begin()
if err != nil {
@@ -211,9 +239,12 @@ func (m *Sqlite) SetVersion(version int, dirty bool) error {
return &database.Error{OrigErr: err, Query: []byte(query)}
}
if version >= 0 {
query := fmt.Sprintf(`INSERT INTO %s (version, dirty) VALUES (%d, '%t')`, m.config.MigrationsTable, version, dirty)
if _, err := tx.Exec(query); err != nil {
// Also re-write the schema version for nil dirty versions to prevent
// empty schema version for failed down migration on the first migration
// See: https://github.com/golang-migrate/migrate/issues/330
if version >= 0 || (version == database.NilVersion && dirty) {
query := fmt.Sprintf(`INSERT INTO %s (version, dirty) VALUES (?, ?)`, m.config.MigrationsTable)
if _, err := tx.Exec(query, version, dirty); err != nil {
if errRollback := tx.Rollback(); errRollback != nil {
err = multierror.Append(err, errRollback)
}

View File

@@ -2,6 +2,7 @@ package database
import (
"fmt"
"go.uber.org/atomic"
"hash/crc32"
"strings"
)
@@ -17,3 +18,16 @@ func GenerateAdvisoryLockId(databaseName string, additionalNames ...string) (str
sum = sum * uint32(advisoryLockIDSalt)
return fmt.Sprint(sum), nil
}
// CasRestoreOnErr CAS wrapper to automatically restore the lock state on error
func CasRestoreOnErr(lock *atomic.Bool, o, n bool, casErr error, f func() error) error {
if !lock.CAS(o, n) {
return casErr
}
if err := f(); err != nil {
// Automatically unlock/lock on error
lock.Store(o)
return err
}
return nil
}

View File

@@ -0,0 +1,25 @@
package url
import (
"errors"
"strings"
)
var errNoScheme = errors.New("no scheme")
var errEmptyURL = errors.New("URL cannot be empty")
// schemeFromURL returns the scheme from a URL string
func SchemeFromURL(url string) (string, error) {
if url == "" {
return "", errEmptyURL
}
i := strings.Index(url, ":")
// No : or : is the first character.
if i < 1 {
return "", errNoScheme
}
return url[0:i], nil
}

View File

@@ -1,18 +1,20 @@
// Package migrate reads migrations from sources and runs them against databases.
// Sources are defined by the `source.Driver` and databases by the `database.Driver`
// interface. The driver interfaces are kept "dump", all migration logic is kept
// interface. The driver interfaces are kept "dumb", all migration logic is kept
// in this package.
package migrate
import (
"errors"
"fmt"
"github.com/hashicorp/go-multierror"
"os"
"sync"
"time"
"github.com/hashicorp/go-multierror"
"github.com/golang-migrate/migrate/v4/database"
iurl "github.com/golang-migrate/migrate/v4/internal/url"
"github.com/golang-migrate/migrate/v4/source"
)
@@ -85,13 +87,13 @@ type Migrate struct {
func New(sourceURL, databaseURL string) (*Migrate, error) {
m := newCommon()
sourceName, err := sourceSchemeFromURL(sourceURL)
sourceName, err := iurl.SchemeFromURL(sourceURL)
if err != nil {
return nil, err
}
m.sourceName = sourceName
databaseName, err := databaseSchemeFromURL(databaseURL)
databaseName, err := iurl.SchemeFromURL(databaseURL)
if err != nil {
return nil, err
}
@@ -119,7 +121,7 @@ func New(sourceURL, databaseURL string) (*Migrate, error) {
func NewWithDatabaseInstance(sourceURL string, databaseName string, databaseInstance database.Driver) (*Migrate, error) {
m := newCommon()
sourceName, err := schemeFromURL(sourceURL)
sourceName, err := iurl.SchemeFromURL(sourceURL)
if err != nil {
return nil, err
}
@@ -145,7 +147,7 @@ func NewWithDatabaseInstance(sourceURL string, databaseName string, databaseInst
func NewWithSourceInstance(sourceName string, sourceInstance source.Driver, databaseURL string) (*Migrate, error) {
m := newCommon()
databaseName, err := schemeFromURL(databaseURL)
databaseName, err := iurl.SchemeFromURL(databaseURL)
if err != nil {
return nil, err
}
@@ -485,7 +487,7 @@ func (m *Migrate) read(from int, to int, ret chan<- interface{}) {
}
prev, err := m.sourceDrv.Prev(suint(from))
if os.IsNotExist(err) && to == -1 {
if errors.Is(err, os.ErrNotExist) && to == -1 {
// apply nil migration
migr, err := m.newMigration(suint(from), -1)
if err != nil {
@@ -578,7 +580,7 @@ func (m *Migrate) readUp(from int, limit int, ret chan<- interface{}) {
// apply next migration
next, err := m.sourceDrv.Next(suint(from))
if os.IsNotExist(err) {
if errors.Is(err, os.ErrNotExist) {
// no limit, but no migrations applied?
if limit == -1 && count == 0 {
ret <- ErrNoChange
@@ -664,7 +666,7 @@ func (m *Migrate) readDown(from int, limit int, ret chan<- interface{}) {
}
prev, err := m.sourceDrv.Prev(suint(from))
if os.IsNotExist(err) {
if errors.Is(err, os.ErrNotExist) {
// no limit or haven't reached limit, apply "first" migration
if limit == -1 || limit-count > 0 {
firstVersion, err := m.sourceDrv.First()
@@ -783,9 +785,9 @@ func (m *Migrate) versionExists(version uint) (result error) {
}
}()
}
if os.IsExist(err) {
if errors.Is(err, os.ErrExist) {
return nil
} else if !os.IsNotExist(err) {
} else if !errors.Is(err, os.ErrNotExist) {
return err
}
@@ -798,13 +800,15 @@ func (m *Migrate) versionExists(version uint) (result error) {
}
}()
}
if os.IsExist(err) {
if errors.Is(err, os.ErrExist) {
return nil
} else if !os.IsNotExist(err) {
} else if !errors.Is(err, os.ErrNotExist) {
return err
}
return os.ErrNotExist
err = fmt.Errorf("no migration found for version %d: %w", version, err)
m.logErr(err)
return err
}
// stop returns true if no more migrations should be run against the database
@@ -832,7 +836,7 @@ func (m *Migrate) newMigration(version uint, targetVersion int) (*Migration, err
if targetVersion >= int(version) {
r, identifier, err := m.sourceDrv.ReadUp(version)
if os.IsNotExist(err) {
if errors.Is(err, os.ErrNotExist) {
// create "empty" migration
migr, err = NewMigration(nil, "", version, targetVersion)
if err != nil {
@@ -852,7 +856,7 @@ func (m *Migrate) newMigration(version uint, targetVersion int) (*Migration, err
} else {
r, identifier, err := m.sourceDrv.ReadDown(version)
if os.IsNotExist(err) {
if errors.Is(err, os.ErrNotExist) {
// create "empty" migration
migr, err = NewMigration(nil, "", version, targetVersion)
if err != nil {
@@ -950,7 +954,7 @@ func (m *Migrate) unlock() error {
// if a prevErr is not nil.
func (m *Migrate) unlockErr(prevErr error) error {
if err := m.unlock(); err != nil {
return NewMultiError(prevErr, err)
return multierror.Append(prevErr, err)
}
return prevErr
}

View File

@@ -87,7 +87,7 @@ func Open(url string) (Driver, error) {
d, ok := drivers[u.Scheme]
driversMu.RUnlock()
if !ok {
return nil, fmt.Errorf("source driver: unknown driver %v (forgotten import?)", u.Scheme)
return nil, fmt.Errorf("source driver: unknown driver '%s' (forgotten import?)", u.Scheme)
}
return d.Open(url)

View File

@@ -0,0 +1,15 @@
package source
import "os"
// ErrDuplicateMigration is an error type for reporting duplicate migration
// files.
type ErrDuplicateMigration struct {
Migration
os.FileInfo
}
// Error implements error interface.
func (e ErrDuplicateMigration) Error() string {
return "duplicate migration file: " + e.Name()
}

View File

@@ -1,12 +1,8 @@
package file
import (
"fmt"
"io"
"io/ioutil"
nurl "net/url"
"os"
"path"
"path/filepath"
"github.com/golang-migrate/migrate/v4/source"
@@ -16,18 +12,11 @@ func init() {
source.Register("file", &File{})
}
type File struct {
url string
path string
migrations *source.Migrations
}
func (f *File) Open(url string) (source.Driver, error) {
func parseURL(url string) (string, error) {
u, err := nurl.Parse(url)
if err != nil {
return nil, err
return "", err
}
// concat host and path to restore full path
// host might be `.`
p := u.Opaque
@@ -39,7 +28,7 @@ func (f *File) Open(url string) (source.Driver, error) {
// default to current directory if no path
wd, err := os.Getwd()
if err != nil {
return nil, err
return "", err
}
p = wd
@@ -47,81 +36,9 @@ func (f *File) Open(url string) (source.Driver, error) {
// make path absolute if relative
abs, err := filepath.Abs(p)
if err != nil {
return nil, err
return "", err
}
p = abs
}
// scan directory
files, err := ioutil.ReadDir(p)
if err != nil {
return nil, err
}
nf := &File{
url: url,
path: p,
migrations: source.NewMigrations(),
}
for _, fi := range files {
if !fi.IsDir() {
m, err := source.DefaultParse(fi.Name())
if err != nil {
continue // ignore files that we can't parse
}
if !nf.migrations.Append(m) {
return nil, fmt.Errorf("unable to parse file %v", fi.Name())
}
}
}
return nf, nil
}
func (f *File) Close() error {
// nothing do to here
return nil
}
func (f *File) First() (version uint, err error) {
if v, ok := f.migrations.First(); ok {
return v, nil
}
return 0, &os.PathError{Op: "first", Path: f.path, Err: os.ErrNotExist}
}
func (f *File) Prev(version uint) (prevVersion uint, err error) {
if v, ok := f.migrations.Prev(version); ok {
return v, nil
}
return 0, &os.PathError{Op: fmt.Sprintf("prev for version %v", version), Path: f.path, Err: os.ErrNotExist}
}
func (f *File) Next(version uint) (nextVersion uint, err error) {
if v, ok := f.migrations.Next(version); ok {
return v, nil
}
return 0, &os.PathError{Op: fmt.Sprintf("next for version %v", version), Path: f.path, Err: os.ErrNotExist}
}
func (f *File) ReadUp(version uint) (r io.ReadCloser, identifier string, err error) {
if m, ok := f.migrations.Up(version); ok {
r, err := os.Open(path.Join(f.path, m.Raw))
if err != nil {
return nil, "", err
}
return r, m.Identifier, nil
}
return nil, "", &os.PathError{Op: fmt.Sprintf("read version %v", version), Path: f.path, Err: os.ErrNotExist}
}
func (f *File) ReadDown(version uint) (r io.ReadCloser, identifier string, err error) {
if m, ok := f.migrations.Down(version); ok {
r, err := os.Open(path.Join(f.path, m.Raw))
if err != nil {
return nil, "", err
}
return r, m.Identifier, nil
}
return nil, "", &os.PathError{Op: fmt.Sprintf("read version %v", version), Path: f.path, Err: os.ErrNotExist}
return p, nil
}

View File

@@ -0,0 +1,32 @@
// +build !go1.16
package file
import (
"net/http"
"github.com/golang-migrate/migrate/v4/source"
"github.com/golang-migrate/migrate/v4/source/httpfs"
)
type File struct {
httpfs.PartialDriver
url string
path string
}
func (f *File) Open(url string) (source.Driver, error) {
p, err := parseURL(url)
if err != nil {
return nil, err
}
nf := &File{
url: url,
path: p,
}
if err := nf.Init(http.Dir(p), ""); err != nil {
return nil, err
}
return nf, nil
}

View File

@@ -0,0 +1,31 @@
// +build go1.16
package file
import (
"os"
"github.com/golang-migrate/migrate/v4/source"
"github.com/golang-migrate/migrate/v4/source/iofs"
)
type File struct {
iofs.PartialDriver
url string
path string
}
func (f *File) Open(url string) (source.Driver, error) {
p, err := parseURL(url)
if err != nil {
return nil, err
}
nf := &File{
url: url,
path: p,
}
if err := nf.Init(os.DirFS(p), "."); err != nil {
return nil, err
}
return nf, nil
}

View File

@@ -0,0 +1,49 @@
# httpfs
## Usage
This package could be used to create new migration source drivers that uses
`http.FileSystem` to read migration files.
Struct `httpfs.PartialDriver` partly implements the `source.Driver` interface. It has all
the methods except for `Open()`. Embedding this struct and adding `Open()` method
allows users of this package to create new migration sources. Example:
```go
struct mydriver {
httpfs.PartialDriver
}
func (d *mydriver) Open(url string) (source.Driver, error) {
var fs http.FileSystem
var path string
var ds mydriver
// acquire fs and path from url
// set-up ds if necessary
if err := ds.Init(fs, path); err != nil {
return nil, err
}
return &ds, nil
}
```
This package also provides a simple `source.Driver` implementation that works
with `http.FileSystem` provided by the user of this package. It is created with
`httpfs.New()` call.
Example of using `http.Dir()` to read migrations from `sql` directory:
```go
src, err := httpfs.New(http.Dir("sql"))
if err != nil {
// do something
}
m, err := migrate.NewWithSourceInstance("httpfs", src, "database://url")
if err != nil {
// do something
}
err = m.Up()
...
```

View File

@@ -0,0 +1,31 @@
package httpfs
import (
"errors"
"net/http"
"github.com/golang-migrate/migrate/v4/source"
)
// driver is a migration source driver for reading migrations from
// http.FileSystem instances. It implements source.Driver interface and can be
// used as a migration source for the main migrate library.
type driver struct {
PartialDriver
}
// New creates a new migrate source driver from a http.FileSystem instance and a
// relative path to migration files within the virtual FS.
func New(fs http.FileSystem, path string) (source.Driver, error) {
var d driver
if err := d.Init(fs, path); err != nil {
return nil, err
}
return &d, nil
}
// Open completes the implementetion of source.Driver interface. Other methods
// are implemented by the embedded PartialDriver struct.
func (d *driver) Open(url string) (source.Driver, error) {
return nil, errors.New("Open() cannot be called on the httpfs passthrough driver")
}

View File

@@ -0,0 +1,156 @@
package httpfs
import (
"errors"
"io"
"net/http"
"os"
"path"
"strconv"
"github.com/golang-migrate/migrate/v4/source"
)
// PartialDriver is a helper service for creating new source drivers working with
// http.FileSystem instances. It implements all source.Driver interface methods
// except for Open(). New driver could embed this struct and add missing Open()
// method.
//
// To prepare PartialDriver for use Init() function.
type PartialDriver struct {
migrations *source.Migrations
fs http.FileSystem
path string
}
// Init prepares not initialized PartialDriver instance to read migrations from a
// http.FileSystem instance and a relative path.
func (p *PartialDriver) Init(fs http.FileSystem, path string) error {
root, err := fs.Open(path)
if err != nil {
return err
}
files, err := root.Readdir(0)
if err != nil {
_ = root.Close()
return err
}
if err = root.Close(); err != nil {
return err
}
ms := source.NewMigrations()
for _, file := range files {
if file.IsDir() {
continue
}
m, err := source.DefaultParse(file.Name())
if err != nil {
continue // ignore files that we can't parse
}
if !ms.Append(m) {
return source.ErrDuplicateMigration{
Migration: *m,
FileInfo: file,
}
}
}
p.fs = fs
p.path = path
p.migrations = ms
return nil
}
// Close is part of source.Driver interface implementation. This is a no-op.
func (p *PartialDriver) Close() error {
return nil
}
// First is part of source.Driver interface implementation.
func (p *PartialDriver) First() (version uint, err error) {
if version, ok := p.migrations.First(); ok {
return version, nil
}
return 0, &os.PathError{
Op: "first",
Path: p.path,
Err: os.ErrNotExist,
}
}
// Prev is part of source.Driver interface implementation.
func (p *PartialDriver) Prev(version uint) (prevVersion uint, err error) {
if version, ok := p.migrations.Prev(version); ok {
return version, nil
}
return 0, &os.PathError{
Op: "prev for version " + strconv.FormatUint(uint64(version), 10),
Path: p.path,
Err: os.ErrNotExist,
}
}
// Next is part of source.Driver interface implementation.
func (p *PartialDriver) Next(version uint) (nextVersion uint, err error) {
if version, ok := p.migrations.Next(version); ok {
return version, nil
}
return 0, &os.PathError{
Op: "next for version " + strconv.FormatUint(uint64(version), 10),
Path: p.path,
Err: os.ErrNotExist,
}
}
// ReadUp is part of source.Driver interface implementation.
func (p *PartialDriver) ReadUp(version uint) (r io.ReadCloser, identifier string, err error) {
if m, ok := p.migrations.Up(version); ok {
body, err := p.open(path.Join(p.path, m.Raw))
if err != nil {
return nil, "", err
}
return body, m.Identifier, nil
}
return nil, "", &os.PathError{
Op: "read up for version " + strconv.FormatUint(uint64(version), 10),
Path: p.path,
Err: os.ErrNotExist,
}
}
// ReadDown is part of source.Driver interface implementation.
func (p *PartialDriver) ReadDown(version uint) (r io.ReadCloser, identifier string, err error) {
if m, ok := p.migrations.Down(version); ok {
body, err := p.open(path.Join(p.path, m.Raw))
if err != nil {
return nil, "", err
}
return body, m.Identifier, nil
}
return nil, "", &os.PathError{
Op: "read down for version " + strconv.FormatUint(uint64(version), 10),
Path: p.path,
Err: os.ErrNotExist,
}
}
func (p *PartialDriver) open(path string) (http.File, error) {
f, err := p.fs.Open(path)
if err == nil {
return f, nil
}
// Some non-standard file systems may return errors that don't include the path, that
// makes debugging harder.
if !errors.As(err, new(*os.PathError)) {
err = &os.PathError{
Op: "open",
Path: path,
Err: err,
}
}
return nil, err
}

View File

@@ -0,0 +1,3 @@
# iofs
https://pkg.go.dev/github.com/golang-migrate/migrate/v4/source/iofs

View File

@@ -0,0 +1,10 @@
/*
Package iofs provides the Go 1.16+ io/fs#FS driver.
It can accept various file systems (like embed.FS, archive/zip#Reader) implementing io/fs#FS.
This driver cannot be used with Go versions 1.15 and below.
Also, Opening with a URL scheme is not supported.
*/
package iofs

View File

@@ -0,0 +1,175 @@
// +build go1.16
package iofs
import (
"errors"
"fmt"
"io"
"io/fs"
"path"
"strconv"
"github.com/golang-migrate/migrate/v4/source"
)
type driver struct {
PartialDriver
}
// New returns a new Driver from io/fs#FS and a relative path.
func New(fsys fs.FS, path string) (source.Driver, error) {
var i driver
if err := i.Init(fsys, path); err != nil {
return nil, fmt.Errorf("failed to init driver with path %s: %w", path, err)
}
return &i, nil
}
// Open is part of source.Driver interface implementation.
// Open cannot be called on the iofs passthrough driver.
func (d *driver) Open(url string) (source.Driver, error) {
return nil, errors.New("Open() cannot be called on the iofs passthrough driver")
}
// PartialDriver is a helper service for creating new source drivers working with
// io/fs.FS instances. It implements all source.Driver interface methods
// except for Open(). New driver could embed this struct and add missing Open()
// method.
//
// To prepare PartialDriver for use Init() function.
type PartialDriver struct {
migrations *source.Migrations
fsys fs.FS
path string
}
// Init prepares not initialized IoFS instance to read migrations from a
// io/fs#FS instance and a relative path.
func (d *PartialDriver) Init(fsys fs.FS, path string) error {
entries, err := fs.ReadDir(fsys, path)
if err != nil {
return err
}
ms := source.NewMigrations()
for _, e := range entries {
if e.IsDir() {
continue
}
m, err := source.DefaultParse(e.Name())
if err != nil {
continue
}
file, err := e.Info()
if err != nil {
return err
}
if !ms.Append(m) {
return source.ErrDuplicateMigration{
Migration: *m,
FileInfo: file,
}
}
}
d.fsys = fsys
d.path = path
d.migrations = ms
return nil
}
// Close is part of source.Driver interface implementation.
// Closes the file system if possible.
func (d *PartialDriver) Close() error {
c, ok := d.fsys.(io.Closer)
if !ok {
return nil
}
return c.Close()
}
// First is part of source.Driver interface implementation.
func (d *PartialDriver) First() (version uint, err error) {
if version, ok := d.migrations.First(); ok {
return version, nil
}
return 0, &fs.PathError{
Op: "first",
Path: d.path,
Err: fs.ErrNotExist,
}
}
// Prev is part of source.Driver interface implementation.
func (d *PartialDriver) Prev(version uint) (prevVersion uint, err error) {
if version, ok := d.migrations.Prev(version); ok {
return version, nil
}
return 0, &fs.PathError{
Op: "prev for version " + strconv.FormatUint(uint64(version), 10),
Path: d.path,
Err: fs.ErrNotExist,
}
}
// Next is part of source.Driver interface implementation.
func (d *PartialDriver) Next(version uint) (nextVersion uint, err error) {
if version, ok := d.migrations.Next(version); ok {
return version, nil
}
return 0, &fs.PathError{
Op: "next for version " + strconv.FormatUint(uint64(version), 10),
Path: d.path,
Err: fs.ErrNotExist,
}
}
// ReadUp is part of source.Driver interface implementation.
func (d *PartialDriver) ReadUp(version uint) (r io.ReadCloser, identifier string, err error) {
if m, ok := d.migrations.Up(version); ok {
body, err := d.open(path.Join(d.path, m.Raw))
if err != nil {
return nil, "", err
}
return body, m.Identifier, nil
}
return nil, "", &fs.PathError{
Op: "read up for version " + strconv.FormatUint(uint64(version), 10),
Path: d.path,
Err: fs.ErrNotExist,
}
}
// ReadDown is part of source.Driver interface implementation.
func (d *PartialDriver) ReadDown(version uint) (r io.ReadCloser, identifier string, err error) {
if m, ok := d.migrations.Down(version); ok {
body, err := d.open(path.Join(d.path, m.Raw))
if err != nil {
return nil, "", err
}
return body, m.Identifier, nil
}
return nil, "", &fs.PathError{
Op: "read down for version " + strconv.FormatUint(uint64(version), 10),
Path: d.path,
Err: fs.ErrNotExist,
}
}
func (d *PartialDriver) open(path string) (fs.File, error) {
f, err := d.fsys.Open(path)
if err == nil {
return f, nil
}
// Some non-standard file systems may return errors that don't include the path, that
// makes debugging harder.
if !errors.As(err, new(*fs.PathError)) {
err = &fs.PathError{
Op: "open",
Path: path,
Err: err,
}
}
return nil, err
}

View File

@@ -0,0 +1 @@
1 down

View File

@@ -0,0 +1 @@
1 up

View File

@@ -0,0 +1 @@
3 up

View File

@@ -0,0 +1 @@
4 down

View File

@@ -0,0 +1 @@
4 up

View File

@@ -0,0 +1 @@
5 down

View File

@@ -0,0 +1 @@
7 down

View File

@@ -0,0 +1 @@
7 up

View File

@@ -1,18 +1,22 @@
package migrate
import (
"errors"
"fmt"
nurl "net/url"
"strings"
)
// MultiError holds multiple errors.
//
// Deprecated: Use github.com/hashicorp/go-multierror instead
type MultiError struct {
Errs []error
}
// NewMultiError returns an error type holding multiple errors.
//
// Deprecated: Use github.com/hashicorp/go-multierror instead
//
func NewMultiError(errs ...error) MultiError {
compactErrs := make([]error, 0)
for _, e := range errs {
@@ -44,42 +48,6 @@ func suint(n int) uint {
return uint(n)
}
var errNoScheme = errors.New("no scheme")
var errEmptyURL = errors.New("URL cannot be empty")
func sourceSchemeFromURL(url string) (string, error) {
u, err := schemeFromURL(url)
if err != nil {
return "", fmt.Errorf("source: %v", err)
}
return u, nil
}
func databaseSchemeFromURL(url string) (string, error) {
u, err := schemeFromURL(url)
if err != nil {
return "", fmt.Errorf("database: %v", err)
}
return u, nil
}
// schemeFromURL returns the scheme from a URL string
func schemeFromURL(url string) (string, error) {
if url == "" {
return "", errEmptyURL
}
u, err := nurl.Parse(url)
if err != nil {
return "", err
}
if len(u.Scheme) == 0 {
return "", errNoScheme
}
return u.Scheme, nil
}
// FilterCustomQuery filters all query values starting with `x-`
func FilterCustomQuery(u *nurl.URL) *nurl.URL {
ux := *u