diff --git a/.drone.yml b/.drone.yml index 68d7198365..560df26606 100644 --- a/.drone.yml +++ b/.drone.yml @@ -127,6 +127,7 @@ pipeline: - curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash - apt-get install -y git-lfs - (sleep 1200 && (echo 'kill -ABRT $(pidof gitea) $(pidof integrations.sqlite.test)' | sh)) & + - make test-sqlite-migration - make test-sqlite when: event: [ push, tag, pull_request ] @@ -141,6 +142,7 @@ pipeline: commands: - curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash - apt-get install -y git-lfs + - make test-mysql-migration - make integration-test-coverage when: event: [ push, pull_request ] @@ -157,6 +159,7 @@ pipeline: - curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash - apt-get install -y git-lfs - (sleep 1200 && (echo 'kill -ABRT $(pidof gitea) $(pidof integrations.test)' | sh)) & + - make test-mysql-migration - make test-mysql when: event: [ tag ] @@ -172,6 +175,7 @@ pipeline: - curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash - apt-get install -y git-lfs - (sleep 1200 && (echo 'kill -ABRT $(pidof gitea) $(pidof integrations.test)' | sh)) & + - make test-pgsql-migration - make test-pgsql when: event: [ push, tag, pull_request ] @@ -186,6 +190,7 @@ pipeline: commands: - curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash - apt-get install -y git-lfs + - make test-mssql-migration - make test-mssql when: event: [ push, tag, pull_request ] diff --git a/Makefile b/Makefile index b27937b0c7..5201e99b04 100644 --- a/Makefile +++ b/Makefile @@ -35,7 +35,7 @@ endif LDFLAGS := -X "main.Version=$(GITEA_VERSION)" -X "main.Tags=$(TAGS)" -PACKAGES ?= $(filter-out code.gitea.io/gitea/integrations,$(shell $(GO) list ./... | grep -v /vendor/)) +PACKAGES ?= $(filter-out code.gitea.io/gitea/integrations/migration-test,$(filter-out code.gitea.io/gitea/integrations,$(shell $(GO) list ./... | grep -v /vendor/))) SOURCES ?= $(shell find . -name "*.go" -type f) TAGS ?= @@ -197,6 +197,10 @@ test-vendor: vendor test-sqlite: integrations.sqlite.test GITEA_ROOT=${CURDIR} GITEA_CONF=integrations/sqlite.ini ./integrations.sqlite.test +.PHONY: test-sqlite-migration +test-sqlite-migration: migrations.sqlite.test + GITEA_ROOT=${CURDIR} GITEA_CONF=integrations/sqlite.ini ./migrations.sqlite.test + generate-ini: sed -e 's|{{TEST_MYSQL_HOST}}|${TEST_MYSQL_HOST}|g' \ -e 's|{{TEST_MYSQL_DBNAME}}|${TEST_MYSQL_DBNAME}|g' \ @@ -218,14 +222,28 @@ generate-ini: test-mysql: integrations.test generate-ini GITEA_ROOT=${CURDIR} GITEA_CONF=integrations/mysql.ini ./integrations.test +.PHONY: test-mysql-migration +test-mysql-migration: migrations.test generate-ini + GITEA_ROOT=${CURDIR} GITEA_CONF=integrations/mysql.ini ./migrations.test + .PHONY: test-pgsql test-pgsql: integrations.test generate-ini GITEA_ROOT=${CURDIR} GITEA_CONF=integrations/pgsql.ini ./integrations.test +.PHONY: test-pgsql-migration +test-pgsql-migration: migrations.test generate-ini + GITEA_ROOT=${CURDIR} GITEA_CONF=integrations/pgsql.ini ./migrations.test + + .PHONY: test-mssql test-mssql: integrations.test generate-ini GITEA_ROOT=${CURDIR} GITEA_CONF=integrations/mssql.ini ./integrations.test +.PHONY: test-mssql-migration +test-mssql-migration: migrations.test generate-ini + GITEA_ROOT=${CURDIR} GITEA_CONF=integrations/mssql.ini ./migrations.test + + .PHONY: bench-sqlite bench-sqlite: integrations.sqlite.test GITEA_ROOT=${CURDIR} GITEA_CONF=integrations/sqlite.ini ./integrations.sqlite.test -test.cpuprofile=cpu.out -test.run DontRunTests -test.bench . @@ -252,6 +270,14 @@ integrations.sqlite.test: $(SOURCES) integrations.cover.test: $(SOURCES) $(GO) test -c code.gitea.io/gitea/integrations -coverpkg $(shell echo $(PACKAGES) | tr ' ' ',') -o integrations.cover.test +.PHONY: migrations.test +migrations.test: $(SOURCES) + $(GO) test -c code.gitea.io/gitea/integrations/migration-test -o migrations.test + +.PHONY: migrations.sqlite.test +migrations.sqlite.test: $(SOURCES) + $(GO) test -c code.gitea.io/gitea/integrations/migration-test -o migrations.sqlite.test -tags 'sqlite sqlite_unlock_notify' + .PHONY: check check: test diff --git a/integrations/migration-test/gitea-v1.5.3.mysql.sql.gz b/integrations/migration-test/gitea-v1.5.3.mysql.sql.gz new file mode 100644 index 0000000000..643b6bab9c Binary files /dev/null and b/integrations/migration-test/gitea-v1.5.3.mysql.sql.gz differ diff --git a/integrations/migration-test/gitea-v1.5.3.postgres.sql.gz b/integrations/migration-test/gitea-v1.5.3.postgres.sql.gz new file mode 100644 index 0000000000..2fcad82111 Binary files /dev/null and b/integrations/migration-test/gitea-v1.5.3.postgres.sql.gz differ diff --git a/integrations/migration-test/gitea-v1.5.3.sqlite3.sql.gz b/integrations/migration-test/gitea-v1.5.3.sqlite3.sql.gz new file mode 100644 index 0000000000..f13bc68a47 Binary files /dev/null and b/integrations/migration-test/gitea-v1.5.3.sqlite3.sql.gz differ diff --git a/integrations/migration-test/gitea-v1.6.4.mysql.sql.gz b/integrations/migration-test/gitea-v1.6.4.mysql.sql.gz new file mode 100644 index 0000000000..30cca8b382 Binary files /dev/null and b/integrations/migration-test/gitea-v1.6.4.mysql.sql.gz differ diff --git a/integrations/migration-test/gitea-v1.6.4.postgres.sql.gz b/integrations/migration-test/gitea-v1.6.4.postgres.sql.gz new file mode 100644 index 0000000000..8db9022d1c Binary files /dev/null and b/integrations/migration-test/gitea-v1.6.4.postgres.sql.gz differ diff --git a/integrations/migration-test/gitea-v1.6.4.sqlite3.sql.gz b/integrations/migration-test/gitea-v1.6.4.sqlite3.sql.gz new file mode 100644 index 0000000000..a777c53025 Binary files /dev/null and b/integrations/migration-test/gitea-v1.6.4.sqlite3.sql.gz differ diff --git a/integrations/migration-test/gitea-v1.7.0.mysql.sql.gz b/integrations/migration-test/gitea-v1.7.0.mysql.sql.gz new file mode 100644 index 0000000000..d0ab10891c Binary files /dev/null and b/integrations/migration-test/gitea-v1.7.0.mysql.sql.gz differ diff --git a/integrations/migration-test/gitea-v1.7.0.postgres.sql.gz b/integrations/migration-test/gitea-v1.7.0.postgres.sql.gz new file mode 100644 index 0000000000..ed66d41b89 Binary files /dev/null and b/integrations/migration-test/gitea-v1.7.0.postgres.sql.gz differ diff --git a/integrations/migration-test/gitea-v1.7.0.sqlite3.sql.gz b/integrations/migration-test/gitea-v1.7.0.sqlite3.sql.gz new file mode 100644 index 0000000000..3155249b07 Binary files /dev/null and b/integrations/migration-test/gitea-v1.7.0.sqlite3.sql.gz differ diff --git a/integrations/migration-test/migration_test.go b/integrations/migration-test/migration_test.go new file mode 100644 index 0000000000..6fd7af832e --- /dev/null +++ b/integrations/migration-test/migration_test.go @@ -0,0 +1,245 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package migrations + +import ( + "compress/gzip" + "database/sql" + "fmt" + "io/ioutil" + "log" + "os" + "path" + "regexp" + "sort" + "testing" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/models/migrations" + "code.gitea.io/gitea/modules/setting" + + "github.com/go-xorm/xorm" + "github.com/stretchr/testify/assert" +) + +var currentEngine *xorm.Engine + +func initMigrationTest() { + giteaRoot := os.Getenv("GITEA_ROOT") + if giteaRoot == "" { + fmt.Println("Environment variable $GITEA_ROOT not set") + os.Exit(1) + } + setting.AppPath = path.Join(giteaRoot, "gitea") + if _, err := os.Stat(setting.AppPath); err != nil { + fmt.Printf("Could not find gitea binary at %s\n", setting.AppPath) + os.Exit(1) + } + + giteaConf := os.Getenv("GITEA_CONF") + if giteaConf == "" { + fmt.Println("Environment variable $GITEA_CONF not set") + os.Exit(1) + } else if !path.IsAbs(giteaConf) { + setting.CustomConf = path.Join(giteaRoot, giteaConf) + } else { + setting.CustomConf = giteaConf + } + + setting.NewContext() + setting.CheckLFSVersion() + models.LoadConfigs() +} + +func getDialect() string { + dialect := "sqlite" + switch { + case setting.UseSQLite3: + dialect = "sqlite" + case setting.UseMySQL: + dialect = "mysql" + case setting.UsePostgreSQL: + dialect = "pgsql" + case setting.UseMSSQL: + dialect = "mssql" + } + return dialect +} + +func availableVersions() ([]string, error) { + migrationsDir, err := os.Open("integrations/migration-test") + if err != nil { + return nil, err + } + defer migrationsDir.Close() + versionRE, err := regexp.Compile("gitea-v(?P.+)\\." + regexp.QuoteMeta(models.DbCfg.Type) + "\\.sql.gz") + if err != nil { + return nil, err + } + + filenames, err := migrationsDir.Readdirnames(-1) + if err != nil { + return nil, err + } + versions := []string{} + for _, filename := range filenames { + if versionRE.MatchString(filename) { + substrings := versionRE.FindStringSubmatch(filename) + versions = append(versions, substrings[1]) + } + } + sort.Strings(versions) + return versions, nil +} + +func readSQLFromFile(version string) (string, error) { + filename := fmt.Sprintf("integrations/migration-test/gitea-v%s.%s.sql.gz", version, models.DbCfg.Type) + + if _, err := os.Stat(filename); os.IsNotExist(err) { + return "", nil + } + + file, err := os.Open(filename) + if err != nil { + return "", err + } + defer file.Close() + + gr, err := gzip.NewReader(file) + if err != nil { + return "", err + } + defer gr.Close() + + bytes, err := ioutil.ReadAll(gr) + if err != nil { + return "", err + } + + return string(bytes), nil +} + +func restoreOldDB(t *testing.T, version string) bool { + data, err := readSQLFromFile(version) + assert.NoError(t, err) + if len(data) == 0 { + log.Printf("No db found to restore for %s version: %s\n", models.DbCfg.Type, version) + return false + } + + switch { + case setting.UseSQLite3: + os.Remove(models.DbCfg.Path) + err := os.MkdirAll(path.Dir(models.DbCfg.Path), os.ModePerm) + assert.NoError(t, err) + + db, err := sql.Open("sqlite3", fmt.Sprintf("file:%s?cache=shared&mode=rwc&_busy_timeout=%d", models.DbCfg.Path, models.DbCfg.Timeout)) + assert.NoError(t, err) + defer db.Close() + + _, err = db.Exec(data) + assert.NoError(t, err) + db.Close() + + case setting.UseMySQL: + db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@tcp(%s)/", + models.DbCfg.User, models.DbCfg.Passwd, models.DbCfg.Host)) + assert.NoError(t, err) + defer db.Close() + + _, err = db.Exec(fmt.Sprintf("DROP DATABASE IF EXISTS %s", models.DbCfg.Name)) + assert.NoError(t, err) + + _, err = db.Exec(fmt.Sprintf("CREATE DATABASE IF NOT EXISTS %s", models.DbCfg.Name)) + assert.NoError(t, err) + + db, err = sql.Open("mysql", fmt.Sprintf("%s:%s@tcp(%s)/%s?multiStatements=true", + models.DbCfg.User, models.DbCfg.Passwd, models.DbCfg.Host, models.DbCfg.Name)) + assert.NoError(t, err) + defer db.Close() + + _, err = db.Exec(data) + assert.NoError(t, err) + db.Close() + + case setting.UsePostgreSQL: + db, err := sql.Open("postgres", fmt.Sprintf("postgres://%s:%s@%s/?sslmode=%s", + models.DbCfg.User, models.DbCfg.Passwd, models.DbCfg.Host, models.DbCfg.SSLMode)) + assert.NoError(t, err) + defer db.Close() + + _, err = db.Exec(fmt.Sprintf("DROP DATABASE IF EXISTS %s", models.DbCfg.Name)) + assert.NoError(t, err) + + _, err = db.Exec(fmt.Sprintf("CREATE DATABASE %s", models.DbCfg.Name)) + assert.NoError(t, err) + db.Close() + + db, err = sql.Open("postgres", fmt.Sprintf("postgres://%s:%s@%s/%s?sslmode=%s", + models.DbCfg.User, models.DbCfg.Passwd, models.DbCfg.Host, models.DbCfg.Name, models.DbCfg.SSLMode)) + assert.NoError(t, err) + defer db.Close() + + _, err = db.Exec(data) + assert.NoError(t, err) + db.Close() + + case setting.UseMSSQL: + host, port := models.ParseMSSQLHostPort(models.DbCfg.Host) + db, err := sql.Open("mssql", fmt.Sprintf("server=%s; port=%s; database=%s; user id=%s; password=%s;", + host, port, "master", models.DbCfg.User, models.DbCfg.Passwd)) + assert.NoError(t, err) + defer db.Close() + + _, err = db.Exec("DROP DATABASE IF EXISTS gitea") + assert.NoError(t, err) + + _, err = db.Exec("CREATE DATABASE gitea") + assert.NoError(t, err) + + _, err = db.Exec(data) + assert.NoError(t, err) + db.Close() + } + return true +} + +func wrappedMigrate(x *xorm.Engine) error { + currentEngine = x + return migrations.Migrate(x) +} + +func doMigrationTest(t *testing.T, version string) { + log.Printf("Performing migration test for %s version: %s", models.DbCfg.Type, version) + if !restoreOldDB(t, version) { + return + } + + setting.NewXORMLogService(false) + err := models.SetEngine() + assert.NoError(t, err) + + err = models.NewEngine(wrappedMigrate) + assert.NoError(t, err) + currentEngine.Close() +} + +func TestMigrations(t *testing.T) { + initMigrationTest() + + dialect := models.DbCfg.Type + versions, err := availableVersions() + assert.NoError(t, err) + + if len(versions) == 0 { + log.Printf("No old database versions available to migration test for %s\n", dialect) + return + } + + log.Printf("Preparing to test %d migrations for %s\n", len(versions), dialect) + for _, version := range versions { + doMigrationTest(t, version) + } +}