Improve migrations to support migrating milestones/labels/issues/comments/pullrequests (#6290)

* add migrations

* fix package dependency

* fix lints

* implements migrations except pull requests

* add releases

* migrating releases

* fix bug

* fix lint

* fix migrate releases

* fix tests

* add rollback

* pull request migtations

* fix import

* fix go module vendor

* add tests for upload to gitea

* more migrate options

* fix swagger-check

* fix misspell

* add options on migration UI

* fix log error

* improve UI options on migrating

* add support for username password when migrating from github

* fix tests

* remove comments and fix migrate limitation

* improve error handles

* migrate API will also support migrate milestones/labels/issues/pulls/releases

* fix tests and remove unused codes

* add DownloaderFactory and docs about how to create a new Downloader

* fix misspell

* fix migration docs

* Add hints about migrate options on migration page

* fix tests
pull/6872/head
Lunny Xiao 2019-05-07 09:12:51 +08:00 committed by GitHub
parent 1c7c739eb9
commit 08069dc465
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
128 changed files with 33540 additions and 75 deletions

2
.gitignore vendored
View File

@ -66,7 +66,7 @@ coverage.all
/integrations/mssql.ini
/node_modules
/modules/indexer/issues/indexers
routers/repo/authorized_keys
# Snapcraft
snap/.snapcraft/

View File

@ -0,0 +1,72 @@
---
date: "2019-04-15T17:29:00+08:00"
title: "Advanced: Migrations Interfaces"
slug: "migrations-interfaces"
weight: 30
toc: true
draft: false
menu:
sidebar:
parent: "advanced"
name: "Migrations Interfaces"
weight: 55
identifier: "migrations-interfaces"
---
# Migration Features
The new migration features were introduced in Gitea 1.9.0. It defines two interfaces to support migrating
repositories data from other git host platforms to gitea or, in the future migrating gitea data to other
git host platforms. Currently, only the migrations from github via APIv3 to Gitea is implemented.
First of all, Gitea defines some standard objects in packages `modules/migrations/base`. They are
`Repository`, `Milestone`, `Release`, `Label`, `Issue`, `Comment`, `PullRequest`.
## Downloader Interfaces
To migrate from a new git host platform, there are two steps to be updated.
- You should implement a `Downloader` which will get all kinds of repository informations.
- You should implement a `DownloaderFactory` which is used to detect if the URL matches and
create a Downloader.
- You'll need to register the `DownloaderFactory` via `RegisterDownloaderFactory` on init.
```Go
type Downloader interface {
GetRepoInfo() (*Repository, error)
GetMilestones() ([]*Milestone, error)
GetReleases() ([]*Release, error)
GetLabels() ([]*Label, error)
GetIssues(start, limit int) ([]*Issue, error)
GetComments(issueNumber int64) ([]*Comment, error)
GetPullRequests(start, limit int) ([]*PullRequest, error)
}
```
```Go
type DownloaderFactory interface {
Match(opts MigrateOptions) (bool, error)
New(opts MigrateOptions) (Downloader, error)
}
```
## Uploader Interface
Currently, only a `GiteaLocalUploader` is implemented, so we only save downloaded
data via this `Uploader` on the local Gitea instance. Other uploaders are not supported
and will be implemented in future.
```Go
// Uploader uploads all the informations
type Uploader interface {
CreateRepo(repo *Repository, includeWiki bool) error
CreateMilestone(milestone *Milestone) error
CreateRelease(release *Release) error
CreateLabel(label *Label) error
CreateIssue(issue *Issue) error
CreateComment(issueNumber int64, comment *Comment) error
CreatePullRequest(pr *PullRequest) error
Rollback() error
}
```

3
go.mod
View File

@ -62,6 +62,7 @@ require (
github.com/gogits/chardet v0.0.0-20150115103509-2404f7772561
github.com/gogits/cron v0.0.0-20160810035002-7f3990acf183
github.com/gogo/protobuf v1.2.1 // indirect
github.com/google/go-github/v24 v24.0.1
github.com/gorilla/context v1.1.1
github.com/issue9/assert v1.3.2 // indirect
github.com/issue9/identicon v0.0.0-20160320065130-d36b54562f4c
@ -115,7 +116,7 @@ require (
go.etcd.io/bbolt v1.3.2 // indirect
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793
golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519
golang.org/x/oauth2 v0.0.0-20181101160152-c453e0c75759 // indirect
golang.org/x/oauth2 v0.0.0-20181101160152-c453e0c75759
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223
golang.org/x/text v0.3.0
gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect

11
go.sum
View File

@ -142,6 +142,12 @@ github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pO
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY=
github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
github.com/google/go-github/v24 v24.0.1 h1:KCt1LjMJEey1qvPXxa9SjaWxwTsCWSq6p2Ju57UR4Q4=
github.com/google/go-github/v24 v24.0.1/go.mod h1:CRqaW1Uns1TCkP0wqTpxYyRxRjxwvKU/XSS44u6X74M=
github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gorilla/context v1.1.1 h1:AWwleXJkX/nhcU9bZSnZoi3h/qGYqQAGhq6zZe/aQW8=
@ -309,17 +315,21 @@ github.com/ziutek/mymysql v1.5.4 h1:GB0qdRGsTwQSBVYuVShFBKaXSnSnYYC2d9knnE1LHFs=
github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0=
go.etcd.io/bbolt v1.3.2 h1:Z/90sZLPOeCy2PwprqkFa25PdkusRzaj9P8zm/KNyvk=
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
golang.org/x/crypto v0.0.0-20180820150726-614d502a4dac/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793 h1:u+LnwYTOOW7Ukr/fppxEb1Nwz0AtPflrblfvUudpo+I=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519 h1:x6rhz8Y9CjbgQkccRGmELH6K+LJj7tOoh3XWeC1yaQM=
golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/oauth2 v0.0.0-20180620175406-ef147856a6dd/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20181101160152-c453e0c75759 h1:TMrx+Qdx7uJAeUbv15N72h5Hmyb5+VDjEiMufAEAM04=
golang.org/x/oauth2 v0.0.0-20181101160152-c453e0c75759/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180824143301-4910a1d54f87/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180903190138-2b024373dcd9/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223 h1:DH4skfRX4EBpamg7iV4ZlCpblAHI6s6TDM39bFZumv8=
@ -327,6 +337,7 @@ golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5h
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.2.0 h1:S0iUepdCWODXRvtE+gcRDd15L+k+k1AiHlMiMjefH24=
google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc h1:2gGKlE2+asNV9m7xrywl36YYNnBG5ZQ0r/BOOxqPpmk=

145
models/migrate.go Normal file
View File

@ -0,0 +1,145 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package models
import "github.com/go-xorm/xorm"
// InsertIssue insert one issue to database
func InsertIssue(issue *Issue, labelIDs []int64) error {
sess := x.NewSession()
if err := sess.Begin(); err != nil {
return err
}
if err := insertIssue(sess, issue, labelIDs); err != nil {
return err
}
return sess.Commit()
}
func insertIssue(sess *xorm.Session, issue *Issue, labelIDs []int64) error {
if issue.MilestoneID > 0 {
sess.Incr("num_issues")
if issue.IsClosed {
sess.Incr("num_closed_issues")
}
if _, err := sess.ID(issue.MilestoneID).NoAutoTime().Update(new(Milestone)); err != nil {
return err
}
}
if _, err := sess.NoAutoTime().Insert(issue); err != nil {
return err
}
var issueLabels = make([]IssueLabel, 0, len(labelIDs))
for _, labelID := range labelIDs {
issueLabels = append(issueLabels, IssueLabel{
IssueID: issue.ID,
LabelID: labelID,
})
}
if _, err := sess.Insert(issueLabels); err != nil {
return err
}
if !issue.IsPull {
sess.ID(issue.RepoID).Incr("num_issues")
if issue.IsClosed {
sess.Incr("num_closed_issues")
}
} else {
sess.ID(issue.RepoID).Incr("num_pulls")
if issue.IsClosed {
sess.Incr("num_closed_pulls")
}
}
if _, err := sess.NoAutoTime().Update(issue.Repo); err != nil {
return err
}
sess.Incr("num_issues")
if issue.IsClosed {
sess.Incr("num_closed_issues")
}
if _, err := sess.In("id", labelIDs).Update(new(Label)); err != nil {
return err
}
if issue.MilestoneID > 0 {
if _, err := sess.ID(issue.MilestoneID).SetExpr("completeness", "num_closed_issues * 100 / num_issues").Update(new(Milestone)); err != nil {
return err
}
}
return nil
}
// InsertComment inserted a comment
func InsertComment(comment *Comment) error {
sess := x.NewSession()
defer sess.Close()
if err := sess.Begin(); err != nil {
return err
}
if _, err := sess.NoAutoTime().Insert(comment); err != nil {
return err
}
if _, err := sess.ID(comment.IssueID).Incr("num_comments").Update(new(Issue)); err != nil {
return err
}
return sess.Commit()
}
// InsertPullRequest inserted a pull request
func InsertPullRequest(pr *PullRequest, labelIDs []int64) error {
sess := x.NewSession()
defer sess.Close()
if err := sess.Begin(); err != nil {
return err
}
if err := insertIssue(sess, pr.Issue, labelIDs); err != nil {
return err
}
pr.IssueID = pr.Issue.ID
if _, err := sess.NoAutoTime().Insert(pr); err != nil {
return err
}
return sess.Commit()
}
// MigrateRelease migrates release
func MigrateRelease(rel *Release) error {
sess := x.NewSession()
if err := sess.Begin(); err != nil {
return err
}
var oriRel = Release{
RepoID: rel.RepoID,
TagName: rel.TagName,
}
exist, err := sess.Get(&oriRel)
if err != nil {
return err
}
if !exist {
if _, err := sess.NoAutoTime().Insert(rel); err != nil {
return err
}
} else {
rel.ID = oriRel.ID
if _, err := sess.ID(rel.ID).Cols("target, title, note, is_tag, num_commits").Update(rel); err != nil {
return err
}
}
for i := 0; i < len(rel.Attachments); i++ {
rel.Attachments[i].ReleaseID = rel.ID
}
if _, err := sess.NoAutoTime().Insert(rel.Attachments); err != nil {
return err
}
return sess.Commit()
}

View File

@ -107,6 +107,7 @@ func TestRelease_MirrorDelete(t *testing.T) {
IsPrivate: false,
IsMirror: true,
RemoteAddr: repoPath,
Wiki: true,
}
mirror, err := MigrateRepository(user, user, migrationOptions)
assert.NoError(t, err)

View File

@ -896,6 +896,7 @@ type MigrateRepoOptions struct {
IsPrivate bool
IsMirror bool
RemoteAddr string
Wiki bool // include wiki repository
}
/*
@ -917,7 +918,7 @@ func wikiRemoteURL(remote string) string {
return ""
}
// MigrateRepository migrates a existing repository from other project hosting.
// MigrateRepository migrates an existing repository from other project hosting.
func MigrateRepository(doer, u *User, opts MigrateRepoOptions) (*Repository, error) {
repo, err := CreateRepository(doer, u, CreateRepoOptions{
Name: opts.Name,
@ -930,7 +931,6 @@ func MigrateRepository(doer, u *User, opts MigrateRepoOptions) (*Repository, err
}
repoPath := RepoPath(u.Name, opts.Name)
wikiPath := WikiPath(u.Name, opts.Name)
if u.IsOrganization() {
t, err := u.GetOwnerTeam()
@ -956,22 +956,25 @@ func MigrateRepository(doer, u *User, opts MigrateRepoOptions) (*Repository, err
return repo, fmt.Errorf("Clone: %v", err)
}
wikiRemotePath := wikiRemoteURL(opts.RemoteAddr)
if len(wikiRemotePath) > 0 {
if err := os.RemoveAll(wikiPath); err != nil {
return repo, fmt.Errorf("Failed to remove %s: %v", wikiPath, err)
}
if err = git.Clone(wikiRemotePath, wikiPath, git.CloneRepoOptions{
Mirror: true,
Quiet: true,
Timeout: migrateTimeout,
Branch: "master",
}); err != nil {
log.Warn("Clone wiki: %v", err)
if opts.Wiki {
wikiPath := WikiPath(u.Name, opts.Name)
wikiRemotePath := wikiRemoteURL(opts.RemoteAddr)
if len(wikiRemotePath) > 0 {
if err := os.RemoveAll(wikiPath); err != nil {
return repo, fmt.Errorf("Failed to remove %s: %v", wikiPath, err)
}
if err = git.Clone(wikiRemotePath, wikiPath, git.CloneRepoOptions{
Mirror: true,
Quiet: true,
Timeout: migrateTimeout,
Branch: "master",
}); err != nil {
log.Warn("Clone wiki: %v", err)
if err := os.RemoveAll(wikiPath); err != nil {
return repo, fmt.Errorf("Failed to remove %s: %v", wikiPath, err)
}
}
}
}

View File

@ -51,10 +51,16 @@ type MigrateRepoForm struct {
// required: true
UID int64 `json:"uid" binding:"Required"`
// required: true
RepoName string `json:"repo_name" binding:"Required;AlphaDashDot;MaxSize(100)"`
Mirror bool `json:"mirror"`
Private bool `json:"private"`
Description string `json:"description" binding:"MaxSize(255)"`
RepoName string `json:"repo_name" binding:"Required;AlphaDashDot;MaxSize(100)"`
Mirror bool `json:"mirror"`
Private bool `json:"private"`
Description string `json:"description" binding:"MaxSize(255)"`
Wiki bool `json:"wiki"`
Milestones bool `json:"milestones"`
Labels bool `json:"labels"`
Issues bool `json:"issues"`
PullRequests bool `json:"pull_requests"`
Releases bool `json:"releases"`
}
// Validate validates the fields

View File

@ -0,0 +1,17 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// Copyright 2018 Jonas Franz. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package base
import "time"
// Comment is a standard comment information
type Comment struct {
PosterName string
PosterEmail string
Created time.Time
Content string
Reactions *Reactions
}

View File

@ -0,0 +1,23 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// Copyright 2018 Jonas Franz. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package base
// Downloader downloads the site repo informations
type Downloader interface {
GetRepoInfo() (*Repository, error)
GetMilestones() ([]*Milestone, error)
GetReleases() ([]*Release, error)
GetLabels() ([]*Label, error)
GetIssues(start, limit int) ([]*Issue, error)
GetComments(issueNumber int64) ([]*Comment, error)
GetPullRequests(start, limit int) ([]*PullRequest, error)
}
// DownloaderFactory defines an interface to match a downloader implementation and create a downloader
type DownloaderFactory interface {
Match(opts MigrateOptions) (bool, error)
New(opts MigrateOptions) (Downloader, error)
}

View File

@ -0,0 +1,24 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// Copyright 2018 Jonas Franz. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package base
import "time"
// Issue is a standard issue information
type Issue struct {
Number int64
PosterName string
PosterEmail string
Title string
Content string
Milestone string
State string // closed, open
IsLocked bool
Created time.Time
Closed *time.Time
Labels []*Label
Reactions *Reactions
}

View File

@ -0,0 +1,13 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// Copyright 2018 Jonas Franz. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package base
// Label defines a standard label informations
type Label struct {
Name string
Color string
Description string
}

View File

@ -0,0 +1,19 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// Copyright 2018 Jonas Franz. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package base
import "time"
// Milestone defines a standard milestone
type Milestone struct {
Title string
Description string
Deadline *time.Time
Created time.Time
Updated *time.Time
Closed *time.Time
State string
}

View File

@ -0,0 +1,26 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// Copyright 2018 Jonas Franz. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package base
// MigrateOptions defines the way a repository gets migrated
type MigrateOptions struct {
RemoteURL string
AuthUsername string
AuthPassword string
Name string
Description string
Wiki bool
Issues bool
Milestones bool
Labels bool
Releases bool
Comments bool
PullRequests bool
Private bool
Mirror bool
IgnoreIssueAuthor bool // if true will not add original author information before issues or comments content.
}

View File

@ -0,0 +1,53 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// Copyright 2018 Jonas Franz. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package base
import (
"fmt"
"time"
)
// PullRequest defines a standard pull request information
type PullRequest struct {
Number int64
Title string
PosterName string
PosterEmail string
Content string
Milestone string
State string
Created time.Time
Closed *time.Time
Labels []*Label
PatchURL string
Merged bool
MergedTime *time.Time
MergeCommitSHA string
Head PullRequestBranch
Base PullRequestBranch
Assignee string
Assignees []string
IsLocked bool
}
// IsForkPullRequest returns true if the pull request from a forked repository but not the same repository
func (p *PullRequest) IsForkPullRequest() bool {
return p.Head.RepoPath() != p.Base.RepoPath()
}
// PullRequestBranch represents a pull request branch
type PullRequestBranch struct {
CloneURL string
Ref string
SHA string
RepoName string
OwnerName string
}
// RepoPath returns pull request repo path
func (p PullRequestBranch) RepoPath() string {
return fmt.Sprintf("%s/%s", p.OwnerName, p.RepoName)
}

View File

@ -0,0 +1,17 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// Copyright 2018 Jonas Franz. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package base
// Reactions represents a summary of reactions.
type Reactions struct {
TotalCount int
PlusOne int
MinusOne int
Laugh int
Confused int
Heart int
Hooray int
}

View File

@ -0,0 +1,31 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package base
import "time"
// ReleaseAsset represents a release asset
type ReleaseAsset struct {
URL string
Name string
ContentType *string
Size *int
DownloadCount *int
Created time.Time
Updated time.Time
}
// Release represents a release
type Release struct {
TagName string
TargetCommitish string
Name string
Body string
Draft bool
Prerelease bool
Assets []ReleaseAsset
Created time.Time
Published time.Time
}

View File

@ -0,0 +1,18 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// Copyright 2018 Jonas Franz. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package base
// Repository defines a standard repository information
type Repository struct {
Name string
Owner string
IsPrivate bool
IsMirror bool
Description string
AuthUsername string
AuthPassword string
CloneURL string
}

View File

@ -0,0 +1,18 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// Copyright 2018 Jonas Franz. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package base
// Uploader uploads all the informations
type Uploader interface {
CreateRepo(repo *Repository, includeWiki bool) error
CreateMilestone(milestone *Milestone) error
CreateRelease(release *Release) error
CreateLabel(label *Label) error
CreateIssue(issue *Issue) error
CreateComment(issueNumber int64, comment *Comment) error
CreatePullRequest(pr *PullRequest) error
Rollback() error
}

View File

@ -0,0 +1,29 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// Copyright 2018 Jonas Franz. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package migrations
import (
"errors"
"github.com/google/go-github/v24/github"
)
var (
// ErrNotSupported returns the error not supported
ErrNotSupported = errors.New("not supported")
)
// IsRateLimitError returns true if the err is github.RateLimitError
func IsRateLimitError(err error) bool {
_, ok := err.(*github.RateLimitError)
return ok
}
// IsTwoFactorAuthError returns true if the err is github.TwoFactorAuthError
func IsTwoFactorAuthError(err error) bool {
_, ok := err.(*github.TwoFactorAuthError)
return ok
}

69
modules/migrations/git.go Normal file
View File

@ -0,0 +1,69 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package migrations
import (
"code.gitea.io/gitea/modules/migrations/base"
)
var (
_ base.Downloader = &PlainGitDownloader{}
)
// PlainGitDownloader implements a Downloader interface to clone git from a http/https URL
type PlainGitDownloader struct {
ownerName string
repoName string
remoteURL string
}
// NewPlainGitDownloader creates a git Downloader
func NewPlainGitDownloader(ownerName, repoName, remoteURL string) *PlainGitDownloader {
return &PlainGitDownloader{
ownerName: ownerName,
repoName: repoName,
remoteURL: remoteURL,
}
}
// GetRepoInfo returns a repository information
func (g *PlainGitDownloader) GetRepoInfo() (*base.Repository, error) {
// convert github repo to stand Repo
return &base.Repository{
Owner: g.ownerName,
Name: g.repoName,
CloneURL: g.remoteURL,
}, nil
}
// GetMilestones returns milestones
func (g *PlainGitDownloader) GetMilestones() ([]*base.Milestone, error) {
return nil, ErrNotSupported
}
// GetLabels returns labels
func (g *PlainGitDownloader) GetLabels() ([]*base.Label, error) {
return nil, ErrNotSupported
}
// GetReleases returns releases
func (g *PlainGitDownloader) GetReleases() ([]*base.Release, error) {
return nil, ErrNotSupported
}
// GetIssues returns issues according start and limit
func (g *PlainGitDownloader) GetIssues(start, limit int) ([]*base.Issue, error) {
return nil, ErrNotSupported
}
// GetComments returns comments according issueNumber
func (g *PlainGitDownloader) GetComments(issueNumber int64) ([]*base.Comment, error) {
return nil, ErrNotSupported
}
// GetPullRequests returns pull requests according start and limit
func (g *PlainGitDownloader) GetPullRequests(start, limit int) ([]*base.PullRequest, error) {
return nil, ErrNotSupported
}

403
modules/migrations/gitea.go Normal file
View File

@ -0,0 +1,403 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// Copyright 2018 Jonas Franz. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package migrations
import (
"fmt"
"io"
"net/http"
"os"
"path"
"path/filepath"
"strings"
"sync"
"time"
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/migrations/base"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/util"
gouuid "github.com/satori/go.uuid"
)
var (
_ base.Uploader = &GiteaLocalUploader{}
)
// GiteaLocalUploader implements an Uploader to gitea sites
type GiteaLocalUploader struct {
doer *models.User
repoOwner string
repoName string
repo *models.Repository
labels sync.Map
milestones sync.Map
issues sync.Map
gitRepo *git.Repository
prHeadCache map[string]struct{}
}
// NewGiteaLocalUploader creates an gitea Uploader via gitea API v1
func NewGiteaLocalUploader(doer *models.User, repoOwner, repoName string) *GiteaLocalUploader {
return &GiteaLocalUploader{
doer: doer,
repoOwner: repoOwner,
repoName: repoName,
prHeadCache: make(map[string]struct{}),
}
}
// CreateRepo creates a repository
func (g *GiteaLocalUploader) CreateRepo(repo *base.Repository, includeWiki bool) error {
owner, err := models.GetUserByName(g.repoOwner)
if err != nil {
return err
}
r, err := models.MigrateRepository(g.doer, owner, models.MigrateRepoOptions{
Name: g.repoName,
Description: repo.Description,
IsMirror: repo.IsMirror,
RemoteAddr: repo.CloneURL,
IsPrivate: repo.IsPrivate,
Wiki: includeWiki,
})
if err != nil {
return err
}
g.repo = r
g.gitRepo, err = git.OpenRepository(r.RepoPath())
return err
}
// CreateMilestone creates milestone
func (g *GiteaLocalUploader) CreateMilestone(milestone *base.Milestone) error {
var deadline util.TimeStamp
if milestone.Deadline != nil {
deadline = util.TimeStamp(milestone.Deadline.Unix())
}
if deadline == 0 {
deadline = util.TimeStamp(time.Date(9999, 1, 1, 0, 0, 0, 0, setting.UILocation).Unix())
}
var ms = models.Milestone{
RepoID: g.repo.ID,
Name: milestone.Title,
Content: milestone.Description,
IsClosed: milestone.State == "close",
DeadlineUnix: deadline,
}
if ms.IsClosed && milestone.Closed != nil {
ms.ClosedDateUnix = util.TimeStamp(milestone.Closed.Unix())
}
err := models.NewMilestone(&ms)
if err != nil {
return err
}
g.milestones.Store(ms.Name, ms.ID)
return nil
}
// CreateLabel creates label
func (g *GiteaLocalUploader) CreateLabel(label *base.Label) error {
var lb = models.Label{
RepoID: g.repo.ID,
Name: label.Name,
Description: label.Description,
Color: fmt.Sprintf("#%s", label.Color),
}
err := models.NewLabel(&lb)
if err != nil {
return err
}
g.labels.Store(lb.Name, lb.ID)
return nil
}
// CreateRelease creates release
func (g *GiteaLocalUploader) CreateRelease(release *base.Release) error {
var rel = models.Release{
RepoID: g.repo.ID,
PublisherID: g.doer.ID,
TagName: release.TagName,
LowerTagName: strings.ToLower(release.TagName),
Target: release.TargetCommitish,
Title: release.Name,
Sha1: release.TargetCommitish,
Note: release.Body,
IsDraft: release.Draft,
IsPrerelease: release.Prerelease,
IsTag: false,
CreatedUnix: util.TimeStamp(release.Created.Unix()),
}
// calc NumCommits
commit, err := g.gitRepo.GetCommit(rel.TagName)
if err != nil {
return fmt.Errorf("GetCommit: %v", err)
}
rel.NumCommits, err = commit.CommitsCount()
if err != nil {
return fmt.Errorf("CommitsCount: %v", err)
}
for _, asset := range release.Assets {
var attach = models.Attachment{
UUID: gouuid.NewV4().String(),
Name: asset.Name,
DownloadCount: int64(*asset.DownloadCount),
Size: int64(*asset.Size),
CreatedUnix: util.TimeStamp(asset.Created.Unix()),
}
// download attachment
resp, err := http.Get(asset.URL)
if err != nil {
return err
}
defer resp.Body.Close()
localPath := attach.LocalPath()
if err = os.MkdirAll(path.Dir(localPath), os.ModePerm); err != nil {
return fmt.Errorf("MkdirAll: %v", err)
}
fw, err := os.Create(localPath)
if err != nil {
return fmt.Errorf("Create: %v", err)
}
defer fw.Close()
if _, err := io.Copy(fw, resp.Body); err != nil {
return err
}
rel.Attachments = append(rel.Attachments, &attach)
}
return models.MigrateRelease(&rel)
}
// CreateIssue creates issue
func (g *GiteaLocalUploader) CreateIssue(issue *base.Issue) error {
var labelIDs []int64
for _, label := range issue.Labels {
id, ok := g.labels.Load(label.Name)
if !ok {
return fmt.Errorf("Label %s missing when create issue", label.Name)
}
labelIDs = append(labelIDs, id.(int64))
}
var milestoneID int64
if issue.Milestone != "" {
milestone, ok := g.milestones.Load(issue.Milestone)
if !ok {
return fmt.Errorf("Milestone %s missing when create issue", issue.Milestone)
}
milestoneID = milestone.(int64)
}
var is = models.Issue{
RepoID: g.repo.ID,
Repo: g.repo,
Index: issue.Number,
PosterID: g.doer.ID,
Title: issue.Title,
Content: issue.Content,
IsClosed: issue.State == "closed",
IsLocked: issue.IsLocked,
MilestoneID: milestoneID,
CreatedUnix: util.TimeStamp(issue.Created.Unix()),
}
if issue.Closed != nil {
is.ClosedUnix = util.TimeStamp(issue.Closed.Unix())
}
err := models.InsertIssue(&is, labelIDs)
if err != nil {
return err
}
g.issues.Store(issue.Number, is.ID)
// TODO: add reactions
return err
}
// CreateComment creates comment
func (g *GiteaLocalUploader) CreateComment(issueNumber int64, comment *base.Comment) error {
var issueID int64
if issueIDStr, ok := g.issues.Load(issueNumber); !ok {
issue, err := models.GetIssueByIndex(g.repo.ID, issueNumber)
if err != nil {
return err
}
issueID = issue.ID
g.issues.Store(issueNumber, issueID)
} else {
issueID = issueIDStr.(int64)
}
var cm = models.Comment{
IssueID: issueID,
Type: models.CommentTypeComment,
PosterID: g.doer.ID,
Content: comment.Content,
CreatedUnix: util.TimeStamp(comment.Created.Unix()),
}
err := models.InsertComment(&cm)
// TODO: Reactions
return err
}
// CreatePullRequest creates pull request
func (g *GiteaLocalUploader) CreatePullRequest(pr *base.PullRequest) error {
var labelIDs []int64
for _, label := range pr.Labels {
id, ok := g.labels.Load(label.Name)
if !ok {
return fmt.Errorf("Label %s missing when create issue", label.Name)
}
labelIDs = append(labelIDs, id.(int64))
}
var milestoneID int64
if pr.Milestone != "" {
milestone, ok := g.milestones.Load(pr.Milestone)
if !ok {
return fmt.Errorf("Milestone %s missing when create issue", pr.Milestone)
}
milestoneID = milestone.(int64)
}
// download patch file
resp, err := http.Get(pr.PatchURL)
if err != nil {
return err
}
defer resp.Body.Close()
pullDir := filepath.Join(g.repo.RepoPath(), "pulls")
if err = os.MkdirAll(pullDir, os.ModePerm); err != nil {
return err
}
f, err := os.Create(filepath.Join(pullDir, fmt.Sprintf("%d.patch", pr.Number)))
if err != nil {
return err
}
defer f.Close()
_, err = io.Copy(f, resp.Body)
if err != nil {
return err
}
// set head information
pullHead := filepath.Join(g.repo.RepoPath(), "refs", "pull", fmt.Sprintf("%d", pr.Number))
if err := os.MkdirAll(pullHead, os.ModePerm); err != nil {
return err
}
p, err := os.Create(filepath.Join(pullHead, "head"))
if err != nil {
return err
}
defer p.Close()
_, err = p.WriteString(pr.Head.SHA)
if err != nil {
return err
}
var head = "unknown repository"
if pr.IsForkPullRequest() {
if pr.Head.OwnerName != "" {
remote := pr.Head.OwnerName
_, ok := g.prHeadCache[remote]
if !ok {
// git remote add
err := g.gitRepo.AddRemote(remote, pr.Head.CloneURL, true)
if err != nil {
log.Error("AddRemote failed: %s", err)
} else {
g.prHeadCache[remote] = struct{}{}
ok = true
}
}
if ok {
_, err = git.NewCommand("fetch", remote, pr.Head.Ref).RunInDir(g.repo.RepoPath())
if err != nil {
log.Error("Fetch branch from %s failed: %v", pr.Head.CloneURL, err)
} else {
headBranch := filepath.Join(g.repo.RepoPath(), "refs", "heads", pr.Head.OwnerName, pr.Head.Ref)
if err := os.MkdirAll(filepath.Dir(headBranch), os.ModePerm); err != nil {
return err
}
b, err := os.Create(headBranch)
if err != nil {
return err
}
defer b.Close()
_, err = b.WriteString(pr.Head.SHA)
if err != nil {
return err
}
head = pr.Head.OwnerName + "/" + pr.Head.Ref
}
}
}
} else {
head = pr.Head.Ref
}
var pullRequest = models.PullRequest{
HeadRepoID: g.repo.ID,
HeadBranch: head,
HeadUserName: g.repoOwner,
BaseRepoID: g.repo.ID,
BaseBranch: pr.Base.Ref,
MergeBase: pr.Base.SHA,
Index: pr.Number,
HasMerged: pr.Merged,
Issue: &models.Issue{
RepoID: g.repo.ID,
Repo: g.repo,
Title: pr.Title,
Index: pr.Number,
PosterID: g.doer.ID,
Content: pr.Content,
MilestoneID: milestoneID,
IsPull: true,
IsClosed: pr.State == "closed",
IsLocked: pr.IsLocked,
CreatedUnix: util.TimeStamp(pr.Created.Unix()),
},
}
if pullRequest.Issue.IsClosed && pr.Closed != nil {
pullRequest.Issue.ClosedUnix = util.TimeStamp(pr.Closed.Unix())
}
if pullRequest.HasMerged && pr.MergedTime != nil {
pullRequest.MergedUnix = util.TimeStamp(pr.MergedTime.Unix())
pullRequest.MergedCommitID = pr.MergeCommitSHA
pullRequest.MergerID = g.doer.ID
}
// TODO: reactions
// TODO: assignees
return models.InsertPullRequest(&pullRequest, labelIDs)
}
// Rollback when migrating failed, this will rollback all the changes.
func (g *GiteaLocalUploader) Rollback() error {
if g.repo != nil && g.repo.ID > 0 {
if err := models.DeleteRepository(g.doer, g.repo.OwnerID, g.repo.ID); err != nil {
return err
}
}
return nil
}

View File

@ -0,0 +1,95 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// Copyright 2018 Jonas Franz. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package migrations
import (
"testing"
"time"
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/util"
"github.com/stretchr/testify/assert"
)
func TestGiteaUploadRepo(t *testing.T) {
// FIXME: Since no accesskey or user/password will trigger rate limit of github, just skip
t.Skip()
models.PrepareTestEnv(t)
user := models.AssertExistsAndLoadBean(t, &models.User{ID: 1}).(*models.User)
var (
downloader = NewGithubDownloaderV3("", "", "go-xorm", "builder")
repoName = "builder-" + time.Now().Format("2006-01-02-15-04-05")
uploader = NewGiteaLocalUploader(user, user.Name, repoName)
)
err := migrateRepository(downloader, uploader, MigrateOptions{
RemoteURL: "https://github.com/go-xorm/builder",
Name: repoName,
AuthUsername: "",
Wiki: true,
Issues: true,
Milestones: true,
Labels: true,
Releases: true,
Comments: true,
PullRequests: true,
Private: true,
Mirror: false,
IgnoreIssueAuthor: false,
})
assert.NoError(t, err)
repo := models.AssertExistsAndLoadBean(t, &models.Repository{OwnerID: user.ID, Name: repoName}).(*models.Repository)
assert.True(t, repo.HasWiki())
milestones, err := models.GetMilestones(repo.ID, 0, false, "")
assert.NoError(t, err)
assert.EqualValues(t, 1, len(milestones))
milestones, err = models.GetMilestones(repo.ID, 0, true, "")
assert.NoError(t, err)
assert.EqualValues(t, 0, len(milestones))
labels, err := models.GetLabelsByRepoID(repo.ID, "")
assert.NoError(t, err)
assert.EqualValues(t, 11, len(labels))
releases, err := models.GetReleasesByRepoID(repo.ID, models.FindReleasesOptions{
IncludeTags: true,
}, 0, 10)
assert.NoError(t, err)
assert.EqualValues(t, 8, len(releases))
releases, err = models.GetReleasesByRepoID(repo.ID, models.FindReleasesOptions{
IncludeTags: false,
}, 0, 10)
assert.NoError(t, err)
assert.EqualValues(t, 1, len(releases))
issues, err := models.Issues(&models.IssuesOptions{
RepoIDs: []int64{repo.ID},
IsPull: util.OptionalBoolFalse,
SortType: "oldest",
})
assert.NoError(t, err)
assert.EqualValues(t, 14, len(issues))
assert.NoError(t, issues[0].LoadDiscussComments())
assert.EqualValues(t, 0, len(issues[0].Comments))
pulls, _, err := models.PullRequests(repo.ID, &models.PullRequestsOptions{
SortType: "oldest",
})
assert.NoError(t, err)
assert.EqualValues(t, 34, len(pulls))
assert.NoError(t, pulls[0].LoadIssue())
assert.NoError(t, pulls[0].Issue.LoadDiscussComments())
assert.EqualValues(t, 2, len(pulls[0].Issue.Comments))
}

View File

@ -0,0 +1,475 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// Copyright 2018 Jonas Franz. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package migrations
import (
"context"
"fmt"
"net/http"
"net/url"
"strings"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/migrations/base"
"github.com/google/go-github/v24/github"
"golang.org/x/oauth2"
)
var (
_ base.Downloader = &GithubDownloaderV3{}
_ base.DownloaderFactory = &GithubDownloaderV3Factory{}
)
func init() {
RegisterDownloaderFactory(&GithubDownloaderV3Factory{})
}
// GithubDownloaderV3Factory defines a github downloader v3 factory
type GithubDownloaderV3Factory struct {
}
// Match returns ture if the migration remote URL matched this downloader factory
func (f *GithubDownloaderV3Factory) Match(opts base.MigrateOptions) (bool, error) {
u, err := url.Parse(opts.RemoteURL)
if err != nil {
return false, err
}
return u.Host == "github.com" && opts.AuthUsername != "", nil
}
// New returns a Downloader related to this factory according MigrateOptions
func (f *GithubDownloaderV3Factory) New(opts base.MigrateOptions) (base.Downloader, error) {
u, err := url.Parse(opts.RemoteURL)
if err != nil {
return nil, err
}
fields := strings.Split(u.Path, "/")
oldOwner := fields[1]
oldName := strings.TrimSuffix(fields[2], ".git")
log.Trace("Create github downloader: %s/%s", oldOwner, oldName)
return NewGithubDownloaderV3(opts.AuthUsername, opts.AuthPassword, oldOwner, oldName), nil
}
// GithubDownloaderV3 implements a Downloader interface to get repository informations
// from github via APIv3
type GithubDownloaderV3 struct {
ctx context.Context
client *github.Client
repoOwner string
repoName string
userName string
password string
}
// NewGithubDownloaderV3 creates a github Downloader via github v3 API
func NewGithubDownloaderV3(userName, password, repoOwner, repoName string) *GithubDownloaderV3 {
var downloader = GithubDownloaderV3{
userName: userName,
password: password,
ctx: context.Background(),
repoOwner: repoOwner,
repoName: repoName,
}
var client *http.Client
if userName != "" {
if password == "" {
ts := oauth2.StaticTokenSource(
&oauth2.Token{AccessToken: userName},
)
client = oauth2.NewClient(downloader.ctx, ts)
} else {
client = &http.Client{
Transport: &http.Transport{
Proxy: func(req *http.Request) (*url.URL, error) {
req.SetBasicAuth(userName, password)
return nil, nil
},
},
}
}
}
downloader.client = github.NewClient(client)
return &downloader
}
// GetRepoInfo returns a repository information
func (g *GithubDownloaderV3) GetRepoInfo() (*base.Repository, error) {
gr, _, err := g.client.Repositories.Get(g.ctx, g.repoOwner, g.repoName)
if err != nil {
return nil, err
}
// convert github repo to stand Repo
return &base.Repository{
Owner: g.repoOwner,
Name: gr.GetName(),
IsPrivate: *gr.Private,
Description: gr.GetDescription(),
CloneURL: gr.GetCloneURL(),
}, nil
}
// GetMilestones returns milestones
func (g *GithubDownloaderV3) GetMilestones() ([]*base.Milestone, error) {
var perPage = 100
var milestones = make([]*base.Milestone, 0, perPage)
for i := 1; ; i++ {
ms, _, err := g.client.Issues.ListMilestones(g.ctx, g.repoOwner, g.repoName,
&github.MilestoneListOptions{
State: "all",
ListOptions: github.ListOptions{
Page: i,
PerPage: perPage,
}})
if err != nil {
return nil, err
}
for _, m := range ms {
var desc string
if m.Description != nil {
desc = *m.Description
}
var state = "open"
if m.State != nil {
state = *m.State
}
milestones = append(milestones, &base.Milestone{
Title: *m.Title,
Description: desc,
Deadline: m.DueOn,
State: state,
Created: *m.CreatedAt,
Updated: m.UpdatedAt,
Closed: m.ClosedAt,
})
}
if len(ms) < perPage {
break
}
}
return milestones, nil
}
func convertGithubLabel(label *github.Label) *base.Label {
var desc string
if label.Description != nil {
desc = *label.Description
}
return &base.Label{
Name: *label.Name,
Color: *label.Color,
Description: desc,
}
}
// GetLabels returns labels
func (g *GithubDownloaderV3) GetLabels() ([]*base.Label, error) {
var perPage = 100
var labels = make([]*base.Label, 0, perPage)