mirror of https://github.com/go-gitea/gitea.git
Improve migrations to support migrating milestones/labels/issues/comments/pullrequests (#6290)
* add migrations * fix package dependency * fix lints * implements migrations except pull requests * add releases * migrating releases * fix bug * fix lint * fix migrate releases * fix tests * add rollback * pull request migtations * fix import * fix go module vendor * add tests for upload to gitea * more migrate options * fix swagger-check * fix misspell * add options on migration UI * fix log error * improve UI options on migrating * add support for username password when migrating from github * fix tests * remove comments and fix migrate limitation * improve error handles * migrate API will also support migrate milestones/labels/issues/pulls/releases * fix tests and remove unused codes * add DownloaderFactory and docs about how to create a new Downloader * fix misspell * fix migration docs * Add hints about migrate options on migration page * fix testspull/6872/head
Lunny Xiao
GitHub
parent
1c7c739eb9
commit
08069dc465
|
|
@ -66,7 +66,7 @@ coverage.all
|
|||
/integrations/mssql.ini
|
||||
/node_modules
|
||||
/modules/indexer/issues/indexers
|
||||
|
||||
routers/repo/authorized_keys
|
||||
|
||||
# Snapcraft
|
||||
snap/.snapcraft/
|
||||
|
|
|
|||
|
|
@ -0,0 +1,72 @@
|
|||
---
|
||||
date: "2019-04-15T17:29:00+08:00"
|
||||
title: "Advanced: Migrations Interfaces"
|
||||
slug: "migrations-interfaces"
|
||||
weight: 30
|
||||
toc: true
|
||||
draft: false
|
||||
menu:
|
||||
sidebar:
|
||||
parent: "advanced"
|
||||
name: "Migrations Interfaces"
|
||||
weight: 55
|
||||
identifier: "migrations-interfaces"
|
||||
---
|
||||
|
||||
# Migration Features
|
||||
|
||||
The new migration features were introduced in Gitea 1.9.0. It defines two interfaces to support migrating
|
||||
repositories data from other git host platforms to gitea or, in the future migrating gitea data to other
|
||||
git host platforms. Currently, only the migrations from github via APIv3 to Gitea is implemented.
|
||||
|
||||
First of all, Gitea defines some standard objects in packages `modules/migrations/base`. They are
|
||||
`Repository`, `Milestone`, `Release`, `Label`, `Issue`, `Comment`, `PullRequest`.
|
||||
|
||||
## Downloader Interfaces
|
||||
|
||||
To migrate from a new git host platform, there are two steps to be updated.
|
||||
|
||||
- You should implement a `Downloader` which will get all kinds of repository informations.
|
||||
- You should implement a `DownloaderFactory` which is used to detect if the URL matches and
|
||||
create a Downloader.
|
||||
- You'll need to register the `DownloaderFactory` via `RegisterDownloaderFactory` on init.
|
||||
|
||||
```Go
|
||||
type Downloader interface {
|
||||
GetRepoInfo() (*Repository, error)
|
||||
GetMilestones() ([]*Milestone, error)
|
||||
GetReleases() ([]*Release, error)
|
||||
GetLabels() ([]*Label, error)
|
||||
GetIssues(start, limit int) ([]*Issue, error)
|
||||
GetComments(issueNumber int64) ([]*Comment, error)
|
||||
GetPullRequests(start, limit int) ([]*PullRequest, error)
|
||||
}
|
||||
```
|
||||
|
||||
```Go
|
||||
type DownloaderFactory interface {
|
||||
Match(opts MigrateOptions) (bool, error)
|
||||
New(opts MigrateOptions) (Downloader, error)
|
||||
}
|
||||
```
|
||||
|
||||
## Uploader Interface
|
||||
|
||||
Currently, only a `GiteaLocalUploader` is implemented, so we only save downloaded
|
||||
data via this `Uploader` on the local Gitea instance. Other uploaders are not supported
|
||||
and will be implemented in future.
|
||||
|
||||
```Go
|
||||
// Uploader uploads all the informations
|
||||
type Uploader interface {
|
||||
CreateRepo(repo *Repository, includeWiki bool) error
|
||||
CreateMilestone(milestone *Milestone) error
|
||||
CreateRelease(release *Release) error
|
||||
CreateLabel(label *Label) error
|
||||
CreateIssue(issue *Issue) error
|
||||
CreateComment(issueNumber int64, comment *Comment) error
|
||||
CreatePullRequest(pr *PullRequest) error
|
||||
Rollback() error
|
||||
}
|
||||
|
||||
```
|
||||
3
go.mod
3
go.mod
|
|
@ -62,6 +62,7 @@ require (
|
|||
github.com/gogits/chardet v0.0.0-20150115103509-2404f7772561
|
||||
github.com/gogits/cron v0.0.0-20160810035002-7f3990acf183
|
||||
github.com/gogo/protobuf v1.2.1 // indirect
|
||||
github.com/google/go-github/v24 v24.0.1
|
||||
github.com/gorilla/context v1.1.1
|
||||
github.com/issue9/assert v1.3.2 // indirect
|
||||
github.com/issue9/identicon v0.0.0-20160320065130-d36b54562f4c
|
||||
|
|
@ -115,7 +116,7 @@ require (
|
|||
go.etcd.io/bbolt v1.3.2 // indirect
|
||||
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793
|
||||
golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519
|
||||
golang.org/x/oauth2 v0.0.0-20181101160152-c453e0c75759 // indirect
|
||||
golang.org/x/oauth2 v0.0.0-20181101160152-c453e0c75759
|
||||
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223
|
||||
golang.org/x/text v0.3.0
|
||||
gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect
|
||||
|
|
|
|||
11
go.sum
11
go.sum
|
|
@ -142,6 +142,12 @@ github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pO
|
|||
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
|
||||
github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
|
||||
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
|
||||
github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY=
|
||||
github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
|
||||
github.com/google/go-github/v24 v24.0.1 h1:KCt1LjMJEey1qvPXxa9SjaWxwTsCWSq6p2Ju57UR4Q4=
|
||||
github.com/google/go-github/v24 v24.0.1/go.mod h1:CRqaW1Uns1TCkP0wqTpxYyRxRjxwvKU/XSS44u6X74M=
|
||||
github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
|
||||
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
|
||||
github.com/gorilla/context v1.1.1 h1:AWwleXJkX/nhcU9bZSnZoi3h/qGYqQAGhq6zZe/aQW8=
|
||||
|
|
@ -309,17 +315,21 @@ github.com/ziutek/mymysql v1.5.4 h1:GB0qdRGsTwQSBVYuVShFBKaXSnSnYYC2d9knnE1LHFs=
|
|||
github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0=
|
||||
go.etcd.io/bbolt v1.3.2 h1:Z/90sZLPOeCy2PwprqkFa25PdkusRzaj9P8zm/KNyvk=
|
||||
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
|
||||
golang.org/x/crypto v0.0.0-20180820150726-614d502a4dac/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793 h1:u+LnwYTOOW7Ukr/fppxEb1Nwz0AtPflrblfvUudpo+I=
|
||||
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519 h1:x6rhz8Y9CjbgQkccRGmELH6K+LJj7tOoh3XWeC1yaQM=
|
||||
golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/oauth2 v0.0.0-20180620175406-ef147856a6dd/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20181101160152-c453e0c75759 h1:TMrx+Qdx7uJAeUbv15N72h5Hmyb5+VDjEiMufAEAM04=
|
||||
golang.org/x/oauth2 v0.0.0-20181101160152-c453e0c75759/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA=
|
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20180824143301-4910a1d54f87/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180903190138-2b024373dcd9/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223 h1:DH4skfRX4EBpamg7iV4ZlCpblAHI6s6TDM39bFZumv8=
|
||||
|
|
@ -327,6 +337,7 @@ golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5h
|
|||
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||
google.golang.org/appengine v1.2.0 h1:S0iUepdCWODXRvtE+gcRDd15L+k+k1AiHlMiMjefH24=
|
||||
google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc h1:2gGKlE2+asNV9m7xrywl36YYNnBG5ZQ0r/BOOxqPpmk=
|
||||
|
|
|
|||
|
|
@ -0,0 +1,145 @@
|
|||
// Copyright 2019 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package models
|
||||
|
||||
import "github.com/go-xorm/xorm"
|
||||
|
||||
// InsertIssue insert one issue to database
|
||||
func InsertIssue(issue *Issue, labelIDs []int64) error {
|
||||
sess := x.NewSession()
|
||||
if err := sess.Begin(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := insertIssue(sess, issue, labelIDs); err != nil {
|
||||
return err
|
||||
}
|
||||
return sess.Commit()
|
||||
}
|
||||
|
||||
func insertIssue(sess *xorm.Session, issue *Issue, labelIDs []int64) error {
|
||||
if issue.MilestoneID > 0 {
|
||||
sess.Incr("num_issues")
|
||||
if issue.IsClosed {
|
||||
sess.Incr("num_closed_issues")
|
||||
}
|
||||
if _, err := sess.ID(issue.MilestoneID).NoAutoTime().Update(new(Milestone)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if _, err := sess.NoAutoTime().Insert(issue); err != nil {
|
||||
return err
|
||||
}
|
||||
var issueLabels = make([]IssueLabel, 0, len(labelIDs))
|
||||
for _, labelID := range labelIDs {
|
||||
issueLabels = append(issueLabels, IssueLabel{
|
||||
IssueID: issue.ID,
|
||||
LabelID: labelID,
|
||||
})
|
||||
}
|
||||
if _, err := sess.Insert(issueLabels); err != nil {
|
||||
return err
|
||||
}
|
||||
if !issue.IsPull {
|
||||
sess.ID(issue.RepoID).Incr("num_issues")
|
||||
if issue.IsClosed {
|
||||
sess.Incr("num_closed_issues")
|
||||
}
|
||||
} else {
|
||||
sess.ID(issue.RepoID).Incr("num_pulls")
|
||||
if issue.IsClosed {
|
||||
sess.Incr("num_closed_pulls")
|
||||
}
|
||||
}
|
||||
if _, err := sess.NoAutoTime().Update(issue.Repo); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
sess.Incr("num_issues")
|
||||
if issue.IsClosed {
|
||||
sess.Incr("num_closed_issues")
|
||||
}
|
||||
if _, err := sess.In("id", labelIDs).Update(new(Label)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if issue.MilestoneID > 0 {
|
||||
if _, err := sess.ID(issue.MilestoneID).SetExpr("completeness", "num_closed_issues * 100 / num_issues").Update(new(Milestone)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// InsertComment inserted a comment
|
||||
func InsertComment(comment *Comment) error {
|
||||
sess := x.NewSession()
|
||||
defer sess.Close()
|
||||
if err := sess.Begin(); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := sess.NoAutoTime().Insert(comment); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := sess.ID(comment.IssueID).Incr("num_comments").Update(new(Issue)); err != nil {
|
||||
return err
|
||||
}
|
||||
return sess.Commit()
|
||||
}
|
||||
|
||||
// InsertPullRequest inserted a pull request
|
||||
func InsertPullRequest(pr *PullRequest, labelIDs []int64) error {
|
||||
sess := x.NewSession()
|
||||
defer sess.Close()
|
||||
if err := sess.Begin(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := insertIssue(sess, pr.Issue, labelIDs); err != nil {
|
||||
return err
|
||||
}
|
||||
pr.IssueID = pr.Issue.ID
|
||||
if _, err := sess.NoAutoTime().Insert(pr); err != nil {
|
||||
return err
|
||||
}
|
||||
return sess.Commit()
|
||||
}
|
||||
|
||||
// MigrateRelease migrates release
|
||||
func MigrateRelease(rel *Release) error {
|
||||
sess := x.NewSession()
|
||||
if err := sess.Begin(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var oriRel = Release{
|
||||
RepoID: rel.RepoID,
|
||||
TagName: rel.TagName,
|
||||
}
|
||||
exist, err := sess.Get(&oriRel)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !exist {
|
||||
if _, err := sess.NoAutoTime().Insert(rel); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
rel.ID = oriRel.ID
|
||||
if _, err := sess.ID(rel.ID).Cols("target, title, note, is_tag, num_commits").Update(rel); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
for i := 0; i < len(rel.Attachments); i++ {
|
||||
rel.Attachments[i].ReleaseID = rel.ID
|
||||
}
|
||||
|
||||
if _, err := sess.NoAutoTime().Insert(rel.Attachments); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return sess.Commit()
|
||||
}
|
||||
|
|
@ -107,6 +107,7 @@ func TestRelease_MirrorDelete(t *testing.T) {
|
|||
IsPrivate: false,
|
||||
IsMirror: true,
|
||||
RemoteAddr: repoPath,
|
||||
Wiki: true,
|
||||
}
|
||||
mirror, err := MigrateRepository(user, user, migrationOptions)
|
||||
assert.NoError(t, err)
|
||||
|
|
|
|||
|
|
@ -896,6 +896,7 @@ type MigrateRepoOptions struct {
|
|||
IsPrivate bool
|
||||
IsMirror bool
|
||||
RemoteAddr string
|
||||
Wiki bool // include wiki repository
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
@ -917,7 +918,7 @@ func wikiRemoteURL(remote string) string {
|
|||
return ""
|
||||
}
|
||||
|
||||
// MigrateRepository migrates a existing repository from other project hosting.
|
||||
// MigrateRepository migrates an existing repository from other project hosting.
|
||||
func MigrateRepository(doer, u *User, opts MigrateRepoOptions) (*Repository, error) {
|
||||
repo, err := CreateRepository(doer, u, CreateRepoOptions{
|
||||
Name: opts.Name,
|
||||
|
|
@ -930,7 +931,6 @@ func MigrateRepository(doer, u *User, opts MigrateRepoOptions) (*Repository, err
|
|||
}
|
||||
|
||||
repoPath := RepoPath(u.Name, opts.Name)
|
||||
wikiPath := WikiPath(u.Name, opts.Name)
|
||||
|
||||
if u.IsOrganization() {
|
||||
t, err := u.GetOwnerTeam()
|
||||
|
|
@ -956,22 +956,25 @@ func MigrateRepository(doer, u *User, opts MigrateRepoOptions) (*Repository, err
|
|||
return repo, fmt.Errorf("Clone: %v", err)
|
||||
}
|
||||
|
||||
wikiRemotePath := wikiRemoteURL(opts.RemoteAddr)
|
||||
if len(wikiRemotePath) > 0 {
|
||||
if err := os.RemoveAll(wikiPath); err != nil {
|
||||
return repo, fmt.Errorf("Failed to remove %s: %v", wikiPath, err)
|
||||
}
|
||||
|
||||
if err = git.Clone(wikiRemotePath, wikiPath, git.CloneRepoOptions{
|
||||
Mirror: true,
|
||||
Quiet: true,
|
||||
Timeout: migrateTimeout,
|
||||
Branch: "master",
|
||||
}); err != nil {
|
||||
log.Warn("Clone wiki: %v", err)
|
||||
if opts.Wiki {
|
||||
wikiPath := WikiPath(u.Name, opts.Name)
|
||||
wikiRemotePath := wikiRemoteURL(opts.RemoteAddr)
|
||||
if len(wikiRemotePath) > 0 {
|
||||
if err := os.RemoveAll(wikiPath); err != nil {
|
||||
return repo, fmt.Errorf("Failed to remove %s: %v", wikiPath, err)
|
||||
}
|
||||
|
||||
if err = git.Clone(wikiRemotePath, wikiPath, git.CloneRepoOptions{
|
||||
Mirror: true,
|
||||
Quiet: true,
|
||||
Timeout: migrateTimeout,
|
||||
Branch: "master",
|
||||
}); err != nil {
|
||||
log.Warn("Clone wiki: %v", err)
|
||||
if err := os.RemoveAll(wikiPath); err != nil {
|
||||
return repo, fmt.Errorf("Failed to remove %s: %v", wikiPath, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -51,10 +51,16 @@ type MigrateRepoForm struct {
|
|||
// required: true
|
||||
UID int64 `json:"uid" binding:"Required"`
|
||||
// required: true
|
||||
RepoName string `json:"repo_name" binding:"Required;AlphaDashDot;MaxSize(100)"`
|
||||
Mirror bool `json:"mirror"`
|
||||
Private bool `json:"private"`
|
||||
Description string `json:"description" binding:"MaxSize(255)"`
|
||||
RepoName string `json:"repo_name" binding:"Required;AlphaDashDot;MaxSize(100)"`
|
||||
Mirror bool `json:"mirror"`
|
||||
Private bool `json:"private"`
|
||||
Description string `json:"description" binding:"MaxSize(255)"`
|
||||
Wiki bool `json:"wiki"`
|
||||
Milestones bool `json:"milestones"`
|
||||
Labels bool `json:"labels"`
|
||||
Issues bool `json:"issues"`
|
||||
PullRequests bool `json:"pull_requests"`
|
||||
Releases bool `json:"releases"`
|
||||
}
|
||||
|
||||
// Validate validates the fields
|
||||
|
|
|
|||
|
|
@ -0,0 +1,17 @@
|
|||
// Copyright 2019 The Gitea Authors. All rights reserved.
|
||||
// Copyright 2018 Jonas Franz. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package base
|
||||
|
||||
import "time"
|
||||
|
||||
// Comment is a standard comment information
|
||||
type Comment struct {
|
||||
PosterName string
|
||||
PosterEmail string
|
||||
Created time.Time
|
||||
Content string
|
||||
Reactions *Reactions
|
||||
}
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
// Copyright 2019 The Gitea Authors. All rights reserved.
|
||||
// Copyright 2018 Jonas Franz. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package base
|
||||
|
||||
// Downloader downloads the site repo informations
|
||||
type Downloader interface {
|
||||
GetRepoInfo() (*Repository, error)
|
||||
GetMilestones() ([]*Milestone, error)
|
||||
GetReleases() ([]*Release, error)
|
||||
GetLabels() ([]*Label, error)
|
||||
GetIssues(start, limit int) ([]*Issue, error)
|
||||
GetComments(issueNumber int64) ([]*Comment, error)
|
||||
GetPullRequests(start, limit int) ([]*PullRequest, error)
|
||||
}
|
||||
|
||||
// DownloaderFactory defines an interface to match a downloader implementation and create a downloader
|
||||
type DownloaderFactory interface {
|
||||
Match(opts MigrateOptions) (bool, error)
|
||||
New(opts MigrateOptions) (Downloader, error)
|
||||
}
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
// Copyright 2019 The Gitea Authors. All rights reserved.
|
||||
// Copyright 2018 Jonas Franz. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package base
|
||||
|
||||
import "time"
|
||||
|
||||
// Issue is a standard issue information
|
||||
type Issue struct {
|
||||
Number int64
|
||||
PosterName string
|
||||
PosterEmail string
|
||||
Title string
|
||||
Content string
|
||||
Milestone string
|
||||
State string // closed, open
|
||||
IsLocked bool
|
||||
Created time.Time
|
||||
Closed *time.Time
|
||||
Labels []*Label
|
||||
Reactions *Reactions
|
||||
}
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
// Copyright 2019 The Gitea Authors. All rights reserved.
|
||||
// Copyright 2018 Jonas Franz. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package base
|
||||
|
||||
// Label defines a standard label informations
|
||||
type Label struct {
|
||||
Name string
|
||||
Color string
|
||||
Description string
|
||||
}
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
// Copyright 2019 The Gitea Authors. All rights reserved.
|
||||
// Copyright 2018 Jonas Franz. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package base
|
||||
|
||||
import "time"
|
||||
|
||||
// Milestone defines a standard milestone
|
||||
type Milestone struct {
|
||||
Title string
|
||||
Description string
|
||||
Deadline *time.Time
|
||||
Created time.Time
|
||||
Updated *time.Time
|
||||
Closed *time.Time
|
||||
State string
|
||||
}
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
// Copyright 2019 The Gitea Authors. All rights reserved.
|
||||
// Copyright 2018 Jonas Franz. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package base
|
||||
|
||||
// MigrateOptions defines the way a repository gets migrated
|
||||
type MigrateOptions struct {
|
||||
RemoteURL string
|
||||
AuthUsername string
|
||||
AuthPassword string
|
||||
Name string
|
||||
Description string
|
||||
|
||||
Wiki bool
|
||||
Issues bool
|
||||
Milestones bool
|
||||
Labels bool
|
||||
Releases bool
|
||||
Comments bool
|
||||
PullRequests bool
|
||||
Private bool
|
||||
Mirror bool
|
||||
IgnoreIssueAuthor bool // if true will not add original author information before issues or comments content.
|
||||
}
|
||||
|
|
@ -0,0 +1,53 @@
|
|||
// Copyright 2019 The Gitea Authors. All rights reserved.
|
||||
// Copyright 2018 Jonas Franz. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package base
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
// PullRequest defines a standard pull request information
|
||||
type PullRequest struct {
|
||||
Number int64
|
||||
Title string
|
||||
PosterName string
|
||||
PosterEmail string
|
||||
Content string
|
||||
Milestone string
|
||||
State string
|
||||
Created time.Time
|
||||
Closed *time.Time
|
||||
Labels []*Label
|
||||
PatchURL string
|
||||
Merged bool
|
||||
MergedTime *time.Time
|
||||
MergeCommitSHA string
|
||||
Head PullRequestBranch
|
||||
Base PullRequestBranch
|
||||
Assignee string
|
||||
Assignees []string
|
||||
IsLocked bool
|
||||
}
|
||||
|
||||
// IsForkPullRequest returns true if the pull request from a forked repository but not the same repository
|
||||
func (p *PullRequest) IsForkPullRequest() bool {
|
||||
return p.Head.RepoPath() != p.Base.RepoPath()
|
||||
}
|
||||
|
||||
// PullRequestBranch represents a pull request branch
|
||||
type PullRequestBranch struct {
|
||||
CloneURL string
|
||||
Ref string
|
||||
SHA string
|
||||
RepoName string
|
||||
OwnerName string
|
||||
}
|
||||
|
||||
// RepoPath returns pull request repo path
|
||||
func (p PullRequestBranch) RepoPath() string {
|
||||
return fmt.Sprintf("%s/%s", p.OwnerName, p.RepoName)
|
||||
}
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
// Copyright 2019 The Gitea Authors. All rights reserved.
|
||||
// Copyright 2018 Jonas Franz. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package base
|
||||
|
||||
// Reactions represents a summary of reactions.
|
||||
type Reactions struct {
|
||||
TotalCount int
|
||||
PlusOne int
|
||||
MinusOne int
|
||||
Laugh int
|
||||
Confused int
|
||||
Heart int
|
||||
Hooray int
|
||||
}
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
// Copyright 2019 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package base
|
||||
|
||||
import "time"
|
||||
|
||||
// ReleaseAsset represents a release asset
|
||||
type ReleaseAsset struct {
|
||||
URL string
|
||||
Name string
|
||||
ContentType *string
|
||||
Size *int
|
||||
DownloadCount *int
|
||||
Created time.Time
|
||||
Updated time.Time
|
||||
}
|
||||
|
||||
// Release represents a release
|
||||
type Release struct {
|
||||
TagName string
|
||||
TargetCommitish string
|
||||
Name string
|
||||
Body string
|
||||
Draft bool
|
||||
Prerelease bool
|
||||
Assets []ReleaseAsset
|
||||
Created time.Time
|
||||
Published time.Time
|
||||
}
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
// Copyright 2019 The Gitea Authors. All rights reserved.
|
||||
// Copyright 2018 Jonas Franz. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package base
|
||||
|
||||
// Repository defines a standard repository information
|
||||
type Repository struct {
|
||||
Name string
|
||||
Owner string
|
||||
IsPrivate bool
|
||||
IsMirror bool
|
||||
Description string
|
||||
AuthUsername string
|
||||
AuthPassword string
|
||||
CloneURL string
|
||||
}
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
// Copyright 2019 The Gitea Authors. All rights reserved.
|
||||
// Copyright 2018 Jonas Franz. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package base
|
||||
|
||||
// Uploader uploads all the informations
|
||||
type Uploader interface {
|
||||
CreateRepo(repo *Repository, includeWiki bool) error
|
||||
CreateMilestone(milestone *Milestone) error
|
||||
CreateRelease(release *Release) error
|
||||
CreateLabel(label *Label) error
|
||||
CreateIssue(issue *Issue) error
|
||||
CreateComment(issueNumber int64, comment *Comment) error
|
||||
CreatePullRequest(pr *PullRequest) error
|
||||
Rollback() error
|
||||
}
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
// Copyright 2019 The Gitea Authors. All rights reserved.
|
||||
// Copyright 2018 Jonas Franz. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package migrations
|
||||
|
||||
import (
|
||||
"errors"
|
||||
|
||||
"github.com/google/go-github/v24/github"
|
||||
)
|
||||
|
||||
var (
|
||||
// ErrNotSupported returns the error not supported
|
||||
ErrNotSupported = errors.New("not supported")
|
||||
)
|
||||
|
||||
// IsRateLimitError returns true if the err is github.RateLimitError
|
||||
func IsRateLimitError(err error) bool {
|
||||
_, ok := err.(*github.RateLimitError)
|
||||
return ok
|
||||
}
|
||||
|
||||
// IsTwoFactorAuthError returns true if the err is github.TwoFactorAuthError
|
||||
func IsTwoFactorAuthError(err error) bool {
|
||||
_, ok := err.(*github.TwoFactorAuthError)
|
||||
return ok
|
||||
}
|
||||
|
|
@ -0,0 +1,69 @@
|
|||
// Copyright 2019 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package migrations
|
||||
|
||||
import (
|
||||
"code.gitea.io/gitea/modules/migrations/base"
|
||||
)
|
||||
|
||||
var (
|
||||
_ base.Downloader = &PlainGitDownloader{}
|
||||
)
|
||||
|
||||
// PlainGitDownloader implements a Downloader interface to clone git from a http/https URL
|
||||
type PlainGitDownloader struct {
|
||||
ownerName string
|
||||
repoName string
|
||||
remoteURL string
|
||||
}
|
||||
|
||||
// NewPlainGitDownloader creates a git Downloader
|
||||
func NewPlainGitDownloader(ownerName, repoName, remoteURL string) *PlainGitDownloader {
|
||||
return &PlainGitDownloader{
|
||||
ownerName: ownerName,
|
||||
repoName: repoName,
|
||||
remoteURL: remoteURL,
|
||||
}
|
||||
}
|
||||
|
||||
// GetRepoInfo returns a repository information
|
||||
func (g *PlainGitDownloader) GetRepoInfo() (*base.Repository, error) {
|
||||
// convert github repo to stand Repo
|
||||
return &base.Repository{
|
||||
Owner: g.ownerName,
|
||||
Name: g.repoName,
|
||||
CloneURL: g.remoteURL,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// GetMilestones returns milestones
|
||||
func (g *PlainGitDownloader) GetMilestones() ([]*base.Milestone, error) {
|
||||
return nil, ErrNotSupported
|
||||
}
|
||||
|
||||
// GetLabels returns labels
|
||||
func (g *PlainGitDownloader) GetLabels() ([]*base.Label, error) {
|
||||
return nil, ErrNotSupported
|
||||
}
|
||||
|
||||
// GetReleases returns releases
|
||||
func (g *PlainGitDownloader) GetReleases() ([]*base.Release, error) {
|
||||
return nil, ErrNotSupported
|
||||
}
|
||||
|
||||
// GetIssues returns issues according start and limit
|
||||
func (g *PlainGitDownloader) GetIssues(start, limit int) ([]*base.Issue, error) {
|
||||
return nil, ErrNotSupported
|
||||
}
|
||||
|
||||
// GetComments returns comments according issueNumber
|
||||
func (g *PlainGitDownloader) GetComments(issueNumber int64) ([]*base.Comment, error) {
|
||||
return nil, ErrNotSupported
|
||||
}
|
||||
|
||||
// GetPullRequests returns pull requests according start and limit
|
||||
func (g *PlainGitDownloader) GetPullRequests(start, limit int) ([]*base.PullRequest, error) {
|
||||
return nil, ErrNotSupported
|
||||
}
|
||||
|
|
@ -0,0 +1,403 @@
|
|||
// Copyright 2019 The Gitea Authors. All rights reserved.
|
||||
// Copyright 2018 Jonas Franz. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package migrations
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"code.gitea.io/gitea/models"
|
||||
"code.gitea.io/gitea/modules/git"
|
||||
"code.gitea.io/gitea/modules/log"
|
||||
"code.gitea.io/gitea/modules/migrations/base"
|
||||
"code.gitea.io/gitea/modules/setting"
|
||||
"code.gitea.io/gitea/modules/util"
|
||||
|
||||
gouuid "github.com/satori/go.uuid"
|
||||
)
|
||||
|
||||
var (
|
||||
_ base.Uploader = &GiteaLocalUploader{}
|
||||
)
|
||||
|
||||
// GiteaLocalUploader implements an Uploader to gitea sites
|
||||
type GiteaLocalUploader struct {
|
||||
doer *models.User
|
||||
repoOwner string
|
||||
repoName string
|
||||
repo *models.Repository
|
||||
labels sync.Map
|
||||
milestones sync.Map
|
||||
issues sync.Map
|
||||
gitRepo *git.Repository
|
||||
prHeadCache map[string]struct{}
|
||||
}
|
||||
|
||||
// NewGiteaLocalUploader creates an gitea Uploader via gitea API v1
|
||||
func NewGiteaLocalUploader(doer *models.User, repoOwner, repoName string) *GiteaLocalUploader {
|
||||
return &GiteaLocalUploader{
|
||||
doer: doer,
|
||||
repoOwner: repoOwner,
|
||||
repoName: repoName,
|
||||
prHeadCache: make(map[string]struct{}),
|
||||
}
|
||||
}
|
||||
|
||||
// CreateRepo creates a repository
|
||||
func (g *GiteaLocalUploader) CreateRepo(repo *base.Repository, includeWiki bool) error {
|
||||
owner, err := models.GetUserByName(g.repoOwner)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
r, err := models.MigrateRepository(g.doer, owner, models.MigrateRepoOptions{
|
||||
Name: g.repoName,
|
||||
Description: repo.Description,
|
||||
IsMirror: repo.IsMirror,
|
||||
RemoteAddr: repo.CloneURL,
|
||||
IsPrivate: repo.IsPrivate,
|
||||
Wiki: includeWiki,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
g.repo = r
|
||||
g.gitRepo, err = git.OpenRepository(r.RepoPath())
|
||||
return err
|
||||
}
|
||||
|
||||
// CreateMilestone creates milestone
|
||||
func (g *GiteaLocalUploader) CreateMilestone(milestone *base.Milestone) error {
|
||||
var deadline util.TimeStamp
|
||||
if milestone.Deadline != nil {
|
||||
deadline = util.TimeStamp(milestone.Deadline.Unix())
|
||||
}
|
||||
if deadline == 0 {
|
||||
deadline = util.TimeStamp(time.Date(9999, 1, 1, 0, 0, 0, 0, setting.UILocation).Unix())
|
||||
}
|
||||
var ms = models.Milestone{
|
||||
RepoID: g.repo.ID,
|
||||
Name: milestone.Title,
|
||||
Content: milestone.Description,
|
||||
IsClosed: milestone.State == "close",
|
||||
DeadlineUnix: deadline,
|
||||
}
|
||||
if ms.IsClosed && milestone.Closed != nil {
|
||||
ms.ClosedDateUnix = util.TimeStamp(milestone.Closed.Unix())
|
||||
}
|
||||
err := models.NewMilestone(&ms)
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
g.milestones.Store(ms.Name, ms.ID)
|
||||
return nil
|
||||
}
|
||||
|
||||
// CreateLabel creates label
|
||||
func (g *GiteaLocalUploader) CreateLabel(label *base.Label) error {
|
||||
var lb = models.Label{
|
||||
RepoID: g.repo.ID,
|
||||
Name: label.Name,
|
||||
Description: label.Description,
|
||||
Color: fmt.Sprintf("#%s", label.Color),
|
||||
}
|
||||
err := models.NewLabel(&lb)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
g.labels.Store(lb.Name, lb.ID)
|
||||
return nil
|
||||
}
|
||||
|
||||
// CreateRelease creates release
|
||||
func (g *GiteaLocalUploader) CreateRelease(release *base.Release) error {
|
||||
var rel = models.Release{
|
||||
RepoID: g.repo.ID,
|
||||
PublisherID: g.doer.ID,
|
||||
TagName: release.TagName,
|
||||
LowerTagName: strings.ToLower(release.TagName),
|
||||
Target: release.TargetCommitish,
|
||||
Title: release.Name,
|
||||
Sha1: release.TargetCommitish,
|
||||
Note: release.Body,
|
||||
IsDraft: release.Draft,
|
||||
IsPrerelease: release.Prerelease,
|
||||
IsTag: false,
|
||||
CreatedUnix: util.TimeStamp(release.Created.Unix()),
|
||||
}
|
||||
|
||||
// calc NumCommits
|
||||
commit, err := g.gitRepo.GetCommit(rel.TagName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("GetCommit: %v", err)
|
||||
}
|
||||
rel.NumCommits, err = commit.CommitsCount()
|
||||
if err != nil {
|
||||
return fmt.Errorf("CommitsCount: %v", err)
|
||||
}
|
||||
|
||||
for _, asset := range release.Assets {
|
||||
var attach = models.Attachment{
|
||||
UUID: gouuid.NewV4().String(),
|
||||
Name: asset.Name,
|
||||
DownloadCount: int64(*asset.DownloadCount),
|
||||
Size: int64(*asset.Size),
|
||||
CreatedUnix: util.TimeStamp(asset.Created.Unix()),
|
||||
}
|
||||
|
||||
// download attachment
|
||||
resp, err := http.Get(asset.URL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
localPath := attach.LocalPath()
|
||||
if err = os.MkdirAll(path.Dir(localPath), os.ModePerm); err != nil {
|
||||
return fmt.Errorf("MkdirAll: %v", err)
|
||||
}
|
||||
|
||||
fw, err := os.Create(localPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Create: %v", err)
|
||||
}
|
||||
defer fw.Close()
|
||||
|
||||
if _, err := io.Copy(fw, resp.Body); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
rel.Attachments = append(rel.Attachments, &attach)
|
||||
}
|
||||
|
||||
return models.MigrateRelease(&rel)
|
||||
}
|
||||
|
||||
// CreateIssue creates issue
|
||||
func (g *GiteaLocalUploader) CreateIssue(issue *base.Issue) error {
|
||||
var labelIDs []int64
|
||||
for _, label := range issue.Labels {
|
||||
id, ok := g.labels.Load(label.Name)
|
||||
if !ok {
|
||||
return fmt.Errorf("Label %s missing when create issue", label.Name)
|
||||
}
|
||||
labelIDs = append(labelIDs, id.(int64))
|
||||
}
|
||||
|
||||
var milestoneID int64
|
||||
if issue.Milestone != "" {
|
||||
milestone, ok := g.milestones.Load(issue.Milestone)
|
||||
if !ok {
|
||||
return fmt.Errorf("Milestone %s missing when create issue", issue.Milestone)
|
||||
}
|
||||
milestoneID = milestone.(int64)
|
||||
}
|
||||
|
||||
var is = models.Issue{
|
||||
RepoID: g.repo.ID,
|
||||
Repo: g.repo,
|
||||
Index: issue.Number,
|
||||
PosterID: g.doer.ID,
|
||||
Title: issue.Title,
|
||||
Content: issue.Content,
|
||||
IsClosed: issue.State == "closed",
|
||||
IsLocked: issue.IsLocked,
|
||||
MilestoneID: milestoneID,
|
||||
CreatedUnix: util.TimeStamp(issue.Created.Unix()),
|
||||
}
|
||||
if issue.Closed != nil {
|
||||
is.ClosedUnix = util.TimeStamp(issue.Closed.Unix())
|
||||
}
|
||||
|
||||
err := models.InsertIssue(&is, labelIDs)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
g.issues.Store(issue.Number, is.ID)
|
||||
// TODO: add reactions
|
||||
return err
|
||||
}
|
||||
|
||||
// CreateComment creates comment
|
||||
func (g *GiteaLocalUploader) CreateComment(issueNumber int64, comment *base.Comment) error {
|
||||
var issueID int64
|
||||
if issueIDStr, ok := g.issues.Load(issueNumber); !ok {
|
||||
issue, err := models.GetIssueByIndex(g.repo.ID, issueNumber)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
issueID = issue.ID
|
||||
g.issues.Store(issueNumber, issueID)
|
||||
} else {
|
||||
issueID = issueIDStr.(int64)
|
||||
}
|
||||
|
||||
var cm = models.Comment{
|
||||
IssueID: issueID,
|
||||
Type: models.CommentTypeComment,
|
||||
PosterID: g.doer.ID,
|
||||
Content: comment.Content,
|
||||
CreatedUnix: util.TimeStamp(comment.Created.Unix()),
|
||||
}
|
||||
err := models.InsertComment(&cm)
|
||||
// TODO: Reactions
|
||||
return err
|
||||
}
|
||||
|
||||
// CreatePullRequest creates pull request
|
||||
func (g *GiteaLocalUploader) CreatePullRequest(pr *base.PullRequest) error {
|
||||
var labelIDs []int64
|
||||
for _, label := range pr.Labels {
|
||||
id, ok := g.labels.Load(label.Name)
|
||||
if !ok {
|
||||
return fmt.Errorf("Label %s missing when create issue", label.Name)
|
||||
}
|
||||
labelIDs = append(labelIDs, id.(int64))
|
||||
}
|
||||
|
||||
var milestoneID int64
|
||||
if pr.Milestone != "" {
|
||||
milestone, ok := g.milestones.Load(pr.Milestone)
|
||||
if !ok {
|
||||
return fmt.Errorf("Milestone %s missing when create issue", pr.Milestone)
|
||||
}
|
||||
milestoneID = milestone.(int64)
|
||||
}
|
||||
|
||||
// download patch file
|
||||
resp, err := http.Get(pr.PatchURL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
pullDir := filepath.Join(g.repo.RepoPath(), "pulls")
|
||||
if err = os.MkdirAll(pullDir, os.ModePerm); err != nil {
|
||||
return err
|
||||
}
|
||||
f, err := os.Create(filepath.Join(pullDir, fmt.Sprintf("%d.patch", pr.Number)))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
_, err = io.Copy(f, resp.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// set head information
|
||||
pullHead := filepath.Join(g.repo.RepoPath(), "refs", "pull", fmt.Sprintf("%d", pr.Number))
|
||||
if err := os.MkdirAll(pullHead, os.ModePerm); err != nil {
|
||||
return err
|
||||
}
|
||||
p, err := os.Create(filepath.Join(pullHead, "head"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer p.Close()
|
||||
_, err = p.WriteString(pr.Head.SHA)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var head = "unknown repository"
|
||||
if pr.IsForkPullRequest() {
|
||||
if pr.Head.OwnerName != "" {
|
||||
remote := pr.Head.OwnerName
|
||||
_, ok := g.prHeadCache[remote]
|
||||
if !ok {
|
||||
// git remote add
|
||||
err := g.gitRepo.AddRemote(remote, pr.Head.CloneURL, true)
|
||||
if err != nil {
|
||||
log.Error("AddRemote failed: %s", err)
|
||||
} else {
|
||||
g.prHeadCache[remote] = struct{}{}
|
||||
ok = true
|
||||
}
|
||||
}
|
||||
|
||||
if ok {
|
||||
_, err = git.NewCommand("fetch", remote, pr.Head.Ref).RunInDir(g.repo.RepoPath())
|
||||
if err != nil {
|
||||
log.Error("Fetch branch from %s failed: %v", pr.Head.CloneURL, err)
|
||||
} else {
|
||||
headBranch := filepath.Join(g.repo.RepoPath(), "refs", "heads", pr.Head.OwnerName, pr.Head.Ref)
|
||||
if err := os.MkdirAll(filepath.Dir(headBranch), os.ModePerm); err != nil {
|
||||
return err
|
||||
}
|
||||
b, err := os.Create(headBranch)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer b.Close()
|
||||
_, err = b.WriteString(pr.Head.SHA)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
head = pr.Head.OwnerName + "/" + pr.Head.Ref
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
head = pr.Head.Ref
|
||||
}
|
||||
|
||||
var pullRequest = models.PullRequest{
|
||||
HeadRepoID: g.repo.ID,
|
||||
HeadBranch: head,
|
||||
HeadUserName: g.repoOwner,
|
||||
BaseRepoID: g.repo.ID,
|
||||
BaseBranch: pr.Base.Ref,
|
||||
MergeBase: pr.Base.SHA,
|
||||
Index: pr.Number,
|
||||
HasMerged: pr.Merged,
|
||||
|
||||
Issue: &models.Issue{
|
||||
RepoID: g.repo.ID,
|
||||
Repo: g.repo,
|
||||
Title: pr.Title,
|
||||
Index: pr.Number,
|
||||
PosterID: g.doer.ID,
|
||||
Content: pr.Content,
|
||||
MilestoneID: milestoneID,
|
||||
IsPull: true,
|
||||
IsClosed: pr.State == "closed",
|
||||
IsLocked: pr.IsLocked,
|
||||
CreatedUnix: util.TimeStamp(pr.Created.Unix()),
|
||||
},
|
||||
}
|
||||
|
||||
if pullRequest.Issue.IsClosed && pr.Closed != nil {
|
||||
pullRequest.Issue.ClosedUnix = util.TimeStamp(pr.Closed.Unix())
|
||||
}
|
||||
if pullRequest.HasMerged && pr.MergedTime != nil {
|
||||
pullRequest.MergedUnix = util.TimeStamp(pr.MergedTime.Unix())
|
||||
pullRequest.MergedCommitID = pr.MergeCommitSHA
|
||||
pullRequest.MergerID = g.doer.ID
|
||||
}
|
||||
|
||||
// TODO: reactions
|
||||
// TODO: assignees
|
||||
|
||||
return models.InsertPullRequest(&pullRequest, labelIDs)
|
||||
}
|
||||
|
||||
// Rollback when migrating failed, this will rollback all the changes.
|
||||
func (g *GiteaLocalUploader) Rollback() error {
|
||||
if g.repo != nil && g.repo.ID > 0 {
|
||||
if err := models.DeleteRepository(g.doer, g.repo.OwnerID, g.repo.ID); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
@ -0,0 +1,95 @@
|
|||
// Copyright 2019 The Gitea Authors. All rights reserved.
|
||||
// Copyright 2018 Jonas Franz. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package migrations
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"code.gitea.io/gitea/models"
|
||||
"code.gitea.io/gitea/modules/util"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestGiteaUploadRepo(t *testing.T) {
|
||||
// FIXME: Since no accesskey or user/password will trigger rate limit of github, just skip
|
||||
t.Skip()
|
||||
|
||||
models.PrepareTestEnv(t)
|
||||
|
||||
user := models.AssertExistsAndLoadBean(t, &models.User{ID: 1}).(*models.User)
|
||||
|
||||
var (
|
||||
downloader = NewGithubDownloaderV3("", "", "go-xorm", "builder")
|
||||
repoName = "builder-" + time.Now().Format("2006-01-02-15-04-05")
|
||||
uploader = NewGiteaLocalUploader(user, user.Name, repoName)
|
||||
)
|
||||
|
||||
err := migrateRepository(downloader, uploader, MigrateOptions{
|
||||
RemoteURL: "https://github.com/go-xorm/builder",
|
||||
Name: repoName,
|
||||
AuthUsername: "",
|
||||
|
||||
Wiki: true,
|
||||
Issues: true,
|
||||
Milestones: true,
|
||||
Labels: true,
|
||||
Releases: true,
|
||||
Comments: true,
|
||||
PullRequests: true,
|
||||
Private: true,
|
||||
Mirror: false,
|
||||
IgnoreIssueAuthor: false,
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
|
||||
repo := models.AssertExistsAndLoadBean(t, &models.Repository{OwnerID: user.ID, Name: repoName}).(*models.Repository)
|
||||
assert.True(t, repo.HasWiki())
|
||||
|
||||
milestones, err := models.GetMilestones(repo.ID, 0, false, "")
|
||||
assert.NoError(t, err)
|
||||
assert.EqualValues(t, 1, len(milestones))
|
||||
|
||||
milestones, err = models.GetMilestones(repo.ID, 0, true, "")
|
||||
assert.NoError(t, err)
|
||||
assert.EqualValues(t, 0, len(milestones))
|
||||
|
||||
labels, err := models.GetLabelsByRepoID(repo.ID, "")
|
||||
assert.NoError(t, err)
|
||||
assert.EqualValues(t, 11, len(labels))
|
||||
|
||||
releases, err := models.GetReleasesByRepoID(repo.ID, models.FindReleasesOptions{
|
||||
IncludeTags: true,
|
||||
}, 0, 10)
|
||||
assert.NoError(t, err)
|
||||
assert.EqualValues(t, 8, len(releases))
|
||||
|
||||
releases, err = models.GetReleasesByRepoID(repo.ID, models.FindReleasesOptions{
|
||||
IncludeTags: false,
|
||||
}, 0, 10)
|
||||
assert.NoError(t, err)
|
||||
assert.EqualValues(t, 1, len(releases))
|
||||
|
||||
issues, err := models.Issues(&models.IssuesOptions{
|
||||
RepoIDs: []int64{repo.ID},
|
||||
IsPull: util.OptionalBoolFalse,
|
||||
SortType: "oldest",
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
assert.EqualValues(t, 14, len(issues))
|
||||
assert.NoError(t, issues[0].LoadDiscussComments())
|
||||
assert.EqualValues(t, 0, len(issues[0].Comments))
|
||||
|
||||
pulls, _, err := models.PullRequests(repo.ID, &models.PullRequestsOptions{
|
||||
SortType: "oldest",
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
assert.EqualValues(t, 34, len(pulls))
|
||||
assert.NoError(t, pulls[0].LoadIssue())
|
||||
assert.NoError(t, pulls[0].Issue.LoadDiscussComments())
|
||||
assert.EqualValues(t, 2, len(pulls[0].Issue.Comments))
|
||||
}
|
||||
|
|
@ -0,0 +1,475 @@
|
|||
// Copyright 2019 The Gitea Authors. All rights reserved.
|
||||
// Copyright 2018 Jonas Franz. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package migrations
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
|
||||
"code.gitea.io/gitea/modules/log"
|
||||
"code.gitea.io/gitea/modules/migrations/base"
|
||||
|
||||
"github.com/google/go-github/v24/github"
|
||||
"golang.org/x/oauth2"
|
||||
)
|
||||
|
||||
var (
|
||||
_ base.Downloader = &GithubDownloaderV3{}
|
||||
_ base.DownloaderFactory = &GithubDownloaderV3Factory{}
|
||||
)
|
||||
|
||||
func init() {
|
||||
RegisterDownloaderFactory(&GithubDownloaderV3Factory{})
|
||||
}
|
||||
|
||||
// GithubDownloaderV3Factory defines a github downloader v3 factory
|
||||
type GithubDownloaderV3Factory struct {
|
||||
}
|
||||
|
||||
// Match returns ture if the migration remote URL matched this downloader factory
|
||||
func (f *GithubDownloaderV3Factory) Match(opts base.MigrateOptions) (bool, error) {
|
||||
u, err := url.Parse(opts.RemoteURL)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
return u.Host == "github.com" && opts.AuthUsername != "", nil
|
||||
}
|
||||
|
||||
// New returns a Downloader related to this factory according MigrateOptions
|
||||
func (f *GithubDownloaderV3Factory) New(opts base.MigrateOptions) (base.Downloader, error) {
|
||||
u, err := url.Parse(opts.RemoteURL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
fields := strings.Split(u.Path, "/")
|
||||
oldOwner := fields[1]
|
||||
oldName := strings.TrimSuffix(fields[2], ".git")
|
||||
|
||||
log.Trace("Create github downloader: %s/%s", oldOwner, oldName)
|
||||
|
||||
return NewGithubDownloaderV3(opts.AuthUsername, opts.AuthPassword, oldOwner, oldName), nil
|
||||
}
|
||||
|
||||
// GithubDownloaderV3 implements a Downloader interface to get repository informations
|
||||
// from github via APIv3
|
||||
type GithubDownloaderV3 struct {
|
||||
ctx context.Context
|
||||
client *github.Client
|
||||
repoOwner string
|
||||
repoName string
|
||||
userName string
|
||||
password string
|
||||
}
|
||||
|
||||
// NewGithubDownloaderV3 creates a github Downloader via github v3 API
|
||||
func NewGithubDownloaderV3(userName, password, repoOwner, repoName string) *GithubDownloaderV3 {
|
||||
var downloader = GithubDownloaderV3{
|
||||
userName: userName,
|
||||
password: password,
|
||||
ctx: context.Background(),
|
||||
repoOwner: repoOwner,
|
||||
repoName: repoName,
|
||||
}
|
||||
|
||||
var client *http.Client
|
||||
if userName != "" {
|
||||
if password == "" {
|
||||
ts := oauth2.StaticTokenSource(
|
||||
&oauth2.Token{AccessToken: userName},
|
||||
)
|
||||
client = oauth2.NewClient(downloader.ctx, ts)
|
||||
} else {
|
||||
client = &http.Client{
|
||||
Transport: &http.Transport{
|
||||
Proxy: func(req *http.Request) (*url.URL, error) {
|
||||
req.SetBasicAuth(userName, password)
|
||||
return nil, nil
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
downloader.client = github.NewClient(client)
|
||||
return &downloader
|
||||
}
|
||||
|
||||
// GetRepoInfo returns a repository information
|
||||
func (g *GithubDownloaderV3) GetRepoInfo() (*base.Repository, error) {
|
||||
gr, _, err := g.client.Repositories.Get(g.ctx, g.repoOwner, g.repoName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// convert github repo to stand Repo
|
||||
return &base.Repository{
|
||||
Owner: g.repoOwner,
|
||||
Name: gr.GetName(),
|
||||
IsPrivate: *gr.Private,
|
||||
Description: gr.GetDescription(),
|
||||
CloneURL: gr.GetCloneURL(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// GetMilestones returns milestones
|
||||
func (g *GithubDownloaderV3) GetMilestones() ([]*base.Milestone, error) {
|
||||
var perPage = 100
|
||||
var milestones = make([]*base.Milestone, 0, perPage)
|
||||
for i := 1; ; i++ {
|
||||
ms, _, err := g.client.Issues.ListMilestones(g.ctx, g.repoOwner, g.repoName,
|
||||
&github.MilestoneListOptions{
|
||||
State: "all",
|
||||
ListOptions: github.ListOptions{
|
||||
Page: i,
|
||||
PerPage: perPage,
|
||||
}})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, m := range ms {
|
||||
var desc string
|
||||
if m.Description != nil {
|
||||
desc = *m.Description
|
||||
}
|
||||
var state = "open"
|
||||
if m.State != nil {
|
||||
state = *m.State
|
||||
}
|
||||
milestones = append(milestones, &base.Milestone{
|
||||
Title: *m.Title,
|
||||
Description: desc,
|
||||
Deadline: m.DueOn,
|
||||
State: state,
|
||||
Created: *m.CreatedAt,
|
||||
Updated: m.UpdatedAt,
|
||||
Closed: m.ClosedAt,
|
||||
})
|
||||
}
|
||||
if len(ms) < perPage {
|
||||
break
|
||||
}
|
||||
}
|
||||
return milestones, nil
|
||||
}
|
||||
|
||||
func convertGithubLabel(label *github.Label) *base.Label {
|
||||
var desc string
|
||||
if label.Description != nil {
|
||||
desc = *label.Description
|
||||
}
|
||||
return &base.Label{
|
||||
Name: *label.Name,
|
||||
Color: *label.Color,
|
||||
Description: desc,
|
||||
}
|
||||
}
|
||||
|
||||
// GetLabels returns labels
|
||||
func (g *GithubDownloaderV3) GetLabels() ([]*base.Label, error) {
|
||||
var perPage = 100
|
||||
var labels = make([]*base.Label, 0, perPage)
|
||||