mirror of https://github.com/go-gitea/gitea.git
Improve migrations to support migrating milestones/labels/issues/comments/pullrequests (#6290)
* add migrations * fix package dependency * fix lints * implements migrations except pull requests * add releases * migrating releases * fix bug * fix lint * fix migrate releases * fix tests * add rollback * pull request migtations * fix import * fix go module vendor * add tests for upload to gitea * more migrate options * fix swagger-check * fix misspell * add options on migration UI * fix log error * improve UI options on migrating * add support for username password when migrating from github * fix tests * remove comments and fix migrate limitation * improve error handles * migrate API will also support migrate milestones/labels/issues/pulls/releases * fix tests and remove unused codes * add DownloaderFactory and docs about how to create a new Downloader * fix misspell * fix migration docs * Add hints about migrate options on migration page * fix testspull/6872/head
Lunny Xiao
4 years ago
committed by
GitHub
parent
1c7c739eb9
commit
08069dc465
@ -0,0 +1,72 @@ |
||||
--- |
||||
date: "2019-04-15T17:29:00+08:00" |
||||
title: "Advanced: Migrations Interfaces" |
||||
slug: "migrations-interfaces" |
||||
weight: 30 |
||||
toc: true |
||||
draft: false |
||||
menu: |
||||
sidebar: |
||||
parent: "advanced" |
||||
name: "Migrations Interfaces" |
||||
weight: 55 |
||||
identifier: "migrations-interfaces" |
||||
--- |
||||
|
||||
# Migration Features |
||||
|
||||
The new migration features were introduced in Gitea 1.9.0. It defines two interfaces to support migrating |
||||
repositories data from other git host platforms to gitea or, in the future migrating gitea data to other |
||||
git host platforms. Currently, only the migrations from github via APIv3 to Gitea is implemented. |
||||
|
||||
First of all, Gitea defines some standard objects in packages `modules/migrations/base`. They are |
||||
`Repository`, `Milestone`, `Release`, `Label`, `Issue`, `Comment`, `PullRequest`. |
||||
|
||||
## Downloader Interfaces |
||||
|
||||
To migrate from a new git host platform, there are two steps to be updated. |
||||
|
||||
- You should implement a `Downloader` which will get all kinds of repository informations. |
||||
- You should implement a `DownloaderFactory` which is used to detect if the URL matches and |
||||
create a Downloader. |
||||
- You'll need to register the `DownloaderFactory` via `RegisterDownloaderFactory` on init. |
||||
|
||||
```Go |
||||
type Downloader interface { |
||||
GetRepoInfo() (*Repository, error) |
||||
GetMilestones() ([]*Milestone, error) |
||||
GetReleases() ([]*Release, error) |
||||
GetLabels() ([]*Label, error) |
||||
GetIssues(start, limit int) ([]*Issue, error) |
||||
GetComments(issueNumber int64) ([]*Comment, error) |
||||
GetPullRequests(start, limit int) ([]*PullRequest, error) |
||||
} |
||||
``` |
||||
|
||||
```Go |
||||
type DownloaderFactory interface { |
||||
Match(opts MigrateOptions) (bool, error) |
||||
New(opts MigrateOptions) (Downloader, error) |
||||
} |
||||
``` |
||||
|
||||
## Uploader Interface |
||||
|
||||
Currently, only a `GiteaLocalUploader` is implemented, so we only save downloaded |
||||
data via this `Uploader` on the local Gitea instance. Other uploaders are not supported |
||||
and will be implemented in future. |
||||
|
||||
```Go |
||||
// Uploader uploads all the informations |
||||
type Uploader interface { |
||||
CreateRepo(repo *Repository, includeWiki bool) error |
||||
CreateMilestone(milestone *Milestone) error |
||||
CreateRelease(release *Release) error |
||||
CreateLabel(label *Label) error |
||||
CreateIssue(issue *Issue) error |
||||
CreateComment(issueNumber int64, comment *Comment) error |
||||
CreatePullRequest(pr *PullRequest) error |
||||
Rollback() error |
||||
} |
||||
|
||||
``` |
||||
@ -0,0 +1,145 @@ |
||||
// Copyright 2019 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package models |
||||
|
||||
import "github.com/go-xorm/xorm" |
||||
|
||||
// InsertIssue insert one issue to database
|
||||
func InsertIssue(issue *Issue, labelIDs []int64) error { |
||||
sess := x.NewSession() |
||||
if err := sess.Begin(); err != nil { |
||||
return err |
||||
} |
||||
|
||||
if err := insertIssue(sess, issue, labelIDs); err != nil { |
||||
return err |
||||
} |
||||
return sess.Commit() |
||||
} |
||||
|
||||
func insertIssue(sess *xorm.Session, issue *Issue, labelIDs []int64) error { |
||||
if issue.MilestoneID > 0 { |
||||
sess.Incr("num_issues") |
||||
if issue.IsClosed { |
||||
sess.Incr("num_closed_issues") |
||||
} |
||||
if _, err := sess.ID(issue.MilestoneID).NoAutoTime().Update(new(Milestone)); err != nil { |
||||
return err |
||||
} |
||||
} |
||||
if _, err := sess.NoAutoTime().Insert(issue); err != nil { |
||||
return err |
||||
} |
||||
var issueLabels = make([]IssueLabel, 0, len(labelIDs)) |
||||
for _, labelID := range labelIDs { |
||||
issueLabels = append(issueLabels, IssueLabel{ |
||||
IssueID: issue.ID, |
||||
LabelID: labelID, |
||||
}) |
||||
} |
||||
if _, err := sess.Insert(issueLabels); err != nil { |
||||
return err |
||||
} |
||||
if !issue.IsPull { |
||||
sess.ID(issue.RepoID).Incr("num_issues") |
||||
if issue.IsClosed { |
||||
sess.Incr("num_closed_issues") |
||||
} |
||||
} else { |
||||
sess.ID(issue.RepoID).Incr("num_pulls") |
||||
if issue.IsClosed { |
||||
sess.Incr("num_closed_pulls") |
||||
} |
||||
} |
||||
if _, err := sess.NoAutoTime().Update(issue.Repo); err != nil { |
||||
return err |
||||
} |
||||
|
||||
sess.Incr("num_issues") |
||||
if issue.IsClosed { |
||||
sess.Incr("num_closed_issues") |
||||
} |
||||
if _, err := sess.In("id", labelIDs).Update(new(Label)); err != nil { |
||||
return err |
||||
} |
||||
|
||||
if issue.MilestoneID > 0 { |
||||
if _, err := sess.ID(issue.MilestoneID).SetExpr("completeness", "num_closed_issues * 100 / num_issues").Update(new(Milestone)); err != nil { |
||||
return err |
||||
} |
||||
} |
||||
|
||||
return nil |
||||
} |
||||
|
||||
// InsertComment inserted a comment
|
||||
func InsertComment(comment *Comment) error { |
||||
sess := x.NewSession() |
||||
defer sess.Close() |
||||
if err := sess.Begin(); err != nil { |
||||
return err |
||||
} |
||||
if _, err := sess.NoAutoTime().Insert(comment); err != nil { |
||||
return err |
||||
} |
||||
if _, err := sess.ID(comment.IssueID).Incr("num_comments").Update(new(Issue)); err != nil { |
||||
return err |
||||
} |
||||
return sess.Commit() |
||||
} |
||||
|
||||
// InsertPullRequest inserted a pull request
|
||||
func InsertPullRequest(pr *PullRequest, labelIDs []int64) error { |
||||
sess := x.NewSession() |
||||
defer sess.Close() |
||||
if err := sess.Begin(); err != nil { |
||||
return err |
||||
} |
||||
if err := insertIssue(sess, pr.Issue, labelIDs); err != nil { |
||||
return err |
||||
} |
||||
pr.IssueID = pr.Issue.ID |
||||
if _, err := sess.NoAutoTime().Insert(pr); err != nil { |
||||
return err |
||||
} |
||||
return sess.Commit() |
||||
} |
||||
|
||||
// MigrateRelease migrates release
|
||||
func MigrateRelease(rel *Release) error { |
||||
sess := x.NewSession() |
||||
if err := sess.Begin(); err != nil { |
||||
return err |
||||
} |
||||
|
||||
var oriRel = Release{ |
||||
RepoID: rel.RepoID, |
||||
TagName: rel.TagName, |
||||
} |
||||
exist, err := sess.Get(&oriRel) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
if !exist { |
||||
if _, err := sess.NoAutoTime().Insert(rel); err != nil { |
||||
return err |
||||
} |
||||
} else { |
||||
rel.ID = oriRel.ID |
||||
if _, err := sess.ID(rel.ID).Cols("target, title, note, is_tag, num_commits").Update(rel); err != nil { |
||||
return err |
||||
} |
||||
} |
||||
|
||||
for i := 0; i < len(rel.Attachments); i++ { |
||||
rel.Attachments[i].ReleaseID = rel.ID |
||||
} |
||||
|
||||
if _, err := sess.NoAutoTime().Insert(rel.Attachments); err != nil { |
||||
return err |
||||
} |
||||
|
||||
return sess.Commit() |
||||
} |
||||
@ -0,0 +1,17 @@ |
||||
// Copyright 2019 The Gitea Authors. All rights reserved.
|
||||
// Copyright 2018 Jonas Franz. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package base |
||||
|
||||
import "time" |
||||
|
||||
// Comment is a standard comment information
|
||||
type Comment struct { |
||||
PosterName string |
||||
PosterEmail string |
||||
Created time.Time |
||||
Content string |
||||
Reactions *Reactions |
||||
} |
||||
@ -0,0 +1,23 @@ |
||||
// Copyright 2019 The Gitea Authors. All rights reserved.
|
||||
// Copyright 2018 Jonas Franz. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package base |
||||
|
||||
// Downloader downloads the site repo informations
|
||||
type Downloader interface { |
||||
GetRepoInfo() (*Repository, error) |
||||
GetMilestones() ([]*Milestone, error) |
||||
GetReleases() ([]*Release, error) |
||||
GetLabels() ([]*Label, error) |
||||
GetIssues(start, limit int) ([]*Issue, error) |
||||
GetComments(issueNumber int64) ([]*Comment, error) |
||||
GetPullRequests(start, limit int) ([]*PullRequest, error) |
||||
} |
||||
|
||||
// DownloaderFactory defines an interface to match a downloader implementation and create a downloader
|
||||
type DownloaderFactory interface { |
||||
Match(opts MigrateOptions) (bool, error) |
||||
New(opts MigrateOptions) (Downloader, error) |
||||
} |
||||
@ -0,0 +1,24 @@ |
||||
// Copyright 2019 The Gitea Authors. All rights reserved.
|
||||
// Copyright 2018 Jonas Franz. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package base |
||||
|
||||
import "time" |
||||
|
||||
// Issue is a standard issue information
|
||||
type Issue struct { |
||||
Number int64 |
||||
PosterName string |
||||
PosterEmail string |
||||
Title string |
||||
Content string |
||||
Milestone string |
||||
State string // closed, open
|
||||
IsLocked bool |
||||
Created time.Time |
||||
Closed *time.Time |
||||
Labels []*Label |
||||
Reactions *Reactions |
||||
} |
||||
@ -0,0 +1,13 @@ |
||||
// Copyright 2019 The Gitea Authors. All rights reserved.
|
||||
// Copyright 2018 Jonas Franz. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package base |
||||
|
||||
// Label defines a standard label informations
|
||||
type Label struct { |
||||
Name string |
||||
Color string |
||||
Description string |
||||
} |
||||
@ -0,0 +1,19 @@ |
||||
// Copyright 2019 The Gitea Authors. All rights reserved.
|
||||
// Copyright 2018 Jonas Franz. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package base |
||||
|
||||
import "time" |
||||
|
||||
// Milestone defines a standard milestone
|
||||
type Milestone struct { |
||||
Title string |
||||
Description string |
||||
Deadline *time.Time |
||||
Created time.Time |
||||
Updated *time.Time |
||||
Closed *time.Time |
||||
State string |
||||
} |
||||
@ -0,0 +1,26 @@ |
||||
// Copyright 2019 The Gitea Authors. All rights reserved.
|
||||
// Copyright 2018 Jonas Franz. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package base |
||||
|
||||
// MigrateOptions defines the way a repository gets migrated
|
||||
type MigrateOptions struct { |
||||
RemoteURL string |
||||
AuthUsername string |
||||
AuthPassword string |
||||
Name string |
||||
Description string |
||||
|
||||
Wiki bool |
||||
Issues bool |
||||
Milestones bool |
||||
Labels bool |
||||
Releases bool |
||||
Comments bool |
||||
PullRequests bool |
||||
Private bool |
||||
Mirror bool |
||||
IgnoreIssueAuthor bool // if true will not add original author information before issues or comments content.
|
||||
} |
||||
@ -0,0 +1,53 @@ |
||||
// Copyright 2019 The Gitea Authors. All rights reserved.
|
||||
// Copyright 2018 Jonas Franz. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package base |
||||
|
||||
import ( |
||||
"fmt" |
||||
"time" |
||||
) |
||||
|
||||
// PullRequest defines a standard pull request information
|
||||
type PullRequest struct { |
||||
Number int64 |
||||
Title string |
||||
PosterName string |
||||
PosterEmail string |
||||
Content string |
||||
Milestone string |
||||
State string |
||||
Created time.Time |
||||
Closed *time.Time |
||||
Labels []*Label |
||||
PatchURL string |
||||
Merged bool |
||||
MergedTime *time.Time |
||||
MergeCommitSHA string |
||||
Head PullRequestBranch |
||||
Base PullRequestBranch |
||||
Assignee string |
||||
Assignees []string |
||||
IsLocked bool |
||||
} |
||||
|
||||
// IsForkPullRequest returns true if the pull request from a forked repository but not the same repository
|
||||
func (p *PullRequest) IsForkPullRequest() bool { |
||||
return p.Head.RepoPath() != p.Base.RepoPath() |
||||
} |
||||
|
||||
// PullRequestBranch represents a pull request branch
|
||||
type PullRequestBranch struct { |
||||
CloneURL string |
||||
Ref string |
||||
SHA string |
||||
RepoName string |
||||
OwnerName string |
||||
} |
||||
|
||||
// RepoPath returns pull request repo path
|
||||
func (p PullRequestBranch) RepoPath() string { |
||||
return fmt.Sprintf("%s/%s", p.OwnerName, p.RepoName) |
||||
} |
||||
@ -0,0 +1,17 @@ |
||||
// Copyright 2019 The Gitea Authors. All rights reserved.
|
||||
// Copyright 2018 Jonas Franz. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package base |
||||
|
||||
// Reactions represents a summary of reactions.
|
||||
type Reactions struct { |
||||
TotalCount int |
||||
PlusOne int |
||||
MinusOne int |
||||
Laugh int |
||||
Confused int |
||||
Heart int |
||||
Hooray int |
||||
} |
||||
@ -0,0 +1,31 @@ |
||||
// Copyright 2019 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package base |
||||
|
||||
import "time" |
||||
|
||||
// ReleaseAsset represents a release asset
|
||||
type ReleaseAsset struct { |
||||
URL string |
||||
Name string |
||||
ContentType *string |
||||
Size *int |
||||
DownloadCount *int |
||||
Created time.Time |
||||
Updated time.Time |
||||
} |
||||
|
||||
// Release represents a release
|
||||
type Release struct { |
||||
TagName string |
||||
TargetCommitish string |
||||
Name string |
||||
Body string |
||||
Draft bool |
||||
Prerelease bool |
||||
Assets []ReleaseAsset |
||||
Created time.Time |
||||
Published time.Time |
||||
} |
||||
@ -0,0 +1,18 @@ |
||||
// Copyright 2019 The Gitea Authors. All rights reserved.
|
||||
// Copyright 2018 Jonas Franz. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package base |
||||
|
||||
// Repository defines a standard repository information
|
||||
type Repository struct { |
||||
Name string |
||||
Owner string |
||||
IsPrivate bool |
||||
IsMirror bool |
||||
Description string |
||||
AuthUsername string |
||||
AuthPassword string |
||||
CloneURL string |
||||
} |
||||
@ -0,0 +1,18 @@ |
||||
// Copyright 2019 The Gitea Authors. All rights reserved.
|
||||
// Copyright 2018 Jonas Franz. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package base |
||||
|
||||
// Uploader uploads all the informations
|
||||
type Uploader interface { |
||||
CreateRepo(repo *Repository, includeWiki bool) error |
||||
CreateMilestone(milestone *Milestone) error |
||||
CreateRelease(release *Release) error |
||||
CreateLabel(label *Label) error |
||||
CreateIssue(issue *Issue) error |
||||
CreateComment(issueNumber int64, comment *Comment) error |
||||
CreatePullRequest(pr *PullRequest) error |
||||
Rollback() error |
||||
} |
||||
@ -0,0 +1,29 @@ |
||||
// Copyright 2019 The Gitea Authors. All rights reserved.
|
||||
// Copyright 2018 Jonas Franz. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package migrations |
||||
|
||||
import ( |
||||
"errors" |
||||
|
||||
"github.com/google/go-github/v24/github" |
||||
) |
||||
|
||||
var ( |
||||
// ErrNotSupported returns the error not supported
|
||||
ErrNotSupported = errors.New("not supported") |
||||
) |
||||
|
||||
// IsRateLimitError returns true if the err is github.RateLimitError
|
||||
func IsRateLimitError(err error) bool { |
||||
_, ok := err.(*github.RateLimitError) |
||||
return ok |
||||
} |
||||
|
||||
// IsTwoFactorAuthError returns true if the err is github.TwoFactorAuthError
|
||||
func IsTwoFactorAuthError(err error) bool { |
||||
_, ok := err.(*github.TwoFactorAuthError) |
||||
return ok |
||||
} |
||||
@ -0,0 +1,69 @@ |
||||
// Copyright 2019 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package migrations |
||||
|
||||
import ( |
||||
"code.gitea.io/gitea/modules/migrations/base" |
||||
) |
||||
|
||||
var ( |
||||
_ base.Downloader = &PlainGitDownloader{} |
||||
) |
||||
|
||||
// PlainGitDownloader implements a Downloader interface to clone git from a http/https URL
|
||||
type PlainGitDownloader struct { |
||||
ownerName string |
||||
repoName string |
||||
remoteURL string |
||||
} |
||||
|
||||
// NewPlainGitDownloader creates a git Downloader
|
||||
func NewPlainGitDownloader(ownerName, repoName, remoteURL string) *PlainGitDownloader { |
||||
return &PlainGitDownloader{ |
||||
ownerName: ownerName, |
||||
repoName: repoName, |
||||
remoteURL: remoteURL, |
||||
} |
||||
} |
||||
|
||||
// GetRepoInfo returns a repository information
|
||||
func (g *PlainGitDownloader) GetRepoInfo() (*base.Repository, error) { |
||||
// convert github repo to stand Repo
|
||||
return &base.Repository{ |
||||
Owner: g.ownerName, |
||||
Name: g.repoName, |
||||
CloneURL: g.remoteURL, |
||||
}, nil |
||||
} |
||||
|
||||
// GetMilestones returns milestones
|
||||
func (g *PlainGitDownloader) GetMilestones() ([]*base.Milestone, error) { |
||||
return nil, ErrNotSupported |
||||
} |
||||
|
||||
// GetLabels returns labels
|
||||
func (g *PlainGitDownloader) GetLabels() ([]*base.Label, error) { |
||||
return nil, ErrNotSupported |
||||
} |
||||
|
||||
// GetReleases returns releases
|
||||
func (g *PlainGitDownloader) GetReleases() ([]*base.Release, error) { |
||||
return nil, ErrNotSupported |
||||
} |
||||
|
||||
// GetIssues returns issues according start and limit
|
||||
func (g *PlainGitDownloader) GetIssues(start, limit int) ([]*base.Issue, error) { |
||||
return nil, ErrNotSupported |
||||
} |
||||
|
||||
// GetComments returns comments according issueNumber
|
||||
func (g *PlainGitDownloader) GetComments(issueNumber int64) ([]*base.Comment, error) { |
||||
return nil, ErrNotSupported |
||||
} |
||||
|
||||
// GetPullRequests returns pull requests according start and limit
|
||||
func (g *PlainGitDownloader) GetPullRequests(start, limit int) ([]*base.PullRequest, error) { |
||||
return nil, ErrNotSupported |
||||
} |
||||
@ -0,0 +1,403 @@ |
||||
// Copyright 2019 The Gitea Authors. All rights reserved.
|
||||
// Copyright 2018 Jonas Franz. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package migrations |
||||
|
||||
import ( |
||||
"fmt" |
||||
"io" |
||||
"net/http" |
||||
"os" |
||||
"path" |
||||
"path/filepath" |
||||
"strings" |
||||
"sync" |
||||
"time" |
||||
|
||||
"code.gitea.io/gitea/models" |
||||
"code.gitea.io/gitea/modules/git" |
||||
"code.gitea.io/gitea/modules/log" |
||||
"code.gitea.io/gitea/modules/migrations/base" |
||||
"code.gitea.io/gitea/modules/setting" |
||||
"code.gitea.io/gitea/modules/util" |
||||
|
||||
gouuid "github.com/satori/go.uuid" |
||||
) |
||||
|
||||
var ( |
||||
_ base.Uploader = &GiteaLocalUploader{} |
||||
) |
||||
|
||||
// GiteaLocalUploader implements an Uploader to gitea sites
|
||||
type GiteaLocalUploader struct { |
||||
doer *models.User |
||||
repoOwner string |
||||
repoName string |
||||
repo *models.Repository |
||||
labels sync.Map |
||||
milestones sync.Map |
||||
issues sync.Map |
||||
gitRepo *git.Repository |
||||
prHeadCache map[string]struct{} |
||||
} |
||||
|
||||
// NewGiteaLocalUploader creates an gitea Uploader via gitea API v1
|
||||
func NewGiteaLocalUploader(doer *models.User, repoOwner, repoName string) *GiteaLocalUploader { |
||||
return &GiteaLocalUploader{ |
||||
doer: doer, |
||||
repoOwner: repoOwner, |
||||
repoName: repoName, |
||||
prHeadCache: make(map[string]struct{}), |
||||
} |
||||
} |
||||
|
||||
// CreateRepo creates a repository
|
||||
func (g *GiteaLocalUploader) CreateRepo(repo *base.Repository, includeWiki bool) error { |
||||
owner, err := models.GetUserByName(g.repoOwner) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
r, err := models.MigrateRepository(g.doer, owner, models.MigrateRepoOptions{ |
||||
Name: g.repoName, |
||||
Description: repo.Description, |
||||
IsMirror: repo.IsMirror, |
||||
RemoteAddr: repo.CloneURL, |
||||
IsPrivate: repo.IsPrivate, |
||||
Wiki: includeWiki, |
||||
}) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
g.repo = r |
||||
g.gitRepo, err = git.OpenRepository(r.RepoPath()) |
||||
return err |
||||
} |
||||
|
||||
// CreateMilestone creates milestone
|
||||
func (g *GiteaLocalUploader) CreateMilestone(milestone *base.Milestone) error { |
||||
var deadline util.TimeStamp |
||||
if milestone.Deadline != nil { |
||||
deadline = util.TimeStamp(milestone.Deadline.Unix()) |
||||
} |
||||
if deadline == 0 { |
||||
deadline = util.TimeStamp(time.Date(9999, 1, 1, 0, 0, 0, 0, setting.UILocation).Unix()) |
||||
} |
||||
var ms = models.Milestone{ |
||||
RepoID: g.repo.ID, |
||||
Name: milestone.Title, |
||||
Content: milestone.Description, |
||||
IsClosed: milestone.State == "close", |
||||
DeadlineUnix: deadline, |
||||
} |
||||
if ms.IsClosed && milestone.Closed != nil { |
||||
ms.ClosedDateUnix = util.TimeStamp(milestone.Closed.Unix()) |
||||
} |
||||
err := models.NewMilestone(&ms) |
||||
|
||||
if err != nil { |
||||
return err |
||||
} |
||||
g.milestones.Store(ms.Name, ms.ID) |
||||
return nil |
||||
} |
||||
|
||||
// CreateLabel creates label
|
||||
func (g *GiteaLocalUploader) CreateLabel(label *base.Label) error { |
||||
var lb = models.Label{ |
||||
RepoID: g.repo.ID, |
||||
Name: label.Name, |
||||
Description: label.Description, |
||||
Color: fmt.Sprintf("#%s", label.Color), |
||||
} |
||||
err := models.NewLabel(&lb) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
g.labels.Store(lb.Name, lb.ID) |
||||
return nil |
||||
} |
||||
|
||||
// CreateRelease creates release
|
||||
func (g *GiteaLocalUploader) CreateRelease(release *base.Release) error { |
||||
var rel = models.Release{ |
||||
RepoID: g.repo.ID, |
||||
PublisherID: g.doer.ID, |
||||
TagName: release.TagName, |
||||
LowerTagName: strings.ToLower(release.TagName), |
||||
Target: release.TargetCommitish, |
||||
Title: release.Name, |
||||
Sha1: release.TargetCommitish, |
||||
Note: release.Body, |
||||
IsDraft: release.Draft, |
||||
IsPrerelease: release.Prerelease, |
||||
IsTag: false, |
||||
CreatedUnix: util.TimeStamp(release.Created.Unix()), |
||||
} |
||||
|
||||
// calc NumCommits
|
||||
commit, err := g.gitRepo.GetCommit(rel.TagName) |
||||
if err != nil { |
||||
return fmt.Errorf("GetCommit: %v", err) |
||||
} |
||||
rel.NumCommits, err = commit.CommitsCount() |
||||
if err != nil { |
||||
return fmt.Errorf("CommitsCount: %v", err) |
||||
} |
||||
|
||||
for _, asset := range release.Assets { |
||||
var attach = models.Attachment{ |
||||
UUID: gouuid.NewV4().String(), |
||||
Name: asset.Name, |
||||
DownloadCount: int64(*asset.DownloadCount), |
||||
Size: int64(*asset.Size), |
||||
CreatedUnix: util.TimeStamp(asset.Created.Unix()), |
||||
} |
||||
|
||||
// download attachment
|
||||
resp, err := http.Get(asset.URL) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
defer resp.Body.Close() |
||||
|
||||
localPath := attach.LocalPath() |
||||
if err = os.MkdirAll(path.Dir(localPath), os.ModePerm); err != nil { |
||||
return fmt.Errorf("MkdirAll: %v", err) |
||||
} |
||||
|
||||
fw, err := os.Create(localPath) |
||||
if err != nil { |
||||
return fmt.Errorf("Create: %v", err) |
||||
} |
||||
defer fw.Close() |
||||
|
||||
if _, err := io.Copy(fw, resp.Body); err != nil { |
||||
return err |
||||
} |
||||
|
||||
rel.Attachments = append(rel.Attachments, &attach) |
||||
} |
||||
|
||||
return models.MigrateRelease(&rel) |
||||
} |
||||
|
||||
// CreateIssue creates issue
|
||||
func (g *GiteaLocalUploader) CreateIssue(issue *base.Issue) error { |
||||
var labelIDs []int64 |
||||
for _, label := range issue.Labels { |
||||
id, ok := g.labels.Load(label.Name) |
||||
if !ok { |
||||
return fmt.Errorf("Label %s missing when create issue", label.Name) |
||||
} |
||||
labelIDs = append(labelIDs, id.(int64)) |
||||
} |
||||
|
||||
var milestoneID int64 |
||||
if issue.Milestone != "" { |
||||
milestone, ok := g.milestones.Load(issue.Milestone) |
||||
if !ok { |
||||
return fmt.Errorf("Milestone %s missing when create issue", issue.Milestone) |
||||
} |
||||
milestoneID = milestone.(int64) |
||||
} |
||||
|
||||
var is = models.Issue{ |
||||
RepoID: g.repo.ID, |
||||
Repo: g.repo, |
||||
Index: issue.Number, |
||||
PosterID: g.doer.ID, |
||||
Title: issue.Title, |
||||
Content: issue.Content, |
||||
IsClosed: issue.State == "closed", |
||||
IsLocked: issue.IsLocked, |
||||
MilestoneID: milestoneID, |
||||
CreatedUnix: util.TimeStamp(issue.Created.Unix()), |
||||
} |
||||
if issue.Closed != nil { |
||||
is.ClosedUnix = util.TimeStamp(issue.Closed.Unix()) |
||||
} |
||||
|
||||
err := models.InsertIssue(&is, labelIDs) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
g.issues.Store(issue.Number, is.ID) |
||||
// TODO: add reactions
|
||||
return err |
||||
} |
||||
|
||||
// CreateComment creates comment
|
||||
func (g *GiteaLocalUploader) CreateComment(issueNumber int64, comment *base.Comment) error { |
||||
var issueID int64 |
||||
if issueIDStr, ok := g.issues.Load(issueNumber); !ok { |
||||
issue, err := models.GetIssueByIndex(g.repo.ID, issueNumber) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
issueID = issue.ID |
||||
g.issues.Store(issueNumber, issueID) |
||||
} else { |
||||
issueID = issueIDStr.(int64) |
||||
} |
||||
|
||||
var cm = models.Comment{ |
||||
IssueID: issueID, |
||||
Type: models.CommentTypeComment, |
||||
PosterID: g.doer.ID, |
||||
Content: comment.Content, |
||||
CreatedUnix: util.TimeStamp(comment.Created.Unix()), |
||||
} |
||||
err := models.InsertComment(&cm) |
||||
// TODO: Reactions
|
||||
return err |
||||
} |
||||
|
||||
// CreatePullRequest creates pull request
|
||||
func (g *GiteaLocalUploader) CreatePullRequest(pr *base.PullRequest) error { |
||||
var labelIDs []int64 |
||||
for _, label := range pr.Labels { |
||||
id, ok := g.labels.Load(label.Name) |
||||
if !ok { |
||||
return fmt.Errorf("Label %s missing when create issue", label.Name) |
||||
} |
||||
labelIDs = append(labelIDs, id.(int64)) |
||||
} |
||||
|
||||
var milestoneID int64 |
||||
if pr.Milestone != "" { |
||||
milestone, ok := g.milestones.Load(pr.Milestone) |
||||
if !ok { |
||||
return fmt.Errorf("Milestone %s missing when create issue", pr.Milestone) |
||||
} |
||||
milestoneID = milestone.(int64) |
||||
} |
||||
|
||||
// download patch file
|
||||
resp, err := http.Get(pr.PatchURL) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
defer resp.Body.Close() |
||||
pullDir := filepath.Join(g.repo.RepoPath(), "pulls") |
||||
if err = os.MkdirAll(pullDir, os.ModePerm); err != nil { |
||||
return err |
||||
} |
||||
f, err := os.Create(filepath.Join(pullDir, fmt.Sprintf("%d.patch", pr.Number))) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
defer f.Close() |
||||
_, err = io.Copy(f, resp.Body) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
// set head information
|
||||
pullHead := filepath.Join(g.repo.RepoPath(), "refs", "pull", fmt.Sprintf("%d", pr.Number)) |
||||
if err := os.MkdirAll(pullHead, os.ModePerm); err != nil { |
||||
return err |
||||
} |
||||
p, err := os.Create(filepath.Join(pullHead, "head")) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
defer p.Close() |
||||
_, err = p.WriteString(pr.Head.SHA) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
var head = "unknown repository" |
||||
if pr.IsForkPullRequest() { |
||||
if pr.Head.OwnerName != "" { |
||||
remote := pr.Head.OwnerName |
||||
_, ok := g.prHeadCache[remote] |
||||
if !ok { |
||||
// git remote add
|
||||
err := g.gitRepo.AddRemote(remote, pr.Head.CloneURL, true) |
||||
if err != nil { |
||||
log.Error("AddRemote failed: %s", err) |
||||
} else { |
||||
g.prHeadCache[remote] = struct{}{} |
||||
ok = true |
||||
} |
||||
} |
||||
|
||||
if ok { |
||||
_, err = git.NewCommand("fetch", remote, pr.Head.Ref).RunInDir(g.repo.RepoPath()) |
||||
if err != nil { |
||||
log.Error("Fetch branch from %s failed: %v", pr.Head.CloneURL, err) |
||||
} else { |
||||
headBranch := filepath.Join(g.repo.RepoPath(), "refs", "heads", pr.Head.OwnerName, pr.Head.Ref) |
||||
if err := os.MkdirAll(filepath.Dir(headBranch), os.ModePerm); err != nil { |
||||
return err |
||||
} |
||||
b, err := os.Create(headBranch) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
defer b.Close() |
||||
_, err = b.WriteString(pr.Head.SHA) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
head = pr.Head.OwnerName + "/" + pr.Head.Ref |
||||
} |
||||
} |
||||
} |
||||
} else { |
||||
head = pr.Head.Ref |
||||
} |
||||
|
||||
var pullRequest = models.PullRequest{ |
||||
HeadRepoID: g.repo.ID, |
||||
HeadBranch: head, |
||||
HeadUserName: g.repoOwner, |
||||
BaseRepoID: g.repo.ID, |
||||
BaseBranch: pr.Base.Ref, |
||||
MergeBase: pr.Base.SHA, |
||||
Index: pr.Number, |
||||
HasMerged: pr.Merged, |
||||
|
||||
Issue: &models.Issue{ |
||||
RepoID: g.repo.ID, |
||||
Repo: g.repo, |
||||
Title: pr.Title, |
||||
Index: pr.Number, |
||||
PosterID: g.doer.ID, |
||||
Content: pr.Content, |
||||
MilestoneID: milestoneID, |
||||
IsPull: true, |
||||
IsClosed: pr.State == "closed", |
||||
IsLocked: pr.IsLocked, |
||||
CreatedUnix: util.TimeStamp(pr.Created.Unix()), |
||||
}, |
||||
} |
||||
|
||||
if pullRequest.Issue.IsClosed && pr.Closed != nil { |
||||
pullRequest.Issue.ClosedUnix = util.TimeStamp(pr.Closed.Unix()) |
||||
} |
||||
if pullRequest.HasMerged && pr.MergedTime != nil { |
||||
pullRequest.MergedUnix = util.TimeStamp(pr.MergedTime.Unix()) |
||||
pullRequest.MergedCommitID = pr.MergeCommitSHA |
||||
pullRequest.MergerID = g.doer.ID |
||||
} |
||||
|
||||
// TODO: reactions
|
||||
// TODO: assignees
|
||||
|
||||
return models.InsertPullRequest(&pullRequest, labelIDs) |
||||
} |
||||
|
||||
// Rollback when migrating failed, this will rollback all the changes.
|
||||
func (g *GiteaLocalUploader) Rollback() error { |
||||
if g.repo != nil && g.repo.ID > 0 { |
||||
if err := models.DeleteRepository(g.doer, g.repo.OwnerID, g.repo.ID); err != nil { |
||||
return err |
||||
} |
||||
} |
||||
return nil |
||||
} |
||||
@ -0,0 +1,95 @@ |
||||
// Copyright 2019 The Gitea Authors. All rights reserved.
|
||||
// Copyright 2018 Jonas Franz. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package migrations |
||||
|
||||
import ( |
||||
"testing" |
||||
"time" |
||||
|
||||
"code.gitea.io/gitea/models" |
||||
"code.gitea.io/gitea/modules/util" |
||||
|
||||
"github.com/stretchr/testify/assert" |
||||
) |
||||
|
||||
func TestGiteaUploadRepo(t *testing.T) { |
||||
// FIXME: Since no accesskey or user/password will trigger rate limit of github, just skip
|
||||
t.Skip() |
||||
|
||||
models.PrepareTestEnv(t) |
||||
|
||||
user := models.AssertExistsAndLoadBean(t, &models.User{ID: 1}).(*models.User) |
||||
|
||||
var ( |
||||
downloader = NewGithubDownloaderV3("", "", "go-xorm", "builder") |
||||
repoName = "builder-" + time.Now().Format("2006-01-02-15-04-05") |
||||
uploader = NewGiteaLocalUploader(user, user.Name, repoName) |
||||
) |
||||
|
||||
err := migrateRepository(downloader, uploader, MigrateOptions{ |
||||
RemoteURL: "https://github.com/go-xorm/builder", |
||||
Name: repoName, |
||||
AuthUsername: "", |
||||
|
||||
Wiki: true, |
||||
Issues: true, |
||||
Milestones: true, |
||||
Labels: true, |
||||
Releases: true, |
||||
Comments: true, |
||||
PullRequests: true, |
||||
Private: true, |
||||
Mirror: false, |
||||
IgnoreIssueAuthor: false, |
||||
}) |
||||
assert.NoError(t, err) |
||||
|
||||
repo := models.AssertExistsAndLoadBean(t, &models.Repository{OwnerID: user.ID, Name: repoName}).(*models.Repository) |
||||
assert.True(t, repo.HasWiki()) |
||||
|
||||
milestones, err := models.GetMilestones(repo.ID, 0, false, "") |
||||
assert.NoError(t, err) |
||||
assert.EqualValues(t, 1, len(milestones)) |
||||
|
||||
milestones, err = models.GetMilestones(repo.ID, 0, true, "") |
||||
assert.NoError(t, err) |
||||
assert.EqualValues(t, 0, len(milestones)) |
||||
|
||||
labels, err := models.GetLabelsByRepoID(repo.ID, "") |
||||
assert.NoError(t, err) |
||||
assert.EqualValues(t, 11, len(labels)) |
||||
|
||||
releases, err := models.GetReleasesByRepoID(repo.ID, models.FindReleasesOptions{ |
||||
IncludeTags: true, |
||||
}, 0, 10) |
||||
assert.NoError(t, err) |
||||
assert.EqualValues(t, 8, len(releases)) |
||||
|
||||
releases, err = models.GetReleasesByRepoID(repo.ID, models.FindReleasesOptions{ |
||||
IncludeTags: false, |
||||
}, 0, 10) |
||||
assert.NoError(t, err) |
||||
assert.EqualValues(t, 1, len(releases)) |
||||
|
||||
issues, err := models.Issues(&models.IssuesOptions{ |
||||
RepoIDs: []int64{repo.ID}, |
||||
IsPull: util.OptionalBoolFalse, |
||||
SortType: "oldest", |
||||
}) |
||||
assert.NoError(t, err) |
||||
assert.EqualValues(t, 14, len(issues)) |
||||
assert.NoError(t, issues[0].LoadDiscussComments()) |
||||
assert.EqualValues(t, 0, len(issues[0].Comments)) |
||||
|
||||
pulls, _, err := models.PullRequests(repo.ID, &models.PullRequestsOptions{ |
||||
SortType: "oldest", |
||||
}) |
||||
assert.NoError(t, err) |
||||
assert.EqualValues(t, 34, len(pulls)) |
||||
assert.NoError(t, pulls[0].LoadIssue()) |
||||
assert.NoError(t, pulls[0].Issue.LoadDiscussComments()) |
||||
assert.EqualValues(t, 2, len(pulls[0].Issue.Comments)) |
||||
} |
||||
@ -0,0 +1,475 @@ |
||||
// Copyright 2019 The Gitea Authors. All rights reserved.
|
||||
// Copyright 2018 Jonas Franz. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package migrations |
||||
|
||||
import ( |
||||
"context" |
||||
"fmt" |
||||
"net/http" |
||||
"net/url" |
||||
"strings" |
||||
|
||||
"code.gitea.io/gitea/modules/log" |
||||
"code.gitea.io/gitea/modules/migrations/base" |
||||
|
||||
"github.com/google/go-github/v24/github" |
||||
"golang.org/x/oauth2" |
||||
) |
||||
|
||||
var ( |
||||
_ base.Downloader = &GithubDownloaderV3{} |
||||
_ base.DownloaderFactory = &GithubDownloaderV3Factory{} |
||||
) |
||||
|
||||
func init() { |
||||
RegisterDownloaderFactory(&GithubDownloaderV3Factory{}) |
||||
} |
||||
|
||||
// GithubDownloaderV3Factory defines a github downloader v3 factory
|
||||
type GithubDownloaderV3Factory struct { |
||||
} |
||||
|
||||
// Match returns ture if the migration remote URL matched this downloader factory
|
||||
func (f *GithubDownloaderV3Factory) Match(opts base.MigrateOptions) (bool, error) { |
||||
u, err := url.Parse(opts.RemoteURL) |
||||
if err != nil { |
||||
return false, err |
||||
} |
||||
|
||||
return u.Host == "github.com" && opts.AuthUsername != "", nil |
||||
} |
||||
|
||||
// New returns a Downloader related to this factory according MigrateOptions
|
||||
func (f *GithubDownloaderV3Factory) New(opts base.MigrateOptions) (base.Downloader, error) { |
||||
u, err := url.Parse(opts.RemoteURL) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
fields := strings.Split(u.Path, "/") |
||||
oldOwner := fields[1] |
||||
oldName := strings.TrimSuffix(fields[2], ".git") |
||||
|
||||