gitea/services/migrations/onedev.go

638 lines
15 KiB
Go

// Copyright 2021 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package migrations
import (
"context"
"fmt"
"net/http"
"net/url"
"strconv"
"strings"
"time"
"code.gitea.io/gitea/modules/json"
"code.gitea.io/gitea/modules/log"
base "code.gitea.io/gitea/modules/migration"
"code.gitea.io/gitea/modules/structs"
)
var (
_ base.Downloader = &OneDevDownloader{}
_ base.DownloaderFactory = &OneDevDownloaderFactory{}
)
func init() {
RegisterDownloaderFactory(&OneDevDownloaderFactory{})
}
// OneDevDownloaderFactory defines a downloader factory
type OneDevDownloaderFactory struct{}
// New returns a downloader related to this factory according MigrateOptions
func (f *OneDevDownloaderFactory) New(ctx context.Context, opts base.MigrateOptions) (base.Downloader, error) {
u, err := url.Parse(opts.CloneAddr)
if err != nil {
return nil, err
}
var repoName string
fields := strings.Split(strings.Trim(u.Path, "/"), "/")
if len(fields) == 2 && fields[0] == "projects" {
repoName = fields[1]
} else if len(fields) == 1 {
repoName = fields[0]
} else {
return nil, fmt.Errorf("invalid path: %s", u.Path)
}
u.Path = ""
u.Fragment = ""
log.Trace("Create onedev downloader. BaseURL: %v RepoName: %s", u, repoName)
return NewOneDevDownloader(ctx, u, opts.AuthUsername, opts.AuthPassword, repoName), nil
}
// GitServiceType returns the type of git service
func (f *OneDevDownloaderFactory) GitServiceType() structs.GitServiceType {
return structs.OneDevService
}
type onedevUser struct {
ID int64 `json:"id"`
Name string `json:"name"`
Email string `json:"email"`
}
// OneDevDownloader implements a Downloader interface to get repository information
// from OneDev
type OneDevDownloader struct {
base.NullDownloader
ctx context.Context
client *http.Client
baseURL *url.URL
repoName string
repoID int64
maxIssueIndex int64
userMap map[int64]*onedevUser
milestoneMap map[int64]string
}
// SetContext set context
func (d *OneDevDownloader) SetContext(ctx context.Context) {
d.ctx = ctx
}
// NewOneDevDownloader creates a new downloader
func NewOneDevDownloader(ctx context.Context, baseURL *url.URL, username, password, repoName string) *OneDevDownloader {
downloader := &OneDevDownloader{
ctx: ctx,
baseURL: baseURL,
repoName: repoName,
client: &http.Client{
Transport: &http.Transport{
Proxy: func(req *http.Request) (*url.URL, error) {
if len(username) > 0 && len(password) > 0 {
req.SetBasicAuth(username, password)
}
return nil, nil
},
},
},
userMap: make(map[int64]*onedevUser),
milestoneMap: make(map[int64]string),
}
return downloader
}
// String implements Stringer
func (d *OneDevDownloader) String() string {
return fmt.Sprintf("migration from oneDev server %s [%d]/%s", d.baseURL, d.repoID, d.repoName)
}
// ColorFormat provides a basic color format for a OneDevDownloader
func (d *OneDevDownloader) ColorFormat(s fmt.State) {
if d == nil {
log.ColorFprintf(s, "<nil: OneDevDownloader>")
return
}
log.ColorFprintf(s, "migration from oneDev server %s [%d]/%s", d.baseURL, d.repoID, d.repoName)
}
func (d *OneDevDownloader) callAPI(endpoint string, parameter map[string]string, result interface{}) error {
u, err := d.baseURL.Parse(endpoint)
if err != nil {
return err
}
if parameter != nil {
query := u.Query()
for k, v := range parameter {
query.Set(k, v)
}
u.RawQuery = query.Encode()
}
req, err := http.NewRequestWithContext(d.ctx, "GET", u.String(), nil)
if err != nil {
return err
}
resp, err := d.client.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
decoder := json.NewDecoder(resp.Body)
return decoder.Decode(&result)
}
// GetRepoInfo returns repository information
func (d *OneDevDownloader) GetRepoInfo() (*base.Repository, error) {
info := make([]struct {
ID int64 `json:"id"`
Name string `json:"name"`
Description string `json:"description"`
}, 0, 1)
err := d.callAPI(
"/api/projects",
map[string]string{
"query": `"Name" is "` + d.repoName + `"`,
"offset": "0",
"count": "1",
},
&info,
)
if err != nil {
return nil, err
}
if len(info) != 1 {
return nil, fmt.Errorf("Project %s not found", d.repoName)
}
d.repoID = info[0].ID
cloneURL, err := d.baseURL.Parse(info[0].Name)
if err != nil {
return nil, err
}
originalURL, err := d.baseURL.Parse("/projects/" + info[0].Name)
if err != nil {
return nil, err
}
return &base.Repository{
Name: info[0].Name,
Description: info[0].Description,
CloneURL: cloneURL.String(),
OriginalURL: originalURL.String(),
}, nil
}
// GetMilestones returns milestones
func (d *OneDevDownloader) GetMilestones() ([]*base.Milestone, error) {
rawMilestones := make([]struct {
ID int64 `json:"id"`
Name string `json:"name"`
Description string `json:"description"`
DueDate *time.Time `json:"dueDate"`
Closed bool `json:"closed"`
}, 0, 100)
endpoint := fmt.Sprintf("/api/projects/%d/milestones", d.repoID)
milestones := make([]*base.Milestone, 0, 100)
offset := 0
for {
err := d.callAPI(
endpoint,
map[string]string{
"offset": strconv.Itoa(offset),
"count": "100",
},
&rawMilestones,
)
if err != nil {
return nil, err
}
if len(rawMilestones) == 0 {
break
}
offset += 100
for _, milestone := range rawMilestones {
d.milestoneMap[milestone.ID] = milestone.Name
closed := milestone.DueDate
if !milestone.Closed {
closed = nil
}
milestones = append(milestones, &base.Milestone{
Title: milestone.Name,
Description: milestone.Description,
Deadline: milestone.DueDate,
Closed: closed,
})
}
}
return milestones, nil
}
// GetLabels returns labels
func (d *OneDevDownloader) GetLabels() ([]*base.Label, error) {
return []*base.Label{
{
Name: "Bug",
Color: "f64e60",
},
{
Name: "Build Failure",
Color: "f64e60",
},
{
Name: "Discussion",
Color: "8950fc",
},
{
Name: "Improvement",
Color: "1bc5bd",
},
{
Name: "New Feature",
Color: "1bc5bd",
},
{
Name: "Support Request",
Color: "8950fc",
},
}, nil
}
type onedevIssueContext struct {
IsPullRequest bool
}
// GetIssues returns issues
func (d *OneDevDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, error) {
rawIssues := make([]struct {
ID int64 `json:"id"`
Number int64 `json:"number"`
State string `json:"state"`
Title string `json:"title"`
Description string `json:"description"`
SubmitterID int64 `json:"submitterId"`
SubmitDate time.Time `json:"submitDate"`
}, 0, perPage)
err := d.callAPI(
"/api/issues",
map[string]string{
"query": `"Project" is "` + d.repoName + `"`,
"offset": strconv.Itoa((page - 1) * perPage),
"count": strconv.Itoa(perPage),
},
&rawIssues,
)
if err != nil {
return nil, false, err
}
issues := make([]*base.Issue, 0, len(rawIssues))
for _, issue := range rawIssues {
fields := make([]struct {
Name string `json:"name"`
Value string `json:"value"`
}, 0, 10)
err := d.callAPI(
fmt.Sprintf("/api/issues/%d/fields", issue.ID),
nil,
&fields,
)
if err != nil {
return nil, false, err
}
var label *base.Label
for _, field := range fields {
if field.Name == "Type" {
label = &base.Label{Name: field.Value}
break
}
}
milestones := make([]struct {
ID int64 `json:"id"`
Name string `json:"name"`
}, 0, 10)
err = d.callAPI(
fmt.Sprintf("/api/issues/%d/milestones", issue.ID),
nil,
&milestones,
)
if err != nil {
return nil, false, err
}
milestoneID := int64(0)
if len(milestones) > 0 {
milestoneID = milestones[0].ID
}
state := strings.ToLower(issue.State)
if state == "released" {
state = "closed"
}
poster := d.tryGetUser(issue.SubmitterID)
issues = append(issues, &base.Issue{
Title: issue.Title,
Number: issue.Number,
PosterName: poster.Name,
PosterEmail: poster.Email,
Content: issue.Description,
Milestone: d.milestoneMap[milestoneID],
State: state,
Created: issue.SubmitDate,
Updated: issue.SubmitDate,
Labels: []*base.Label{label},
ForeignIndex: issue.ID,
Context: onedevIssueContext{IsPullRequest: false},
})
if d.maxIssueIndex < issue.Number {
d.maxIssueIndex = issue.Number
}
}
return issues, len(issues) == 0, nil
}
// GetComments returns comments
func (d *OneDevDownloader) GetComments(commentable base.Commentable) ([]*base.Comment, bool, error) {
context, ok := commentable.GetContext().(onedevIssueContext)
if !ok {
return nil, false, fmt.Errorf("unexpected context: %+v", commentable.GetContext())
}
rawComments := make([]struct {
ID int64 `json:"id"`
Date time.Time `json:"date"`
UserID int64 `json:"userId"`
Content string `json:"content"`
}, 0, 100)
var endpoint string
if context.IsPullRequest {
endpoint = fmt.Sprintf("/api/pull-requests/%d/comments", commentable.GetForeignIndex())
} else {
endpoint = fmt.Sprintf("/api/issues/%d/comments", commentable.GetForeignIndex())
}
err := d.callAPI(
endpoint,
nil,
&rawComments,
)
if err != nil {
return nil, false, err
}
rawChanges := make([]struct {
Date time.Time `json:"date"`
UserID int64 `json:"userId"`
Data map[string]interface{} `json:"data"`
}, 0, 100)
if context.IsPullRequest {
endpoint = fmt.Sprintf("/api/pull-requests/%d/changes", commentable.GetForeignIndex())
} else {
endpoint = fmt.Sprintf("/api/issues/%d/changes", commentable.GetForeignIndex())
}
err = d.callAPI(
endpoint,
nil,
&rawChanges,
)
if err != nil {
return nil, false, err
}
comments := make([]*base.Comment, 0, len(rawComments)+len(rawChanges))
for _, comment := range rawComments {
if len(comment.Content) == 0 {
continue
}
poster := d.tryGetUser(comment.UserID)
comments = append(comments, &base.Comment{
IssueIndex: commentable.GetLocalIndex(),
Index: comment.ID,
PosterID: poster.ID,
PosterName: poster.Name,
PosterEmail: poster.Email,
Content: comment.Content,
Created: comment.Date,
Updated: comment.Date,
})
}
for _, change := range rawChanges {
contentV, ok := change.Data["content"]
if !ok {
contentV, ok = change.Data["comment"]
if !ok {
continue
}
}
content, ok := contentV.(string)
if !ok || len(content) == 0 {
continue
}
poster := d.tryGetUser(change.UserID)
comments = append(comments, &base.Comment{
IssueIndex: commentable.GetLocalIndex(),
PosterID: poster.ID,
PosterName: poster.Name,
PosterEmail: poster.Email,
Content: content,
Created: change.Date,
Updated: change.Date,
})
}
return comments, true, nil
}
// GetPullRequests returns pull requests
func (d *OneDevDownloader) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) {
rawPullRequests := make([]struct {
ID int64 `json:"id"`
Number int64 `json:"number"`
Title string `json:"title"`
SubmitterID int64 `json:"submitterId"`
SubmitDate time.Time `json:"submitDate"`
Description string `json:"description"`
TargetBranch string `json:"targetBranch"`
SourceBranch string `json:"sourceBranch"`
BaseCommitHash string `json:"baseCommitHash"`
CloseInfo *struct {
Date *time.Time `json:"date"`
Status string `json:"status"`
}
}, 0, perPage)
err := d.callAPI(
"/api/pull-requests",
map[string]string{
"query": `"Target Project" is "` + d.repoName + `"`,
"offset": strconv.Itoa((page - 1) * perPage),
"count": strconv.Itoa(perPage),
},
&rawPullRequests,
)
if err != nil {
return nil, false, err
}
pullRequests := make([]*base.PullRequest, 0, len(rawPullRequests))
for _, pr := range rawPullRequests {
var mergePreview struct {
TargetHeadCommitHash string `json:"targetHeadCommitHash"`
HeadCommitHash string `json:"headCommitHash"`
MergeStrategy string `json:"mergeStrategy"`
MergeCommitHash string `json:"mergeCommitHash"`
}
err := d.callAPI(
fmt.Sprintf("/api/pull-requests/%d/merge-preview", pr.ID),
nil,
&mergePreview,
)
if err != nil {
return nil, false, err
}
state := "open"
merged := false
var closeTime *time.Time
var mergedTime *time.Time
if pr.CloseInfo != nil {
state = "closed"
closeTime = pr.CloseInfo.Date
if pr.CloseInfo.Status == "MERGED" { // "DISCARDED"
merged = true
mergedTime = pr.CloseInfo.Date
}
}
poster := d.tryGetUser(pr.SubmitterID)
number := pr.Number + d.maxIssueIndex
pullRequests = append(pullRequests, &base.PullRequest{
Title: pr.Title,
Number: number,
PosterName: poster.Name,
PosterID: poster.ID,
Content: pr.Description,
State: state,
Created: pr.SubmitDate,
Updated: pr.SubmitDate,
Closed: closeTime,
Merged: merged,
MergedTime: mergedTime,
Head: base.PullRequestBranch{
Ref: pr.SourceBranch,
SHA: mergePreview.HeadCommitHash,
RepoName: d.repoName,
},
Base: base.PullRequestBranch{
Ref: pr.TargetBranch,
SHA: mergePreview.TargetHeadCommitHash,
RepoName: d.repoName,
},
ForeignIndex: pr.ID,
Context: onedevIssueContext{IsPullRequest: true},
})
// SECURITY: Ensure that the PR is safe
_ = CheckAndEnsureSafePR(pullRequests[len(pullRequests)-1], d.baseURL.String(), d)
}
return pullRequests, len(pullRequests) == 0, nil
}
// GetReviews returns pull requests reviews
func (d *OneDevDownloader) GetReviews(reviewable base.Reviewable) ([]*base.Review, error) {
rawReviews := make([]struct {
ID int64 `json:"id"`
UserID int64 `json:"userId"`
Result *struct {
Commit string `json:"commit"`
Approved bool `json:"approved"`
Comment string `json:"comment"`
}
}, 0, 100)
err := d.callAPI(
fmt.Sprintf("/api/pull-requests/%d/reviews", reviewable.GetForeignIndex()),
nil,
&rawReviews,
)
if err != nil {
return nil, err
}
reviews := make([]*base.Review, 0, len(rawReviews))
for _, review := range rawReviews {
state := base.ReviewStatePending
content := ""
if review.Result != nil {
if len(review.Result.Comment) > 0 {
state = base.ReviewStateCommented
content = review.Result.Comment
}
if review.Result.Approved {
state = base.ReviewStateApproved
}
}
poster := d.tryGetUser(review.UserID)
reviews = append(reviews, &base.Review{
IssueIndex: reviewable.GetLocalIndex(),
ReviewerID: poster.ID,
ReviewerName: poster.Name,
Content: content,
State: state,
})
}
return reviews, nil
}
// GetTopics return repository topics
func (d *OneDevDownloader) GetTopics() ([]string, error) {
return []string{}, nil
}
func (d *OneDevDownloader) tryGetUser(userID int64) *onedevUser {
user, ok := d.userMap[userID]
if !ok {
err := d.callAPI(
fmt.Sprintf("/api/users/%d", userID),
nil,
&user,
)
if err != nil {
user = &onedevUser{
Name: fmt.Sprintf("User %d", userID),
}
}
d.userMap[userID] = user
}
return user
}