Wiki mirroring implementation (#3233)

* Implement wiki mirroring, add Update mirrors operation to admin dashboard

* bindata.go update after merge

* Implement checking Git repo endpoint existence, support for BB included

* Remove admin dashboard operation
Fix bindata.go

* Apply gofmt to repo model file

* Try to remove bindata from PR

* Revert accepted wiki names change in favor of better system

* Remove unused imports
This commit is contained in:
Andrew 2016-08-12 00:18:51 +07:00 committed by 无闻
parent 3380c946e1
commit 0885784f13
5 changed files with 100 additions and 8 deletions

View File

@ -659,6 +659,51 @@ type MigrateRepoOptions struct {
RemoteAddr string RemoteAddr string
} }
func isGitRepoURL(repoURL string, timeout time.Duration) bool {
cmd := git.NewCommand("ls-remote")
cmd.AddArguments("-q", "-h", repoURL, "HEAD")
res, err := cmd.RunTimeout(timeout)
if err != nil {
return false
}
if strings.Contains(res, "fatal") || strings.Contains(res, "not found") {
return false
}
return true
}
func wikiRemoteURL(remote string, timeout time.Duration) string {
wikiRemoteStd := remote
wikiRemoteBitBucket := remote
/*
GitHub, GitLab, Gogs: NAME.wiki.git
BitBucket: NAME.git/wiki
*/
gitSuffixed := strings.HasSuffix(remote, ".git")
if gitSuffixed {
wikiRemoteStd = wikiRemoteStd[:len(wikiRemoteStd)-4]
wikiRemoteBitBucket += "/wiki"
} else {
wikiRemoteBitBucket += ".git/wiki"
}
wikiRemoteStd += ".wiki.git"
isBB := strings.Contains(remote, "bitbucket")
if isBB {
if isGitRepoURL(wikiRemoteBitBucket, timeout) {
return wikiRemoteBitBucket
} else if isGitRepoURL(wikiRemoteStd, timeout) {
return wikiRemoteStd
}
return ""
}
if isGitRepoURL(wikiRemoteStd, timeout) {
return wikiRemoteStd
} else if isGitRepoURL(wikiRemoteBitBucket, timeout) {
return wikiRemoteBitBucket
}
return ""
}
// MigrateRepository migrates a existing repository from other project hosting. // MigrateRepository migrates a existing repository from other project hosting.
func MigrateRepository(u *User, opts MigrateRepoOptions) (*Repository, error) { func MigrateRepository(u *User, opts MigrateRepoOptions) (*Repository, error) {
repo, err := CreateRepository(u, CreateRepoOptions{ repo, err := CreateRepository(u, CreateRepoOptions{
@ -676,6 +721,7 @@ func MigrateRepository(u *User, opts MigrateRepoOptions) (*Repository, error) {
os.MkdirAll(tmpDir, os.ModePerm) os.MkdirAll(tmpDir, os.ModePerm)
repoPath := RepoPath(u.Name, opts.Name) repoPath := RepoPath(u.Name, opts.Name)
wikiPath := WikiPath(u.Name, opts.Name)
if u.IsOrganization() { if u.IsOrganization() {
t, err := u.GetOwnerTeam() t, err := u.GetOwnerTeam()
@ -687,15 +733,28 @@ func MigrateRepository(u *User, opts MigrateRepoOptions) (*Repository, error) {
repo.NumWatches = 1 repo.NumWatches = 1
} }
gitTimeout := time.Duration(setting.Git.Timeout.Migrate) * time.Second
os.RemoveAll(repoPath) os.RemoveAll(repoPath)
if err = git.Clone(opts.RemoteAddr, repoPath, git.CloneRepoOptions{ if err = git.Clone(opts.RemoteAddr, repoPath, git.CloneRepoOptions{
Mirror: true, Mirror: true,
Quiet: true, Quiet: true,
Timeout: time.Duration(setting.Git.Timeout.Migrate) * time.Second, Timeout: gitTimeout,
}); err != nil { }); err != nil {
return repo, fmt.Errorf("Clone: %v", err) return repo, fmt.Errorf("Clone: %v", err)
} }
wikiRemotePath := wikiRemoteURL(opts.RemoteAddr, gitTimeout)
if wikiRemotePath != "" {
os.RemoveAll(wikiPath)
if err = git.Clone(wikiRemotePath, wikiPath, git.CloneRepoOptions{
Mirror: true,
Quiet: true,
Timeout: gitTimeout,
}); err != nil {
log.Info("Clone wiki failed: %v", err)
}
}
// Check if repository is empty. // Check if repository is empty.
_, stderr, err := com.ExecCmdDir(repoPath, "git", "log", "-1") _, stderr, err := com.ExecCmdDir(repoPath, "git", "log", "-1")
if err != nil { if err != nil {
@ -735,14 +794,22 @@ func MigrateRepository(u *User, opts MigrateRepoOptions) (*Repository, error) {
return repo, UpdateRepository(repo, false) return repo, UpdateRepository(repo, false)
} }
return CleanUpMigrateInfo(repo, repoPath) return CleanUpMigrateInfo(repo)
} }
// Finish migrating repository with things that don't need to be done for mirrors. // Finish migrating repository with things that don't need to be done for mirrors.
func CleanUpMigrateInfo(repo *Repository, repoPath string) (*Repository, error) { func CleanUpMigrateInfo(repo *Repository) (*Repository, error) {
repoPath := repo.RepoPath()
hasWiki := repo.HasWiki()
if err := createUpdateHook(repoPath); err != nil { if err := createUpdateHook(repoPath); err != nil {
return repo, fmt.Errorf("createUpdateHook: %v", err) return repo, fmt.Errorf("createUpdateHook: %v", err)
} }
if hasWiki {
if err := createUpdateHook(repoPath); err != nil {
return repo, fmt.Errorf("createUpdateHook: %v", err)
}
}
// Clean up mirror info which prevents "push --all". // Clean up mirror info which prevents "push --all".
// This also removes possible user credentials. // This also removes possible user credentials.
@ -755,6 +822,17 @@ func CleanUpMigrateInfo(repo *Repository, repoPath string) (*Repository, error)
if err = cfg.SaveToIndent(configPath, "\t"); err != nil { if err = cfg.SaveToIndent(configPath, "\t"); err != nil {
return repo, fmt.Errorf("save config file: %v", err) return repo, fmt.Errorf("save config file: %v", err)
} }
if hasWiki {
wikiConfigPath := filepath.Join(repo.WikiPath(), "config")
cfg, err = ini.Load(wikiConfigPath)
if err != nil {
return repo, fmt.Errorf("open wiki config file: %v", err)
}
cfg.DeleteSection("remote \"origin\"")
if err = cfg.SaveToIndent(wikiConfigPath, "\t"); err != nil {
return repo, fmt.Errorf("save wiki config file: %v", err)
}
}
return repo, UpdateRepository(repo, false) return repo, UpdateRepository(repo, false)
} }
@ -1695,6 +1773,8 @@ func MirrorUpdate() {
} }
repoPath := m.Repo.RepoPath() repoPath := m.Repo.RepoPath()
wikiPath := m.Repo.WikiPath()
timeout := time.Duration(setting.Git.Timeout.Mirror) * time.Second
gitArgs := []string{"remote", "update"} gitArgs := []string{"remote", "update"}
if m.EnablePrune { if m.EnablePrune {
@ -1702,8 +1782,7 @@ func MirrorUpdate() {
} }
if _, stderr, err := process.ExecDir( if _, stderr, err := process.ExecDir(
time.Duration(setting.Git.Timeout.Mirror)*time.Second, timeout, repoPath, fmt.Sprintf("MirrorUpdate: %s", repoPath),
repoPath, fmt.Sprintf("MirrorUpdate: %s", repoPath),
"git", gitArgs...); err != nil { "git", gitArgs...); err != nil {
desc := fmt.Sprintf("Fail to update mirror repository(%s): %s", repoPath, stderr) desc := fmt.Sprintf("Fail to update mirror repository(%s): %s", repoPath, stderr)
log.Error(4, desc) log.Error(4, desc)
@ -1712,6 +1791,18 @@ func MirrorUpdate() {
} }
return nil return nil
} }
if m.Repo.HasWiki() {
if _, stderr, err := process.ExecDir(
timeout, wikiPath, fmt.Sprintf("MirrorUpdate: %s", wikiPath),
"git", "remote", "update", "--prune"); err != nil {
desc := fmt.Sprintf("Fail to update mirror wiki repository(%s): %s", wikiPath, stderr)
log.Error(4, desc)
if err = CreateRepositoryNotice(desc); err != nil {
log.Error(4, "CreateRepositoryNotice: %v", err)
}
return nil
}
}
m.NextUpdate = time.Now().Add(time.Duration(m.Interval) * time.Hour) m.NextUpdate = time.Now().Add(time.Duration(m.Interval) * time.Hour)
mirrors = append(mirrors, m) mirrors = append(mirrors, m)

View File

@ -223,6 +223,7 @@ func RepoAssignment(args ...bool) macaron.Handler {
ctx.Data["IsRepositoryOwner"] = ctx.Repo.IsOwner() ctx.Data["IsRepositoryOwner"] = ctx.Repo.IsOwner()
ctx.Data["IsRepositoryAdmin"] = ctx.Repo.IsAdmin() ctx.Data["IsRepositoryAdmin"] = ctx.Repo.IsAdmin()
ctx.Data["IsRepositoryWriter"] = ctx.Repo.IsWriter() ctx.Data["IsRepositoryWriter"] = ctx.Repo.IsWriter()
ctx.Data["IsRepositoryMirror"] = repo.IsMirror
ctx.Data["DisableSSH"] = setting.SSH.Disabled ctx.Data["DisableSSH"] = setting.SSH.Disabled
ctx.Data["CloneLink"] = repo.CloneLink() ctx.Data["CloneLink"] = repo.CloneLink()

View File

@ -174,7 +174,7 @@ func SettingsPost(ctx *context.Context, form auth.RepoSettingForm) {
} }
repo.IsMirror = false repo.IsMirror = false
if _, err := models.CleanUpMigrateInfo(repo, models.RepoPath(ctx.Repo.Owner.Name, repo.Name)); err != nil { if _, err := models.CleanUpMigrateInfo(repo); err != nil {
ctx.Handle(500, "CleanUpMigrateInfo", err) ctx.Handle(500, "CleanUpMigrateInfo", err)
return return
} else if err = models.DeleteMirrorByRepoID(ctx.Repo.Repository.ID); err != nil { } else if err = models.DeleteMirrorByRepoID(ctx.Repo.Repository.ID); err != nil {

View File

@ -6,7 +6,7 @@
<span class="mega-octicon octicon-book"></span> <span class="mega-octicon octicon-book"></span>
<h2>{{.i18n.Tr "repo.wiki.welcome"}}</h2> <h2>{{.i18n.Tr "repo.wiki.welcome"}}</h2>
<p>{{.i18n.Tr "repo.wiki.welcome_desc"}}</p> <p>{{.i18n.Tr "repo.wiki.welcome_desc"}}</p>
{{if .IsRepositoryWriter}} {{if and .IsRepositoryWriter (not .IsRepositoryMirror)}}
<a class="ui green button" href="{{.RepoLink}}/wiki/_new">{{.i18n.Tr "repo.wiki.create_first_page"}}</a> <a class="ui green button" href="{{.RepoLink}}/wiki/_new">{{.i18n.Tr "repo.wiki.create_first_page"}}</a>
{{end}} {{end}}
</div> </div>

View File

@ -46,7 +46,7 @@
</div> </div>
<div class="ui dividing header"> <div class="ui dividing header">
{{.title}} {{.title}}
{{if .IsRepositoryWriter}} {{if and .IsRepositoryWriter (not .IsRepositoryMirror)}}
<div class="ui right"> <div class="ui right">
<a class="ui small button" href="{{.RepoLink}}/wiki/{{.PageURL}}/_edit">{{.i18n.Tr "repo.wiki.edit_page_button"}}</a> <a class="ui small button" href="{{.RepoLink}}/wiki/{{.PageURL}}/_edit">{{.i18n.Tr "repo.wiki.edit_page_button"}}</a>
<a class="ui green small button" href="{{.RepoLink}}/wiki/_new">{{.i18n.Tr "repo.wiki.new_page_button"}}</a> <a class="ui green small button" href="{{.RepoLink}}/wiki/_new">{{.i18n.Tr "repo.wiki.new_page_button"}}</a>