core
/
gitea
6
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Sync releases table with tags on push and for mirrors (#2459) 

* Sync releases table with tags on push and for mirrors

* Code style fixes

* Fix api to return only releases

* Optimize release creation and update
Minimize posibility of race conditions

* Fix release lower tag name updating

* handle tag reference update by addionally comparing commit id
This commit is contained in:
Lauris BH 2017-09-20 08:26:49 +03:00 committed by Lunny Xiao
parent 8b6236d67b
commit 7a0297819d
10 changed files with 370 additions and 122 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
models/migrations/migrations.go
View File

@ -132,6 +132,8 @@ var migrations = []Migration{
NewMigration("migrate protected branch struct", migrateProtectedBranchStruct),
// v41 -> v42
NewMigration("add default value to user prohibit_login", addDefaultValueToUserProhibitLogin),
// v42 -> v43
NewMigration("add tags to releases and sync existing repositories", releaseAddColumnIsTagAndSyncTags),
}
// Migrate database to current version

57
models/migrations/v42.go Normal file
View File

@ -0,0 +1,57 @@
// Copyright 2017 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package migrations
import (
"fmt"
"code.gitea.io/git"
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/log"
"github.com/go-xorm/xorm"
)
// ReleaseV39 describes the added field for Release
type ReleaseV39 struct {
IsTag bool `xorm:"NOT NULL DEFAULT false"`
}
// TableName will be invoked by XORM to customrize the table name
func (*ReleaseV39) TableName() string {
return "release"
}
func releaseAddColumnIsTagAndSyncTags(x *xorm.Engine) error {
if err := x.Sync2(new(ReleaseV39)); err != nil {
return fmt.Errorf("Sync2: %v", err)
}
// For the sake of SQLite3, we can't use x.Iterate here.
offset := 0
pageSize := 20
for {
repos := make([]*models.Repository, 0, pageSize)
if err := x.Table("repository").Asc("id").Limit(pageSize, offset).Find(&repos); err != nil {
return fmt.Errorf("select repos [offset: %d]: %v", offset, err)
}
for _, repo := range repos {
gitRepo, err := git.OpenRepository(repo.RepoPath())
if err != nil {
log.Warn("OpenRepository: %v", err)
continue
}
if err = models.SyncReleasesWithTags(repo, gitRepo); err != nil {
log.Warn("SyncReleasesWithTags: %v", err)
}
}
if len(repos) < pageSize {
break
}
offset += pageSize
}
return nil
}

69
models/release.go
View File

@ -34,7 +34,8 @@ type Release struct {
NumCommitsBehind int64 `xorm:"-"`
Note string `xorm:"TEXT"`
IsDraft bool `xorm:"NOT NULL DEFAULT false"`
IsPrerelease bool
IsPrerelease bool `xorm:"NOT NULL DEFAULT false"`
IsTag bool `xorm:"NOT NULL DEFAULT false"`
Attachments []*Attachment `xorm:"-"`
@ -139,19 +140,20 @@ func createTag(gitRepo *git.Repository, rel *Release) error {
}
return err
}
} else {
rel.LowerTagName = strings.ToLower(rel.TagName)
}
commit, err := gitRepo.GetTagCommit(rel.TagName)
if err != nil {
return fmt.Errorf("GetTagCommit: %v", err)
}
rel.Sha1 = commit.ID.String()
rel.CreatedUnix = commit.Author.When.Unix()
rel.NumCommits, err = commit.CommitsCount()
if err != nil {
return fmt.Errorf("CommitsCount: %v", err)
}
}
}
return nil
}
@ -236,6 +238,7 @@ func GetReleaseByID(id int64) (*Release, error) {
// FindReleasesOptions describes the conditions to Find releases
type FindReleasesOptions struct {
IncludeDrafts bool
IncludeTags bool
TagNames []string
}
@ -246,6 +249,9 @@ func (opts *FindReleasesOptions) toConds(repoID int64) builder.Cond {
if !opts.IncludeDrafts {
cond = cond.And(builder.Eq{"is_draft": false})
}
if !opts.IncludeTags {
cond = cond.And(builder.Eq{"is_tag": false})
}
if len(opts.TagNames) > 0 {
cond = cond.And(builder.In("tag_name", opts.TagNames))
}
@ -361,6 +367,8 @@ func UpdateRelease(gitRepo *git.Repository, rel *Release, attachmentUUIDs []stri
if err = createTag(gitRepo, rel); err != nil {
return err
}
rel.LowerTagName = strings.ToLower(rel.TagName)
_, err = x.Id(rel.ID).AllCols().Update(rel)
if err != nil {
return err
@ -397,11 +405,64 @@ func DeleteReleaseByID(id int64, u *User, delTag bool) error {
if err != nil && !strings.Contains(stderr, "not found") {
return fmt.Errorf("git tag -d: %v - %s", err, stderr)
}
}
if _, err = x.Id(rel.ID).Delete(new(Release)); err != nil {
return fmt.Errorf("Delete: %v", err)
}
} else {
rel.IsTag = true
rel.IsDraft = false
rel.IsPrerelease = false
rel.Title = ""
rel.Note = ""
if _, err = x.Id(rel.ID).AllCols().Update(rel); err != nil {
return fmt.Errorf("Update: %v", err)
}
}
return nil
}
// SyncReleasesWithTags synchronizes release table with repository tags
func SyncReleasesWithTags(repo *Repository, gitRepo *git.Repository) error {
existingRelTags := make(map[string]struct{})
opts := FindReleasesOptions{IncludeDrafts: true, IncludeTags: true}
for page := 1; ; page++ {
rels, err := GetReleasesByRepoID(repo.ID, opts, page, 100)
if err != nil {
return fmt.Errorf("GetReleasesByRepoID: %v", err)
}
if len(rels) == 0 {
break
}
for _, rel := range rels {
if rel.IsDraft {
continue
}
commitID, err := gitRepo.GetTagCommitID(rel.TagName)
if err != nil {
return fmt.Errorf("GetTagCommitID: %v", err)
}
if !gitRepo.IsTagExist(rel.TagName) || commitID != rel.Sha1 {
if err := pushUpdateDeleteTag(repo, gitRepo, rel.TagName); err != nil {
return fmt.Errorf("pushUpdateDeleteTag: %v", err)
}
} else {
existingRelTags[strings.ToLower(rel.TagName)] = struct{}{}
}
}
}
tags, err := gitRepo.GetTags()
if err != nil {
return fmt.Errorf("GetTags: %v", err)
}
for _, tagName := range tags {
if _, ok := existingRelTags[strings.ToLower(tagName)]; !ok {
if err := pushUpdateAddTag(repo, gitRepo, tagName); err != nil {
return fmt.Errorf("pushUpdateAddTag: %v", err)
}
}
}
return nil
}

4
models/repo.go
View File

@ -940,6 +940,10 @@ func MigrateRepository(doer, u *User, opts MigrateRepoOptions) (*Repository, err
if headBranch != nil {
repo.DefaultBranch = headBranch.Name
}
if err = SyncReleasesWithTags(repo, gitRepo); err != nil {
log.Error(4, "Failed to synchronize tags to releases for repository: %v", err)
}
}
if err = repo.UpdateSize(); err != nil {

10
models/repo_mirror.go
View File

@ -13,6 +13,7 @@ import (
"github.com/go-xorm/xorm"
"gopkg.in/ini.v1"
"code.gitea.io/git"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/process"
"code.gitea.io/gitea/modules/setting"
@ -156,6 +157,15 @@ func (m *Mirror) runSync() bool {
return false
}
gitRepo, err := git.OpenRepository(repoPath)
if err != nil {
log.Error(4, "OpenRepository: %v", err)
return false
}
if err = SyncReleasesWithTags(m.Repo, gitRepo); err != nil {
log.Error(4, "Failed to synchronize tags to releases for repository: %v", err)
}
if err := m.Repo.UpdateSize(); err != nil {
log.Error(4, "Failed to update size for mirror repository: %v", err)
}

107
models/update.go
View File

@ -81,6 +81,93 @@ func PushUpdate(branch string, opt PushUpdateOptions) error {
return nil
}
func pushUpdateDeleteTag(repo *Repository, gitRepo *git.Repository, tagName string) error {
rel, err := GetRelease(repo.ID, tagName)
if err != nil {
if IsErrReleaseNotExist(err) {
return nil
}
return fmt.Errorf("GetRelease: %v", err)
}
if rel.IsTag {
if _, err = x.Id(rel.ID).Delete(new(Release)); err != nil {
return fmt.Errorf("Delete: %v", err)
}
} else {
rel.IsDraft = true
rel.NumCommits = 0
rel.Sha1 = ""
if _, err = x.Id(rel.ID).AllCols().Update(rel); err != nil {
return fmt.Errorf("Update: %v", err)
}
}
return nil
}
func pushUpdateAddTag(repo *Repository, gitRepo *git.Repository, tagName string) error {
rel, err := GetRelease(repo.ID, tagName)
if err != nil && !IsErrReleaseNotExist(err) {
return fmt.Errorf("GetRelease: %v", err)
}
tag, err := gitRepo.GetTag(tagName)
if err != nil {
return fmt.Errorf("GetTag: %v", err)
}
commit, err := tag.Commit()
if err != nil {
return fmt.Errorf("Commit: %v", err)
}
tagCreatedUnix := commit.Author.When.Unix()
author, err := GetUserByEmail(commit.Author.Email)
if err != nil && !IsErrUserNotExist(err) {
return fmt.Errorf("GetUserByEmail: %v", err)
}
commitsCount, err := commit.CommitsCount()
if err != nil {
return fmt.Errorf("CommitsCount: %v", err)
}
if rel == nil {
rel = &Release{
RepoID: repo.ID,
Title: "",
TagName: tagName,
LowerTagName: strings.ToLower(tagName),
Target: "",
Sha1: commit.ID.String(),
NumCommits: commitsCount,
Note: "",
IsDraft: false,
IsPrerelease: false,
IsTag: true,
CreatedUnix: tagCreatedUnix,
}
if author != nil {
rel.PublisherID = author.ID
}
if _, err = x.InsertOne(rel); err != nil {
return fmt.Errorf("InsertOne: %v", err)
}
} else {
rel.Sha1 = commit.ID.String()
rel.CreatedUnix = tagCreatedUnix
rel.NumCommits = commitsCount
rel.IsDraft = false
if rel.IsTag && author != nil {
rel.PublisherID = author.ID
}
if _, err = x.Id(rel.ID).AllCols().Update(rel); err != nil {
return fmt.Errorf("Update: %v", err)
}
}
return nil
}
func pushUpdate(opts PushUpdateOptions) (repo *Repository, err error) {
isNewRef := opts.OldCommitID == git.EmptySHA
isDelRef := opts.NewCommitID == git.EmptySHA
@ -106,23 +193,31 @@ func pushUpdate(opts PushUpdateOptions) (repo *Repository, err error) {
return nil, fmt.Errorf("GetRepositoryByName: %v", err)
}
if isDelRef {
log.GitLogger.Info("Reference '%s' has been deleted from '%s/%s' by %s",
opts.RefFullName, opts.RepoUserName, opts.RepoName, opts.PusherName)
return repo, nil
}
gitRepo, err := git.OpenRepository(repoPath)
if err != nil {
return nil, fmt.Errorf("OpenRepository: %v", err)
}
if isDelRef {
// Tag has been deleted
if strings.HasPrefix(opts.RefFullName, git.TagPrefix) {
err = pushUpdateDeleteTag(repo, gitRepo, opts.RefFullName[len(git.TagPrefix):])
if err != nil {
return nil, fmt.Errorf("pushUpdateDeleteTag: %v", err)
}
}
log.GitLogger.Info("Reference '%s' has been deleted from '%s/%s' by %s",
opts.RefFullName, opts.RepoUserName, opts.RepoName, opts.PusherName)
return repo, nil
}
if err = repo.UpdateSize(); err != nil {
log.Error(4, "Failed to update size for repository: %v", err)
}
// Push tags.
if strings.HasPrefix(opts.RefFullName, git.TagPrefix) {
pushUpdateAddTag(repo, gitRepo, opts.RefFullName[len(git.TagPrefix):])
if err := CommitRepoAction(CommitRepoActionOptions{
PusherName: opts.PusherName,
RepoOwnerID: owner.ID,

1
modules/context/repo.go
View File

@ -357,6 +357,7 @@ func RepoAssignment() macaron.Handler {
count, err := models.GetReleaseCountByRepoID(ctx.Repo.Repository.ID, models.FindReleasesOptions{
IncludeDrafts: false,
IncludeTags: true,
})
if err != nil {
ctx.Handle(500, "GetReleaseCountByRepoID", err)

55
routers/api/v1/repo/release.go
View File

@ -5,8 +5,6 @@
package repo
import (
"strings"
api "code.gitea.io/sdk/gitea"
"code.gitea.io/gitea/models"
@ -36,6 +34,7 @@ func GetRelease(ctx *context.APIContext) {
func ListReleases(ctx *context.APIContext) {
releases, err := models.GetReleasesByRepoID(ctx.Repo.Repository.ID, models.FindReleasesOptions{
IncludeDrafts: ctx.Repo.AccessMode >= models.AccessModeWrite,
IncludeTags: false,
}, 1, 2147483647)
if err != nil {
ctx.Error(500, "GetReleasesByRepoID", err)
@ -62,35 +61,23 @@ func CreateRelease(ctx *context.APIContext, form api.CreateReleaseOption) {
ctx.Status(404)
return
}
tag, err := ctx.Repo.GitRepo.GetTag(form.TagName)
rel, err := models.GetRelease(ctx.Repo.Repository.ID, form.TagName)
if err != nil {
ctx.Error(500, "GetTag", err)
if !models.IsErrReleaseNotExist(err) {
ctx.Handle(500, "GetRelease", err)
return
}
commit, err := tag.Commit()
if err != nil {
ctx.Error(500, "Commit", err)
return
}
commitsCount, err := commit.CommitsCount()
if err != nil {
ctx.Error(500, "CommitsCount", err)
return
}
rel := &models.Release{
rel = &models.Release{
RepoID: ctx.Repo.Repository.ID,
PublisherID: ctx.User.ID,
Publisher: ctx.User,
TagName: form.TagName,
LowerTagName: strings.ToLower(form.TagName),
Target: form.Target,
Title: form.Title,
Sha1: commit.ID.String(),
NumCommits: commitsCount,
Note: form.Note,
IsDraft: form.IsDraft,
IsPrerelease: form.IsPrerelease,
CreatedUnix: commit.Author.When.Unix(),
IsTag: false,
}
if err := models.CreateRelease(ctx.Repo.GitRepo, rel, nil); err != nil {
if models.IsErrReleaseAlreadyExist(err) {
@ -100,6 +87,24 @@ func CreateRelease(ctx *context.APIContext, form api.CreateReleaseOption) {
}
return
}
} else {
if !rel.IsTag {
ctx.Status(409)
return
}
rel.Title = form.Title
rel.Note = form.Note
rel.IsDraft = form.IsDraft
rel.IsPrerelease = form.IsPrerelease
rel.PublisherID = ctx.User.ID
rel.IsTag = false
if err = models.UpdateRelease(ctx.Repo.GitRepo, rel, nil); err != nil {
ctx.Handle(500, "UpdateRelease", err)
return
}
}
ctx.JSON(201, rel.APIFormat())
}
@ -111,11 +116,12 @@ func EditRelease(ctx *context.APIContext, form api.EditReleaseOption) {
}
id := ctx.ParamsInt64(":id")
rel, err := models.GetReleaseByID(id)
if err != nil {
if err != nil && !models.IsErrReleaseNotExist(err) {
ctx.Error(500, "GetReleaseByID", err)
return
}
if rel.RepoID != ctx.Repo.Repository.ID {
if err != nil && models.IsErrReleaseNotExist(err) ||
rel.IsTag || rel.RepoID != ctx.Repo.Repository.ID {
ctx.Status(404)
return
}
@ -162,12 +168,13 @@ func DeleteRelease(ctx *context.APIContext) {
return
}
id := ctx.ParamsInt64(":id")
release, err := models.GetReleaseByID(id)
if err != nil {
rel, err := models.GetReleaseByID(id)
if err != nil && !models.IsErrReleaseNotExist(err) {
ctx.Error(500, "GetReleaseByID", err)
return
}
if release.RepoID != ctx.Repo.Repository.ID {
if err != nil && models.IsErrReleaseNotExist(err) ||
rel.IsTag || rel.RepoID != ctx.Repo.Repository.ID {
ctx.Status(404)
return
}

58
routers/repo/release.go
View File

@ -67,6 +67,7 @@ func Releases(ctx *context.Context) {
opts := models.FindReleasesOptions{
IncludeDrafts: ctx.Repo.IsWriter(),
IncludeTags: true,
}
releases, err := models.GetReleasesByRepoID(ctx.Repo.Repository.ID, opts, page, limit)
@ -145,24 +146,15 @@ func NewReleasePost(ctx *context.Context, form auth.NewReleaseForm) {
return
}
var tagCreatedUnix int64
tag, err := ctx.Repo.GitRepo.GetTag(form.TagName)
if err == nil {
commit, err := tag.Commit()
if err == nil {
tagCreatedUnix = commit.Author.When.Unix()
}
var attachmentUUIDs []string
if setting.AttachmentEnabled {
attachmentUUIDs = form.Files
}
commit, err := ctx.Repo.GitRepo.GetBranchCommit(form.Target)
rel, err := models.GetRelease(ctx.Repo.Repository.ID, form.TagName)
if err != nil {
ctx.Handle(500, "GetBranchCommit", err)
return
}
commitsCount, err := commit.CommitsCount()
if err != nil {
ctx.Handle(500, "CommitsCount", err)
if !models.IsErrReleaseNotExist(err) {
ctx.Handle(500, "GetRelease", err)
return
}
@ -172,17 +164,10 @@ func NewReleasePost(ctx *context.Context, form auth.NewReleaseForm) {
Title: form.Title,
TagName: form.TagName,
Target: form.Target,
Sha1: commit.ID.String(),
NumCommits: commitsCount,
Note: form.Content,
IsDraft: len(form.Draft) > 0,
IsPrerelease: form.Prerelease,
CreatedUnix: tagCreatedUnix,
}
var attachmentUUIDs []string
if setting.AttachmentEnabled {
attachmentUUIDs = form.Files
IsTag: false,
}
if err = models.CreateRelease(ctx.Repo.GitRepo, rel, attachmentUUIDs); err != nil {
@ -197,6 +182,26 @@ func NewReleasePost(ctx *context.Context, form auth.NewReleaseForm) {
}
return
}
} else {
if !rel.IsTag {
ctx.Data["Err_TagName"] = true
ctx.RenderWithErr(ctx.Tr("repo.release.tag_name_already_exist"), tplReleaseNew, &form)
return
}
rel.Title = form.Title
rel.Note = form.Content
rel.IsDraft = len(form.Draft) > 0
rel.IsPrerelease = form.Prerelease
rel.PublisherID = ctx.User.ID
rel.IsTag = false
if err = models.UpdateRelease(ctx.Repo.GitRepo, rel, attachmentUUIDs); err != nil {
ctx.Data["Err_TagName"] = true
ctx.Handle(500, "UpdateRelease", err)
return
}
}
log.Trace("Release created: %s/%s:%s", ctx.User.LowerName, ctx.Repo.Repository.Name, form.TagName)
ctx.Redirect(ctx.Repo.RepoLink + "/releases")
@ -246,6 +251,10 @@ func EditReleasePost(ctx *context.Context, form auth.EditReleaseForm) {
}
return
}
if rel.IsTag {
ctx.Handle(404, "GetRelease", err)
return
}
ctx.Data["tag_name"] = rel.TagName
ctx.Data["tag_target"] = rel.Target
ctx.Data["title"] = rel.Title
@ -275,8 +284,7 @@ func EditReleasePost(ctx *context.Context, form auth.EditReleaseForm) {
// DeleteRelease delete a release
func DeleteRelease(ctx *context.Context) {
delTag := ctx.QueryBool("delTag")
if err := models.DeleteReleaseByID(ctx.QueryInt64("id"), ctx.User, delTag); err != nil {
if err := models.DeleteReleaseByID(ctx.QueryInt64("id"), ctx.User, true); err != nil {
ctx.Flash.Error("DeleteReleaseByID: " + err.Error())
} else {
ctx.Flash.Success(ctx.Tr("repo.release.deletion_success"))

25
templates/repo/release/list.tmpl
View File

@ -17,7 +17,9 @@
{{range .Releases}}
<li class="ui grid">
<div class="ui four wide column meta">
{{if .PublisherID}}
{{if .IsTag}}
{{if .Created}}<span class="time">{{TimeSince .Created $.Lang}}</span>{{end}}
{{else}}
{{if .IsDraft}}
<span class="ui yellow label">{{$.i18n.Tr "repo.release.draft"}}</span>
{{else if .IsPrerelease}}
@ -28,13 +30,22 @@
<span class="tag text blue">
<a href="{{$.RepoLink}}/src/{{.TagName}}" rel="nofollow"><i class="tag icon"></i> {{.TagName}}</a>
</span>
{{end}}
<span class="commit">
<a href="{{$.RepoLink}}/src/{{.Sha1}}" rel="nofollow"><i class="code icon"></i> {{ShortSha .Sha1}}</a>
</span>
{{end}}
</div>
<div class="ui twelve wide column detail">
{{if .PublisherID}}
{{if .IsTag}}
<h4>
<a href="{{$.RepoLink}}/src/{{.TagName}}" rel="nofollow"><i class="tag icon"></i> {{.TagName}}</a>
</h4>
<div class="download">
<a href="{{$.RepoLink}}/src/{{.Sha1}}" rel="nofollow"><i class="code icon"></i> {{ShortSha .Sha1}}</a>
<a href="{{$.RepoLink}}/archive/{{.TagName}}.zip" rel="nofollow"><i class="octicon octicon-file-zip"></i> ZIP</a>
<a href="{{$.RepoLink}}/archive/{{.TagName}}.tar.gz"><i class="octicon octicon-file-zip"></i> TAR.GZ</a>
</div>
{{else}}
<h3>
<a href="{{$.RepoLink}}/src/{{.TagName}}">{{.Title}}</a>
{{if $.IsRepositoryWriter}}<small>(<a href="{{$.RepoLink}}/releases/edit/{{.TagName}}" rel="nofollow">{{$.i18n.Tr "repo.release.edit"}}</a>)</small>{{end}}
@ -70,14 +81,6 @@
{{end}}
</ul>
</div>
{{else}}
<h4>
<a href="{{$.RepoLink}}/src/{{.TagName}}" rel="nofollow"><i class="tag icon"></i> {{.TagName}}</a>
</h4>
<div class="download">
<a href="{{$.RepoLink}}/archive/{{.TagName}}.zip" rel="nofollow"><i class="octicon octicon-file-zip"></i> ZIP</a>
<a href="{{$.RepoLink}}/archive/{{.TagName}}.tar.gz"><i class="octicon octicon-file-zip"></i> TAR.GZ</a>
</div>
{{end}}
<span class="dot">&nbsp;</span>
</div>