gamzinav/forgejo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

improve github downloader on migrations (#7049)

Browse Source 
* improve github downloader on migrations

* fix tests

* fix  uppercase function parameters
This commit is contained in:
Lunny Xiao 2019-05-31 04:26:57 +08:00 committed by techknowlogick
parent 43cf2f3b55
commit 7d12ec2abd
5 changed files with 141 additions and 151 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
modules/migrations/base/downloader.go
View File

@ -11,9 +11,9 @@ type Downloader interface {
GetMilestones() ([]*Milestone, error)
GetReleases() ([]*Release, error)
GetLabels() ([]*Label, error)
GetIssues(start, limit int) ([]*Issue, error)
GetIssues(page, perPage int) ([]*Issue, bool, error)
GetComments(issueNumber int64) ([]*Comment, error)
GetPullRequests(start, limit int) ([]*PullRequest, error)
GetPullRequests(page, perPage int) ([]*PullRequest, error)
}
// DownloaderFactory defines an interface to match a downloader implementation and create a downloader

8
modules/migrations/git.go
View File

@ -53,9 +53,9 @@ func (g *PlainGitDownloader) GetReleases() ([]*base.Release, error) {
return nil, ErrNotSupported
}
// GetIssues returns issues according start and limit
func (g *PlainGitDownloader) GetIssues(start, limit int) ([]*base.Issue, error) {
return nil, ErrNotSupported
// GetIssues returns issues according page and perPage
func (g *PlainGitDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, error) {
return nil, false, ErrNotSupported
}
// GetComments returns comments according issueNumber
@ -63,7 +63,7 @@ func (g *PlainGitDownloader) GetComments(issueNumber int64) ([]*base.Comment, er
return nil, ErrNotSupported
}
// GetPullRequests returns pull requests according start and limit
// GetPullRequests returns pull requests according page and perPage
func (g *PlainGitDownloader) GetPullRequests(start, limit int) ([]*base.PullRequest, error) {
return nil, ErrNotSupported
}

264
modules/migrations/github.go
View File

@ -272,71 +272,65 @@ func convertGithubReactions(reactions *github.Reactions) *base.Reactions {
}
// GetIssues returns issues according start and limit
func (g *GithubDownloaderV3) GetIssues(start, limit int) ([]*base.Issue, error) {
var perPage = 100
func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool, error) {
opt := &github.IssueListByRepoOptions{
Sort: "created",
Direction: "asc",
State: "all",
ListOptions: github.ListOptions{
PerPage: perPage,
Page: page,
},
}
var allIssues = make([]*base.Issue, 0, limit)
for {
issues, resp, err := g.client.Issues.ListByRepo(g.ctx, g.repoOwner, g.repoName, opt)
if err != nil {
return nil, fmt.Errorf("error while listing repos: %v", err)
}
for _, issue := range issues {
if issue.IsPullRequest() {
continue
}
var body string
if issue.Body != nil {
body = *issue.Body
}
var milestone string
if issue.Milestone != nil {
milestone = *issue.Milestone.Title
}
var labels = make([]*base.Label, 0, len(issue.Labels))
for _, l := range issue.Labels {
labels = append(labels, convertGithubLabel(&l))
}
var reactions *base.Reactions
if issue.Reactions != nil {
reactions = convertGithubReactions(issue.Reactions)
}
var email string
if issue.User.Email != nil {
email = *issue.User.Email
}
allIssues = append(allIssues, &base.Issue{
Title: *issue.Title,
Number: int64(*issue.Number),
PosterName: *issue.User.Login,
PosterEmail: email,
Content: body,
Milestone: milestone,
State: *issue.State,
Created: *issue.CreatedAt,
Labels: labels,
Reactions: reactions,
Closed: issue.ClosedAt,
IsLocked: *issue.Locked,
})
if len(allIssues) >= limit {
return allIssues, nil
}
}
if resp.NextPage == 0 {
break
}
opt.Page = resp.NextPage
var allIssues = make([]*base.Issue, 0, perPage)
issues, _, err := g.client.Issues.ListByRepo(g.ctx, g.repoOwner, g.repoName, opt)
if err != nil {
return nil, false, fmt.Errorf("error while listing repos: %v", err)
}
return allIssues, nil
for _, issue := range issues {
if issue.IsPullRequest() {
continue
}
var body string
if issue.Body != nil {
body = *issue.Body
}
var milestone string
if issue.Milestone != nil {
milestone = *issue.Milestone.Title
}
var labels = make([]*base.Label, 0, len(issue.Labels))
for _, l := range issue.Labels {
labels = append(labels, convertGithubLabel(&l))
}
var reactions *base.Reactions
if issue.Reactions != nil {
reactions = convertGithubReactions(issue.Reactions)
}
var email string
if issue.User.Email != nil {
email = *issue.User.Email
}
allIssues = append(allIssues, &base.Issue{
Title: *issue.Title,
Number: int64(*issue.Number),
PosterName: *issue.User.Login,
PosterEmail: email,
Content: body,
Milestone: milestone,
State: *issue.State,
Created: *issue.CreatedAt,
Labels: labels,
Reactions: reactions,
Closed: issue.ClosedAt,
IsLocked: *issue.Locked,
})
}
return allIssues, len(issues) < perPage, nil
}
// GetComments returns comments according issueNumber
@ -379,97 +373,91 @@ func (g *GithubDownloaderV3) GetComments(issueNumber int64) ([]*base.Comment, er
return allComments, nil
}
// GetPullRequests returns pull requests according start and limit
func (g *GithubDownloaderV3) GetPullRequests(start, limit int) ([]*base.PullRequest, error) {
// GetPullRequests returns pull requests according page and perPage
func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullRequest, error) {
opt := &github.PullRequestListOptions{
Sort: "created",
Direction: "asc",
State: "all",
ListOptions: github.ListOptions{
PerPage: 100,
PerPage: perPage,
Page: page,
},
}
var allPRs = make([]*base.PullRequest, 0, 100)
for {
prs, resp, err := g.client.PullRequests.List(g.ctx, g.repoOwner, g.repoName, opt)
if err != nil {
return nil, fmt.Errorf("error while listing repos: %v", err)
}
for _, pr := range prs {
var body string
if pr.Body != nil {
body = *pr.Body
}
var milestone string
if pr.Milestone != nil {
milestone = *pr.Milestone.Title
}
var labels = make([]*base.Label, 0, len(pr.Labels))
for _, l := range pr.Labels {
labels = append(labels, convertGithubLabel(l))
}
var allPRs = make([]*base.PullRequest, 0, perPage)
// FIXME: This API missing reactions, we may need another extra request to get reactions
var email string
if pr.User.Email != nil {
email = *pr.User.Email
}
var merged bool
// pr.Merged is not valid, so use MergedAt to test if it's merged
if pr.MergedAt != nil {
merged = true
}
var headRepoName string
var cloneURL string
if pr.Head.Repo != nil {
headRepoName = *pr.Head.Repo.Name
cloneURL = *pr.Head.Repo.CloneURL
}
var mergeCommitSHA string
if pr.MergeCommitSHA != nil {
mergeCommitSHA = *pr.MergeCommitSHA
}
allPRs = append(allPRs, &base.PullRequest{
Title: *pr.Title,
Number: int64(*pr.Number),
PosterName: *pr.User.Login,
PosterEmail: email,
Content: body,
Milestone: milestone,
State: *pr.State,
Created: *pr.CreatedAt,
Closed: pr.ClosedAt,
Labels: labels,
Merged: merged,
MergeCommitSHA: mergeCommitSHA,
MergedTime: pr.MergedAt,
IsLocked: pr.ActiveLockReason != nil,
Head: base.PullRequestBranch{
Ref: *pr.Head.Ref,
SHA: *pr.Head.SHA,
RepoName: headRepoName,
OwnerName: *pr.Head.User.Login,
CloneURL: cloneURL,
},
Base: base.PullRequestBranch{
Ref: *pr.Base.Ref,
SHA: *pr.Base.SHA,
RepoName: *pr.Base.Repo.Name,
OwnerName: *pr.Base.User.Login,
},
PatchURL: *pr.PatchURL,
})
if len(allPRs) >= limit {
return allPRs, nil
}
}
if resp.NextPage == 0 {
break
}
opt.Page = resp.NextPage
prs, _, err := g.client.PullRequests.List(g.ctx, g.repoOwner, g.repoName, opt)
if err != nil {
return nil, fmt.Errorf("error while listing repos: %v", err)
}
for _, pr := range prs {
var body string
if pr.Body != nil {
body = *pr.Body
}
var milestone string
if pr.Milestone != nil {
milestone = *pr.Milestone.Title
}
var labels = make([]*base.Label, 0, len(pr.Labels))
for _, l := range pr.Labels {
labels = append(labels, convertGithubLabel(l))
}
// FIXME: This API missing reactions, we may need another extra request to get reactions
var email string
if pr.User.Email != nil {
email = *pr.User.Email
}
var merged bool
// pr.Merged is not valid, so use MergedAt to test if it's merged
if pr.MergedAt != nil {
merged = true
}
var headRepoName string
var cloneURL string
if pr.Head.Repo != nil {
headRepoName = *pr.Head.Repo.Name
cloneURL = *pr.Head.Repo.CloneURL
}
var mergeCommitSHA string
if pr.MergeCommitSHA != nil {
mergeCommitSHA = *pr.MergeCommitSHA
}
allPRs = append(allPRs, &base.PullRequest{
Title: *pr.Title,
Number: int64(*pr.Number),
PosterName: *pr.User.Login,
PosterEmail: email,
Content: body,
Milestone: milestone,
State: *pr.State,
Created: *pr.CreatedAt,
Closed: pr.ClosedAt,
Labels: labels,
Merged: merged,
MergeCommitSHA: mergeCommitSHA,
MergedTime: pr.MergedAt,
IsLocked: pr.ActiveLockReason != nil,
Head: base.PullRequestBranch{
Ref: *pr.Head.Ref,
SHA: *pr.Head.SHA,
RepoName: headRepoName,
OwnerName: *pr.Head.User.Login,
CloneURL: cloneURL,
},
Base: base.PullRequestBranch{
Ref: *pr.Base.Ref,
SHA: *pr.Base.SHA,
RepoName: *pr.Base.Repo.Name,
OwnerName: *pr.Base.User.Login,
},
PatchURL: *pr.PatchURL,
})
}
return allPRs, nil
}

6
modules/migrations/github_test.go
View File

@ -166,9 +166,11 @@ func TestGitHubDownloadRepo(t *testing.T) {
}, releases[len(releases)-1:])
// downloader.GetIssues()
issues, err := downloader.GetIssues(0, 3)
issues, isEnd, err := downloader.GetIssues(1, 8)
assert.NoError(t, err)
assert.EqualValues(t, 3, len(issues))
assert.False(t, isEnd)
var (
closed1 = time.Date(2018, 10, 23, 02, 57, 43, 0, time.UTC)
)
@ -319,7 +321,7 @@ something like in the latest 15days could be enough don't you think ?
}, comments[:3])
// downloader.GetPullRequests()
prs, err := downloader.GetPullRequests(0, 3)
prs, err := downloader.GetPullRequests(1, 3)
assert.NoError(t, err)
assert.EqualValues(t, 3, len(prs))

10
modules/migrations/migrate.go
View File

@ -128,8 +128,8 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
if opts.Issues {
log.Trace("migrating issues and comments")
for {
issues, err := downloader.GetIssues(0, 100)
for i := 1; ; i++ {
issues, isEnd, err := downloader.GetIssues(i, 100)
if err != nil {
return err
}
@ -160,7 +160,7 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
}
}
if len(issues) < 100 {
if isEnd {
break
}
}
@ -168,8 +168,8 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
if opts.PullRequests {
log.Trace("migrating pull requests and comments")
for {
prs, err := downloader.GetPullRequests(0, 100)
for i := 1; ; i++ {
prs, err := downloader.GetPullRequests(i, 100)
if err != nil {
return err
}