2019-05-07 04:12:51 +03:00
// Copyright 2019 The Gitea Authors. All rights reserved.
// Copyright 2018 Jonas Franz. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package migrations
import (
"context"
"fmt"
"net/http"
"net/url"
"strings"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/migrations/base"
"github.com/google/go-github/v24/github"
"golang.org/x/oauth2"
)
var (
_ base . Downloader = & GithubDownloaderV3 { }
_ base . DownloaderFactory = & GithubDownloaderV3Factory { }
)
func init ( ) {
RegisterDownloaderFactory ( & GithubDownloaderV3Factory { } )
}
// GithubDownloaderV3Factory defines a github downloader v3 factory
type GithubDownloaderV3Factory struct {
}
// Match returns ture if the migration remote URL matched this downloader factory
func ( f * GithubDownloaderV3Factory ) Match ( opts base . MigrateOptions ) ( bool , error ) {
u , err := url . Parse ( opts . RemoteURL )
if err != nil {
return false , err
}
return u . Host == "github.com" && opts . AuthUsername != "" , nil
}
// New returns a Downloader related to this factory according MigrateOptions
func ( f * GithubDownloaderV3Factory ) New ( opts base . MigrateOptions ) ( base . Downloader , error ) {
u , err := url . Parse ( opts . RemoteURL )
if err != nil {
return nil , err
}
fields := strings . Split ( u . Path , "/" )
oldOwner := fields [ 1 ]
oldName := strings . TrimSuffix ( fields [ 2 ] , ".git" )
log . Trace ( "Create github downloader: %s/%s" , oldOwner , oldName )
return NewGithubDownloaderV3 ( opts . AuthUsername , opts . AuthPassword , oldOwner , oldName ) , nil
}
// GithubDownloaderV3 implements a Downloader interface to get repository informations
// from github via APIv3
type GithubDownloaderV3 struct {
ctx context . Context
client * github . Client
repoOwner string
repoName string
userName string
password string
}
// NewGithubDownloaderV3 creates a github Downloader via github v3 API
func NewGithubDownloaderV3 ( userName , password , repoOwner , repoName string ) * GithubDownloaderV3 {
var downloader = GithubDownloaderV3 {
userName : userName ,
password : password ,
ctx : context . Background ( ) ,
repoOwner : repoOwner ,
repoName : repoName ,
}
var client * http . Client
if userName != "" {
if password == "" {
ts := oauth2 . StaticTokenSource (
& oauth2 . Token { AccessToken : userName } ,
)
client = oauth2 . NewClient ( downloader . ctx , ts )
} else {
client = & http . Client {
Transport : & http . Transport {
Proxy : func ( req * http . Request ) ( * url . URL , error ) {
req . SetBasicAuth ( userName , password )
return nil , nil
} ,
} ,
}
}
}
downloader . client = github . NewClient ( client )
return & downloader
}
// GetRepoInfo returns a repository information
func ( g * GithubDownloaderV3 ) GetRepoInfo ( ) ( * base . Repository , error ) {
gr , _ , err := g . client . Repositories . Get ( g . ctx , g . repoOwner , g . repoName )
if err != nil {
return nil , err
}
// convert github repo to stand Repo
return & base . Repository {
Owner : g . repoOwner ,
Name : gr . GetName ( ) ,
IsPrivate : * gr . Private ,
Description : gr . GetDescription ( ) ,
2019-07-08 05:14:12 +03:00
OriginalURL : gr . GetHTMLURL ( ) ,
2019-05-07 04:12:51 +03:00
CloneURL : gr . GetCloneURL ( ) ,
} , nil
}
2019-08-14 09:16:12 +03:00
// GetTopics return github topics
func ( g * GithubDownloaderV3 ) GetTopics ( ) ( [ ] string , error ) {
r , _ , err := g . client . Repositories . Get ( g . ctx , g . repoOwner , g . repoName )
return r . Topics , err
}
2019-05-07 04:12:51 +03:00
// GetMilestones returns milestones
func ( g * GithubDownloaderV3 ) GetMilestones ( ) ( [ ] * base . Milestone , error ) {
var perPage = 100
var milestones = make ( [ ] * base . Milestone , 0 , perPage )
for i := 1 ; ; i ++ {
ms , _ , err := g . client . Issues . ListMilestones ( g . ctx , g . repoOwner , g . repoName ,
& github . MilestoneListOptions {
State : "all" ,
ListOptions : github . ListOptions {
Page : i ,
PerPage : perPage ,
} } )
if err != nil {
return nil , err
}
for _ , m := range ms {
var desc string
if m . Description != nil {
desc = * m . Description
}
var state = "open"
if m . State != nil {
state = * m . State
}
milestones = append ( milestones , & base . Milestone {
Title : * m . Title ,
Description : desc ,
Deadline : m . DueOn ,
State : state ,
Created : * m . CreatedAt ,
Updated : m . UpdatedAt ,
Closed : m . ClosedAt ,
} )
}
if len ( ms ) < perPage {
break
}
}
return milestones , nil
}
func convertGithubLabel ( label * github . Label ) * base . Label {
var desc string
if label . Description != nil {
desc = * label . Description
}
return & base . Label {
Name : * label . Name ,
Color : * label . Color ,
Description : desc ,
}
}
// GetLabels returns labels
func ( g * GithubDownloaderV3 ) GetLabels ( ) ( [ ] * base . Label , error ) {
var perPage = 100
var labels = make ( [ ] * base . Label , 0 , perPage )
for i := 1 ; ; i ++ {
ls , _ , err := g . client . Issues . ListLabels ( g . ctx , g . repoOwner , g . repoName ,
& github . ListOptions {
Page : i ,
PerPage : perPage ,
} )
if err != nil {
return nil , err
}
for _ , label := range ls {
labels = append ( labels , convertGithubLabel ( label ) )
}
if len ( ls ) < perPage {
break
}
}
return labels , nil
}
func ( g * GithubDownloaderV3 ) convertGithubRelease ( rel * github . RepositoryRelease ) * base . Release {
var (
name string
desc string
)
if rel . Body != nil {
desc = * rel . Body
}
if rel . Name != nil {
name = * rel . Name
}
r := & base . Release {
TagName : * rel . TagName ,
TargetCommitish : * rel . TargetCommitish ,
Name : name ,
Body : desc ,
Draft : * rel . Draft ,
Prerelease : * rel . Prerelease ,
Created : rel . CreatedAt . Time ,
Published : rel . PublishedAt . Time ,
}
for _ , asset := range rel . Assets {
u , _ := url . Parse ( * asset . BrowserDownloadURL )
u . User = url . UserPassword ( g . userName , g . password )
r . Assets = append ( r . Assets , base . ReleaseAsset {
URL : u . String ( ) ,
Name : * asset . Name ,
ContentType : asset . ContentType ,
Size : asset . Size ,
DownloadCount : asset . DownloadCount ,
Created : asset . CreatedAt . Time ,
Updated : asset . UpdatedAt . Time ,
} )
}
return r
}
// GetReleases returns releases
func ( g * GithubDownloaderV3 ) GetReleases ( ) ( [ ] * base . Release , error ) {
var perPage = 100
var releases = make ( [ ] * base . Release , 0 , perPage )
for i := 1 ; ; i ++ {
ls , _ , err := g . client . Repositories . ListReleases ( g . ctx , g . repoOwner , g . repoName ,
& github . ListOptions {
Page : i ,
PerPage : perPage ,
} )
if err != nil {
return nil , err
}
for _ , release := range ls {
releases = append ( releases , g . convertGithubRelease ( release ) )
}
if len ( ls ) < perPage {
break
}
}
return releases , nil
}
func convertGithubReactions ( reactions * github . Reactions ) * base . Reactions {
return & base . Reactions {
TotalCount : * reactions . TotalCount ,
PlusOne : * reactions . PlusOne ,
MinusOne : * reactions . MinusOne ,
Laugh : * reactions . Laugh ,
Confused : * reactions . Confused ,
Heart : * reactions . Heart ,
Hooray : * reactions . Hooray ,
}
}
// GetIssues returns issues according start and limit
2019-05-30 23:26:57 +03:00
func ( g * GithubDownloaderV3 ) GetIssues ( page , perPage int ) ( [ ] * base . Issue , bool , error ) {
2019-05-07 04:12:51 +03:00
opt := & github . IssueListByRepoOptions {
Sort : "created" ,
Direction : "asc" ,
State : "all" ,
ListOptions : github . ListOptions {
PerPage : perPage ,
2019-05-30 23:26:57 +03:00
Page : page ,
2019-05-07 04:12:51 +03:00
} ,
}
2019-05-30 23:26:57 +03:00
var allIssues = make ( [ ] * base . Issue , 0 , perPage )
issues , _ , err := g . client . Issues . ListByRepo ( g . ctx , g . repoOwner , g . repoName , opt )
if err != nil {
return nil , false , fmt . Errorf ( "error while listing repos: %v" , err )
}
for _ , issue := range issues {
if issue . IsPullRequest ( ) {
continue
2019-05-07 04:12:51 +03:00
}
2019-05-30 23:26:57 +03:00
var body string
if issue . Body != nil {
body = * issue . Body
2019-05-07 04:12:51 +03:00
}
2019-05-30 23:26:57 +03:00
var milestone string
if issue . Milestone != nil {
milestone = * issue . Milestone . Title
}
var labels = make ( [ ] * base . Label , 0 , len ( issue . Labels ) )
for _ , l := range issue . Labels {
labels = append ( labels , convertGithubLabel ( & l ) )
}
var reactions * base . Reactions
if issue . Reactions != nil {
reactions = convertGithubReactions ( issue . Reactions )
}
var email string
if issue . User . Email != nil {
email = * issue . User . Email
}
allIssues = append ( allIssues , & base . Issue {
Title : * issue . Title ,
Number : int64 ( * issue . Number ) ,
2019-07-08 05:14:12 +03:00
PosterID : * issue . User . ID ,
2019-05-30 23:26:57 +03:00
PosterName : * issue . User . Login ,
PosterEmail : email ,
Content : body ,
Milestone : milestone ,
State : * issue . State ,
Created : * issue . CreatedAt ,
Labels : labels ,
Reactions : reactions ,
Closed : issue . ClosedAt ,
IsLocked : * issue . Locked ,
} )
2019-05-07 04:12:51 +03:00
}
2019-05-30 23:26:57 +03:00
return allIssues , len ( issues ) < perPage , nil
2019-05-07 04:12:51 +03:00
}
// GetComments returns comments according issueNumber
func ( g * GithubDownloaderV3 ) GetComments ( issueNumber int64 ) ( [ ] * base . Comment , error ) {
var allComments = make ( [ ] * base . Comment , 0 , 100 )
opt := & github . IssueListCommentsOptions {
Sort : "created" ,
Direction : "asc" ,
ListOptions : github . ListOptions {
PerPage : 100 ,
} ,
}
for {
comments , resp , err := g . client . Issues . ListComments ( g . ctx , g . repoOwner , g . repoName , int ( issueNumber ) , opt )
if err != nil {
return nil , fmt . Errorf ( "error while listing repos: %v" , err )
}
for _ , comment := range comments {
var email string
if comment . User . Email != nil {
email = * comment . User . Email
}
var reactions * base . Reactions
if comment . Reactions != nil {
reactions = convertGithubReactions ( comment . Reactions )
}
allComments = append ( allComments , & base . Comment {
2019-06-29 16:38:22 +03:00
IssueIndex : issueNumber ,
2019-07-08 05:14:12 +03:00
PosterID : * comment . User . ID ,
2019-05-07 04:12:51 +03:00
PosterName : * comment . User . Login ,
PosterEmail : email ,
Content : * comment . Body ,
Created : * comment . CreatedAt ,
Reactions : reactions ,
} )
}
if resp . NextPage == 0 {
break
}
opt . Page = resp . NextPage
}
return allComments , nil
}
2019-05-30 23:26:57 +03:00
// GetPullRequests returns pull requests according page and perPage
func ( g * GithubDownloaderV3 ) GetPullRequests ( page , perPage int ) ( [ ] * base . PullRequest , error ) {
2019-05-07 04:12:51 +03:00
opt := & github . PullRequestListOptions {
Sort : "created" ,
Direction : "asc" ,
State : "all" ,
ListOptions : github . ListOptions {
2019-05-30 23:26:57 +03:00
PerPage : perPage ,
Page : page ,
2019-05-07 04:12:51 +03:00
} ,
}
2019-05-30 23:26:57 +03:00
var allPRs = make ( [ ] * base . PullRequest , 0 , perPage )
2019-05-07 04:12:51 +03:00
2019-05-30 23:26:57 +03:00
prs , _ , err := g . client . PullRequests . List ( g . ctx , g . repoOwner , g . repoName , opt )
if err != nil {
return nil , fmt . Errorf ( "error while listing repos: %v" , err )
}
for _ , pr := range prs {
var body string
if pr . Body != nil {
body = * pr . Body
}
var milestone string
if pr . Milestone != nil {
milestone = * pr . Milestone . Title
}
var labels = make ( [ ] * base . Label , 0 , len ( pr . Labels ) )
for _ , l := range pr . Labels {
labels = append ( labels , convertGithubLabel ( l ) )
}
2019-05-07 04:12:51 +03:00
2019-05-30 23:26:57 +03:00
// FIXME: This API missing reactions, we may need another extra request to get reactions
2019-05-07 04:12:51 +03:00
2019-05-30 23:26:57 +03:00
var email string
if pr . User . Email != nil {
email = * pr . User . Email
}
var merged bool
// pr.Merged is not valid, so use MergedAt to test if it's merged
if pr . MergedAt != nil {
merged = true
}
2019-05-07 04:12:51 +03:00
2019-06-18 19:15:39 +03:00
var (
headRepoName string
cloneURL string
headRef string
headSHA string
)
2019-05-30 23:26:57 +03:00
if pr . Head . Repo != nil {
2019-06-18 19:15:39 +03:00
if pr . Head . Repo . Name != nil {
headRepoName = * pr . Head . Repo . Name
}
if pr . Head . Repo . CloneURL != nil {
cloneURL = * pr . Head . Repo . CloneURL
}
}
if pr . Head . Ref != nil {
headRef = * pr . Head . Ref
}
if pr . Head . SHA != nil {
headSHA = * pr . Head . SHA
2019-05-07 04:12:51 +03:00
}
2019-05-30 23:26:57 +03:00
var mergeCommitSHA string
if pr . MergeCommitSHA != nil {
mergeCommitSHA = * pr . MergeCommitSHA
2019-05-07 04:12:51 +03:00
}
2019-05-30 23:26:57 +03:00
2019-06-18 19:15:39 +03:00
var headUserName string
if pr . Head . User != nil && pr . Head . User . Login != nil {
headUserName = * pr . Head . User . Login
}
2019-05-30 23:26:57 +03:00
allPRs = append ( allPRs , & base . PullRequest {
Title : * pr . Title ,
Number : int64 ( * pr . Number ) ,
PosterName : * pr . User . Login ,
2019-07-08 05:14:12 +03:00
PosterID : * pr . User . ID ,
2019-05-30 23:26:57 +03:00
PosterEmail : email ,
Content : body ,
Milestone : milestone ,
State : * pr . State ,
Created : * pr . CreatedAt ,
Closed : pr . ClosedAt ,
Labels : labels ,
Merged : merged ,
MergeCommitSHA : mergeCommitSHA ,
MergedTime : pr . MergedAt ,
IsLocked : pr . ActiveLockReason != nil ,
Head : base . PullRequestBranch {
2019-06-18 19:15:39 +03:00
Ref : headRef ,
SHA : headSHA ,
2019-05-30 23:26:57 +03:00
RepoName : headRepoName ,
2019-06-18 19:15:39 +03:00
OwnerName : headUserName ,
2019-05-30 23:26:57 +03:00
CloneURL : cloneURL ,
} ,
Base : base . PullRequestBranch {
Ref : * pr . Base . Ref ,
SHA : * pr . Base . SHA ,
RepoName : * pr . Base . Repo . Name ,
OwnerName : * pr . Base . User . Login ,
} ,
PatchURL : * pr . PatchURL ,
} )
2019-05-07 04:12:51 +03:00
}
2019-05-30 23:26:57 +03:00
2019-05-07 04:12:51 +03:00
return allPRs , nil
}