2020-04-19 18:44:11 +03:00
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package migrations
import (
"context"
"errors"
"fmt"
2020-08-28 04:36:37 +03:00
"io"
"net/http"
2020-04-19 18:44:11 +03:00
"net/url"
"strings"
"time"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/migrations/base"
"code.gitea.io/gitea/modules/structs"
"github.com/xanzy/go-gitlab"
)
var (
_ base . Downloader = & GitlabDownloader { }
_ base . DownloaderFactory = & GitlabDownloaderFactory { }
)
func init ( ) {
RegisterDownloaderFactory ( & GitlabDownloaderFactory { } )
}
// GitlabDownloaderFactory defines a gitlab downloader factory
type GitlabDownloaderFactory struct {
}
// New returns a Downloader related to this factory according MigrateOptions
2020-09-02 20:49:25 +03:00
func ( f * GitlabDownloaderFactory ) New ( ctx context . Context , opts base . MigrateOptions ) ( base . Downloader , error ) {
2020-04-19 18:44:11 +03:00
u , err := url . Parse ( opts . CloneAddr )
if err != nil {
return nil , err
}
baseURL := u . Scheme + "://" + u . Host
repoNameSpace := strings . TrimPrefix ( u . Path , "/" )
2020-08-28 04:36:37 +03:00
repoNameSpace = strings . TrimSuffix ( repoNameSpace , ".git" )
2020-04-19 18:44:11 +03:00
log . Trace ( "Create gitlab downloader. BaseURL: %s RepoName: %s" , baseURL , repoNameSpace )
2020-09-10 11:27:49 +03:00
return NewGitlabDownloader ( ctx , baseURL , repoNameSpace , opts . AuthUsername , opts . AuthPassword , opts . AuthToken )
2020-04-19 18:44:11 +03:00
}
// GitServiceType returns the type of git service
func ( f * GitlabDownloaderFactory ) GitServiceType ( ) structs . GitServiceType {
return structs . GitlabService
}
// GitlabDownloader implements a Downloader interface to get repository informations
// from gitlab via go-gitlab
// - issueCount is incremented in GetIssues() to ensure PR and Issue numbers do not overlap,
// because Gitlab has individual Issue and Pull Request numbers.
// - issueSeen, working alongside issueCount, is checked in GetComments() to see whether we
// need to fetch the Issue or PR comments, as Gitlab stores them separately.
type GitlabDownloader struct {
ctx context . Context
client * gitlab . Client
repoID int
repoName string
issueCount int64
fetchPRcomments bool
}
// NewGitlabDownloader creates a gitlab Downloader via gitlab API
// Use either a username/password, personal token entered into the username field, or anonymous/public access
// Note: Public access only allows very basic access
2020-09-10 11:27:49 +03:00
func NewGitlabDownloader ( ctx context . Context , baseURL , repoPath , username , password , token string ) ( * GitlabDownloader , error ) {
2020-04-19 18:44:11 +03:00
var gitlabClient * gitlab . Client
var err error
2020-08-28 04:36:37 +03:00
if token != "" {
gitlabClient , err = gitlab . NewClient ( token , gitlab . WithBaseURL ( baseURL ) )
} else {
gitlabClient , err = gitlab . NewBasicAuthClient ( username , password , gitlab . WithBaseURL ( baseURL ) )
2020-04-19 18:44:11 +03:00
}
if err != nil {
log . Trace ( "Error logging into gitlab: %v" , err )
2020-09-10 11:27:49 +03:00
return nil , err
2020-04-19 18:44:11 +03:00
}
// Grab and store project/repo ID here, due to issues using the URL escaped path
2020-09-02 20:49:25 +03:00
gr , _ , err := gitlabClient . Projects . GetProject ( repoPath , nil , nil , gitlab . WithContext ( ctx ) )
2020-04-19 18:44:11 +03:00
if err != nil {
log . Trace ( "Error retrieving project: %v" , err )
2020-09-10 11:27:49 +03:00
return nil , err
2020-04-19 18:44:11 +03:00
}
if gr == nil {
log . Trace ( "Error getting project, project is nil" )
2020-09-10 11:27:49 +03:00
return nil , errors . New ( "Error getting project, project is nil" )
2020-04-19 18:44:11 +03:00
}
return & GitlabDownloader {
2020-09-02 20:49:25 +03:00
ctx : ctx ,
2020-04-19 18:44:11 +03:00
client : gitlabClient ,
repoID : gr . ID ,
repoName : gr . Name ,
2020-09-10 11:27:49 +03:00
} , nil
2020-04-19 18:44:11 +03:00
}
// SetContext set context
func ( g * GitlabDownloader ) SetContext ( ctx context . Context ) {
g . ctx = ctx
}
// GetRepoInfo returns a repository information
func ( g * GitlabDownloader ) GetRepoInfo ( ) ( * base . Repository , error ) {
2020-09-02 20:49:25 +03:00
gr , _ , err := g . client . Projects . GetProject ( g . repoID , nil , nil , gitlab . WithContext ( g . ctx ) )
2020-04-19 18:44:11 +03:00
if err != nil {
return nil , err
}
var private bool
switch gr . Visibility {
case gitlab . InternalVisibility :
private = true
case gitlab . PrivateVisibility :
private = true
}
var owner string
if gr . Owner == nil {
log . Trace ( "gr.Owner is nil, trying to get owner from Namespace" )
if gr . Namespace != nil && gr . Namespace . Kind == "user" {
owner = gr . Namespace . Path
}
} else {
owner = gr . Owner . Username
}
// convert gitlab repo to stand Repo
return & base . Repository {
Owner : owner ,
Name : gr . Name ,
IsPrivate : private ,
Description : gr . Description ,
OriginalURL : gr . WebURL ,
CloneURL : gr . HTTPURLToRepo ,
} , nil
}
// GetTopics return gitlab topics
func ( g * GitlabDownloader ) GetTopics ( ) ( [ ] string , error ) {
2020-09-02 20:49:25 +03:00
gr , _ , err := g . client . Projects . GetProject ( g . repoID , nil , nil , gitlab . WithContext ( g . ctx ) )
2020-04-19 18:44:11 +03:00
if err != nil {
return nil , err
}
return gr . TagList , err
}
// GetMilestones returns milestones
func ( g * GitlabDownloader ) GetMilestones ( ) ( [ ] * base . Milestone , error ) {
var perPage = 100
var state = "all"
var milestones = make ( [ ] * base . Milestone , 0 , perPage )
for i := 1 ; ; i ++ {
ms , _ , err := g . client . Milestones . ListMilestones ( g . repoID , & gitlab . ListMilestonesOptions {
State : & state ,
ListOptions : gitlab . ListOptions {
Page : i ,
PerPage : perPage ,
2020-09-02 20:49:25 +03:00
} } , nil , gitlab . WithContext ( g . ctx ) )
2020-04-19 18:44:11 +03:00
if err != nil {
return nil , err
}
for _ , m := range ms {
var desc string
if m . Description != "" {
desc = m . Description
}
var state = "open"
var closedAt * time . Time
if m . State != "" {
state = m . State
if state == "closed" {
closedAt = m . UpdatedAt
}
}
var deadline * time . Time
if m . DueDate != nil {
deadlineParsed , err := time . Parse ( "2006-01-02" , m . DueDate . String ( ) )
if err != nil {
log . Trace ( "Error parsing Milestone DueDate time" )
deadline = nil
} else {
deadline = & deadlineParsed
}
}
milestones = append ( milestones , & base . Milestone {
Title : m . Title ,
Description : desc ,
Deadline : deadline ,
State : state ,
Created : * m . CreatedAt ,
Updated : m . UpdatedAt ,
Closed : closedAt ,
} )
}
if len ( ms ) < perPage {
break
}
}
return milestones , nil
}
// GetLabels returns labels
func ( g * GitlabDownloader ) GetLabels ( ) ( [ ] * base . Label , error ) {
var perPage = 100
var labels = make ( [ ] * base . Label , 0 , perPage )
for i := 1 ; ; i ++ {
2020-09-06 18:37:53 +03:00
ls , _ , err := g . client . Labels . ListLabels ( g . repoID , & gitlab . ListLabelsOptions { ListOptions : gitlab . ListOptions {
2020-04-19 18:44:11 +03:00
Page : i ,
PerPage : perPage ,
2020-09-06 18:37:53 +03:00
} } , nil , gitlab . WithContext ( g . ctx ) )
2020-04-19 18:44:11 +03:00
if err != nil {
return nil , err
}
for _ , label := range ls {
baseLabel := & base . Label {
Name : label . Name ,
Color : strings . TrimLeft ( label . Color , "#)" ) ,
Description : label . Description ,
}
labels = append ( labels , baseLabel )
}
if len ( ls ) < perPage {
break
}
}
return labels , nil
}
func ( g * GitlabDownloader ) convertGitlabRelease ( rel * gitlab . Release ) * base . Release {
2020-08-28 04:36:37 +03:00
var zero int
2020-04-19 18:44:11 +03:00
r := & base . Release {
TagName : rel . TagName ,
TargetCommitish : rel . Commit . ID ,
Name : rel . Name ,
Body : rel . Description ,
Created : * rel . CreatedAt ,
PublisherID : int64 ( rel . Author . ID ) ,
PublisherName : rel . Author . Username ,
}
for k , asset := range rel . Assets . Links {
r . Assets = append ( r . Assets , base . ReleaseAsset {
2020-08-28 04:36:37 +03:00
ID : int64 ( asset . ID ) ,
Name : asset . Name ,
ContentType : & rel . Assets . Sources [ k ] . Format ,
Size : & zero ,
DownloadCount : & zero ,
2020-04-19 18:44:11 +03:00
} )
}
return r
}
// GetReleases returns releases
func ( g * GitlabDownloader ) GetReleases ( ) ( [ ] * base . Release , error ) {
var perPage = 100
var releases = make ( [ ] * base . Release , 0 , perPage )
for i := 1 ; ; i ++ {
ls , _ , err := g . client . Releases . ListReleases ( g . repoID , & gitlab . ListReleasesOptions {
Page : i ,
PerPage : perPage ,
2020-09-02 20:49:25 +03:00
} , nil , gitlab . WithContext ( g . ctx ) )
2020-04-19 18:44:11 +03:00
if err != nil {
return nil , err
}
for _ , release := range ls {
releases = append ( releases , g . convertGitlabRelease ( release ) )
}
if len ( ls ) < perPage {
break
}
}
return releases , nil
}
2020-08-28 04:36:37 +03:00
// GetAsset returns an asset
func ( g * GitlabDownloader ) GetAsset ( tag string , id int64 ) ( io . ReadCloser , error ) {
2020-09-02 20:49:25 +03:00
link , _ , err := g . client . ReleaseLinks . GetReleaseLink ( g . repoID , tag , int ( id ) , gitlab . WithContext ( g . ctx ) )
2020-08-28 04:36:37 +03:00
if err != nil {
return nil , err
}
2020-09-02 20:49:25 +03:00
req , err := http . NewRequest ( "GET" , link . URL , nil )
if err != nil {
return nil , err
}
req = req . WithContext ( g . ctx )
resp , err := http . DefaultClient . Do ( req )
2020-08-28 04:36:37 +03:00
if err != nil {
return nil , err
}
// resp.Body is closed by the uploader
return resp . Body , nil
}
2020-04-19 18:44:11 +03:00
// GetIssues returns issues according start and limit
// Note: issue label description and colors are not supported by the go-gitlab library at this time
func ( g * GitlabDownloader ) GetIssues ( page , perPage int ) ( [ ] * base . Issue , bool , error ) {
state := "all"
sort := "asc"
opt := & gitlab . ListProjectIssuesOptions {
State : & state ,
Sort : & sort ,
ListOptions : gitlab . ListOptions {
PerPage : perPage ,
Page : page ,
} ,
}
var allIssues = make ( [ ] * base . Issue , 0 , perPage )
2020-09-02 20:49:25 +03:00
issues , _ , err := g . client . Issues . ListProjectIssues ( g . repoID , opt , nil , gitlab . WithContext ( g . ctx ) )
2020-04-19 18:44:11 +03:00
if err != nil {
return nil , false , fmt . Errorf ( "error while listing issues: %v" , err )
}
for _ , issue := range issues {
var labels = make ( [ ] * base . Label , 0 , len ( issue . Labels ) )
for _ , l := range issue . Labels {
labels = append ( labels , & base . Label {
Name : l ,
} )
}
var milestone string
if issue . Milestone != nil {
milestone = issue . Milestone . Title
}
2020-09-03 10:35:17 +03:00
var reactions [ ] * base . Reaction
var awardPage = 1
for {
awards , _ , err := g . client . AwardEmoji . ListIssueAwardEmoji ( g . repoID , issue . IID , & gitlab . ListAwardEmojiOptions { Page : awardPage , PerPage : perPage } , gitlab . WithContext ( g . ctx ) )
if err != nil {
return nil , false , fmt . Errorf ( "error while listing issue awards: %v" , err )
}
if len ( awards ) < perPage {
break
}
for i := range awards {
reactions = append ( reactions , g . awardToReaction ( awards [ i ] ) )
}
awardPage ++
}
2020-04-19 18:44:11 +03:00
allIssues = append ( allIssues , & base . Issue {
Title : issue . Title ,
Number : int64 ( issue . IID ) ,
PosterID : int64 ( issue . Author . ID ) ,
PosterName : issue . Author . Username ,
Content : issue . Description ,
Milestone : milestone ,
State : issue . State ,
Created : * issue . CreatedAt ,
Labels : labels ,
2020-09-03 10:35:17 +03:00
Reactions : reactions ,
2020-04-19 18:44:11 +03:00
Closed : issue . ClosedAt ,
IsLocked : issue . DiscussionLocked ,
Updated : * issue . UpdatedAt ,
} )
// increment issueCount, to be used in GetPullRequests()
g . issueCount ++
}
return allIssues , len ( issues ) < perPage , nil
}
// GetComments returns comments according issueNumber
2020-09-03 10:35:17 +03:00
// TODO: figure out how to transfer comment reactions
2020-04-19 18:44:11 +03:00
func ( g * GitlabDownloader ) GetComments ( issueNumber int64 ) ( [ ] * base . Comment , error ) {
var allComments = make ( [ ] * base . Comment , 0 , 100 )
var page = 1
var realIssueNumber int64
for {
var comments [ ] * gitlab . Discussion
var resp * gitlab . Response
var err error
// fetchPRcomments decides whether to fetch Issue or PR comments
if ! g . fetchPRcomments {
realIssueNumber = issueNumber
comments , resp , err = g . client . Discussions . ListIssueDiscussions ( g . repoID , int ( realIssueNumber ) , & gitlab . ListIssueDiscussionsOptions {
Page : page ,
PerPage : 100 ,
2020-09-02 20:49:25 +03:00
} , nil , gitlab . WithContext ( g . ctx ) )
2020-04-19 18:44:11 +03:00
} else {
// If this is a PR, we need to figure out the Gitlab/original PR ID to be passed below
realIssueNumber = issueNumber - g . issueCount
comments , resp , err = g . client . Discussions . ListMergeRequestDiscussions ( g . repoID , int ( realIssueNumber ) , & gitlab . ListMergeRequestDiscussionsOptions {
Page : page ,
PerPage : 100 ,
2020-09-02 20:49:25 +03:00
} , nil , gitlab . WithContext ( g . ctx ) )
2020-04-19 18:44:11 +03:00
}
if err != nil {
return nil , fmt . Errorf ( "error while listing comments: %v %v" , g . repoID , err )
}
for _ , comment := range comments {
// Flatten comment threads
if ! comment . IndividualNote {
for _ , note := range comment . Notes {
allComments = append ( allComments , & base . Comment {
IssueIndex : realIssueNumber ,
PosterID : int64 ( note . Author . ID ) ,
PosterName : note . Author . Username ,
PosterEmail : note . Author . Email ,
Content : note . Body ,
Created : * note . CreatedAt ,
} )
}
} else {
c := comment . Notes [ 0 ]
allComments = append ( allComments , & base . Comment {
IssueIndex : realIssueNumber ,
PosterID : int64 ( c . Author . ID ) ,
PosterName : c . Author . Username ,
PosterEmail : c . Author . Email ,
Content : c . Body ,
Created : * c . CreatedAt ,
} )
}
}
if resp . NextPage == 0 {
break
}
page = resp . NextPage
}
return allComments , nil
}
// GetPullRequests returns pull requests according page and perPage
func ( g * GitlabDownloader ) GetPullRequests ( page , perPage int ) ( [ ] * base . PullRequest , error ) {
opt := & gitlab . ListProjectMergeRequestsOptions {
ListOptions : gitlab . ListOptions {
PerPage : perPage ,
Page : page ,
} ,
}
// Set fetchPRcomments to true here, so PR comments are fetched instead of Issue comments
g . fetchPRcomments = true
var allPRs = make ( [ ] * base . PullRequest , 0 , perPage )
2020-09-02 20:49:25 +03:00
prs , _ , err := g . client . MergeRequests . ListProjectMergeRequests ( g . repoID , opt , nil , gitlab . WithContext ( g . ctx ) )
2020-04-19 18:44:11 +03:00
if err != nil {
return nil , fmt . Errorf ( "error while listing merge requests: %v" , err )
}
for _ , pr := range prs {
var labels = make ( [ ] * base . Label , 0 , len ( pr . Labels ) )
for _ , l := range pr . Labels {
labels = append ( labels , & base . Label {
Name : l ,
} )
}
var merged bool
if pr . State == "merged" {
merged = true
pr . State = "closed"
}
var mergeTime = pr . MergedAt
if merged && pr . MergedAt == nil {
mergeTime = pr . UpdatedAt
}
var closeTime = pr . ClosedAt
if merged && pr . ClosedAt == nil {
closeTime = pr . UpdatedAt
}
var locked bool
if pr . State == "locked" {
locked = true
}
var milestone string
if pr . Milestone != nil {
milestone = pr . Milestone . Title
}
2020-09-03 10:35:17 +03:00
var reactions [ ] * base . Reaction
var awardPage = 1
for {
awards , _ , err := g . client . AwardEmoji . ListMergeRequestAwardEmoji ( g . repoID , pr . IID , & gitlab . ListAwardEmojiOptions { Page : awardPage , PerPage : perPage } , gitlab . WithContext ( g . ctx ) )
if err != nil {
return nil , fmt . Errorf ( "error while listing merge requests awards: %v" , err )
}
if len ( awards ) < perPage {
break
}
for i := range awards {
reactions = append ( reactions , g . awardToReaction ( awards [ i ] ) )
}
awardPage ++
}
2020-04-19 18:44:11 +03:00
// Add the PR ID to the Issue Count because PR and Issues share ID space in Gitea
2020-04-20 15:30:46 +03:00
newPRNumber := g . issueCount + int64 ( pr . IID )
2020-04-19 18:44:11 +03:00
allPRs = append ( allPRs , & base . PullRequest {
Title : pr . Title ,
2020-04-20 15:30:46 +03:00
Number : newPRNumber ,
OriginalNumber : int64 ( pr . IID ) ,
2020-04-19 18:44:11 +03:00
PosterName : pr . Author . Username ,
PosterID : int64 ( pr . Author . ID ) ,
Content : pr . Description ,
Milestone : milestone ,
State : pr . State ,
Created : * pr . CreatedAt ,
Closed : closeTime ,
Labels : labels ,
Merged : merged ,
MergeCommitSHA : pr . MergeCommitSHA ,
MergedTime : mergeTime ,
IsLocked : locked ,
2020-09-03 10:35:17 +03:00
Reactions : reactions ,
2020-04-19 18:44:11 +03:00
Head : base . PullRequestBranch {
Ref : pr . SourceBranch ,
SHA : pr . SHA ,
RepoName : g . repoName ,
OwnerName : pr . Author . Username ,
CloneURL : pr . WebURL ,
} ,
Base : base . PullRequestBranch {
Ref : pr . TargetBranch ,
SHA : pr . DiffRefs . BaseSha ,
RepoName : g . repoName ,
OwnerName : pr . Author . Username ,
} ,
PatchURL : pr . WebURL + ".patch" ,
} )
}
return allPRs , nil
}
// GetReviews returns pull requests review
func ( g * GitlabDownloader ) GetReviews ( pullRequestNumber int64 ) ( [ ] * base . Review , error ) {
2020-09-02 20:49:25 +03:00
state , _ , err := g . client . MergeRequestApprovals . GetApprovalState ( g . repoID , int ( pullRequestNumber ) , gitlab . WithContext ( g . ctx ) )
2020-04-20 15:30:46 +03:00
if err != nil {
return nil , err
}
// GitLab's Approvals are equivalent to Gitea's approve reviews
approvers := make ( map [ int ] string )
for i := range state . Rules {
for u := range state . Rules [ i ] . ApprovedBy {
approvers [ state . Rules [ i ] . ApprovedBy [ u ] . ID ] = state . Rules [ i ] . ApprovedBy [ u ] . Username
}
}
var reviews = make ( [ ] * base . Review , 0 , len ( approvers ) )
for id , name := range approvers {
reviews = append ( reviews , & base . Review {
ReviewerID : int64 ( id ) ,
ReviewerName : name ,
// GitLab API doesn't return a creation date
CreatedAt : time . Now ( ) ,
// All we get are approvals
State : base . ReviewStateApproved ,
} )
}
return reviews , nil
2020-04-19 18:44:11 +03:00
}
2020-09-03 10:35:17 +03:00
func ( g * GitlabDownloader ) awardToReaction ( award * gitlab . AwardEmoji ) * base . Reaction {
return & base . Reaction {
UserID : int64 ( award . User . ID ) ,
UserName : award . User . Username ,
Content : award . Name ,
}
}