2019-05-07 04:12:51 +03:00
// Copyright 2019 The Gitea Authors. All rights reserved.
// Copyright 2018 Jonas Franz. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package migrations
import (
2019-12-17 07:16:54 +03:00
"context"
2019-10-13 16:23:14 +03:00
"fmt"
2020-11-29 03:37:58 +03:00
"net"
"net/url"
2021-03-16 00:52:11 +03:00
"path/filepath"
2020-11-29 03:37:58 +03:00
"strings"
2019-10-13 16:23:14 +03:00
2019-05-07 04:12:51 +03:00
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/log"
2020-11-29 03:37:58 +03:00
"code.gitea.io/gitea/modules/matchlist"
2019-05-07 04:12:51 +03:00
"code.gitea.io/gitea/modules/migrations/base"
2019-11-16 11:30:06 +03:00
"code.gitea.io/gitea/modules/setting"
2021-03-16 00:52:11 +03:00
"code.gitea.io/gitea/modules/util"
2019-05-07 04:12:51 +03:00
)
// MigrateOptions is equal to base.MigrateOptions
type MigrateOptions = base . MigrateOptions
var (
factories [ ] base . DownloaderFactory
2020-11-29 03:37:58 +03:00
allowList * matchlist . Matchlist
blockList * matchlist . Matchlist
2019-05-07 04:12:51 +03:00
)
// RegisterDownloaderFactory registers a downloader factory
func RegisterDownloaderFactory ( factory base . DownloaderFactory ) {
factories = append ( factories , factory )
}
2021-03-16 00:52:11 +03:00
// IsMigrateURLAllowed checks if an URL is allowed to be migrated from
func IsMigrateURLAllowed ( remoteURL string , doer * models . User ) error {
// Remote address can be HTTP/HTTPS/Git URL or local path.
2021-03-18 16:58:47 +03:00
u , err := url . Parse ( remoteURL )
2020-11-29 03:37:58 +03:00
if err != nil {
2021-03-16 00:52:11 +03:00
return & models . ErrInvalidCloneAddr { IsURLError : true }
2020-11-29 03:37:58 +03:00
}
2021-03-16 00:52:11 +03:00
if u . Scheme == "file" || u . Scheme == "" {
if ! doer . CanImportLocal ( ) {
return & models . ErrInvalidCloneAddr { Host : "<LOCAL_FILESYSTEM>" , IsPermissionDenied : true , LocalPath : true }
2020-11-29 03:37:58 +03:00
}
2021-03-16 00:52:11 +03:00
isAbs := filepath . IsAbs ( u . Host + u . Path )
if ! isAbs {
return & models . ErrInvalidCloneAddr { Host : "<LOCAL_FILESYSTEM>" , IsInvalidPath : true , LocalPath : true }
}
isDir , err := util . IsDir ( u . Host + u . Path )
if err != nil {
log . Error ( "Unable to check if %s is a directory: %v" , u . Host + u . Path , err )
return err
}
if ! isDir {
return & models . ErrInvalidCloneAddr { Host : "<LOCAL_FILESYSTEM>" , IsInvalidPath : true , LocalPath : true }
}
return nil
}
if u . Scheme == "git" && u . Port ( ) != "" && ( strings . Contains ( remoteURL , "%0d" ) || strings . Contains ( remoteURL , "%0a" ) ) {
return & models . ErrInvalidCloneAddr { Host : u . Host , IsURLError : true }
2020-11-29 03:37:58 +03:00
}
2021-03-16 00:52:11 +03:00
if u . Opaque != "" || u . Scheme != "" && u . Scheme != "http" && u . Scheme != "https" && u . Scheme != "git" {
return & models . ErrInvalidCloneAddr { Host : u . Host , IsProtocolInvalid : true , IsPermissionDenied : true , IsURLError : true }
}
2021-03-18 16:58:47 +03:00
host := strings . ToLower ( u . Host )
2021-03-16 00:52:11 +03:00
if len ( setting . Migrations . AllowedDomains ) > 0 {
2021-03-18 16:58:47 +03:00
if ! allowList . Match ( host ) {
2021-03-16 00:52:11 +03:00
return & models . ErrInvalidCloneAddr { Host : u . Host , IsPermissionDenied : true }
}
} else {
2021-03-18 16:58:47 +03:00
if blockList . Match ( host ) {
2021-03-16 00:52:11 +03:00
return & models . ErrInvalidCloneAddr { Host : u . Host , IsPermissionDenied : true }
2021-03-08 16:10:17 +03:00
}
}
2020-11-29 03:37:58 +03:00
if ! setting . Migrations . AllowLocalNetworks {
addrList , err := net . LookupIP ( strings . Split ( u . Host , ":" ) [ 0 ] )
if err != nil {
2021-03-16 00:52:11 +03:00
return & models . ErrInvalidCloneAddr { Host : u . Host , NotResolvedIP : true }
2020-11-29 03:37:58 +03:00
}
for _ , addr := range addrList {
2021-11-01 11:39:52 +03:00
if util . IsIPPrivate ( addr ) || ! addr . IsGlobalUnicast ( ) {
2021-03-16 00:52:11 +03:00
return & models . ErrInvalidCloneAddr { Host : u . Host , PrivateNet : addr . String ( ) , IsPermissionDenied : true }
2020-11-29 03:37:58 +03:00
}
}
}
return nil
}
2019-05-07 04:12:51 +03:00
// MigrateRepository migrate repository according MigrateOptions
2021-06-17 01:02:24 +03:00
func MigrateRepository ( ctx context . Context , doer * models . User , ownerName string , opts base . MigrateOptions , messenger base . Messenger ) ( * models . Repository , error ) {
2021-03-16 00:52:11 +03:00
err := IsMigrateURLAllowed ( opts . CloneAddr , doer )
2020-11-29 03:37:58 +03:00
if err != nil {
return nil , err
}
2021-04-09 01:25:57 +03:00
if opts . LFS && len ( opts . LFSEndpoint ) > 0 {
err := IsMigrateURLAllowed ( opts . LFSEndpoint , doer )
if err != nil {
return nil , err
}
}
2020-12-27 06:34:19 +03:00
downloader , err := newDownloader ( ctx , ownerName , opts )
if err != nil {
return nil , err
}
var uploader = NewGiteaLocalUploader ( ctx , doer , ownerName , opts . RepoName )
uploader . gitServiceType = opts . GitServiceType
2021-06-17 01:02:24 +03:00
if err := migrateRepository ( downloader , uploader , opts , messenger ) ; err != nil {
2020-12-27 06:34:19 +03:00
if err1 := uploader . Rollback ( ) ; err1 != nil {
log . Error ( "rollback failed: %v" , err1 )
}
if err2 := models . CreateRepositoryNotice ( fmt . Sprintf ( "Migrate repository from %s failed: %v" , opts . OriginalURL , err ) ) ; err2 != nil {
log . Error ( "create respotiry notice failed: " , err2 )
}
return nil , err
}
return uploader . repo , nil
}
2020-11-29 03:37:58 +03:00
2020-12-27 06:34:19 +03:00
func newDownloader ( ctx context . Context , ownerName string , opts base . MigrateOptions ) ( base . Downloader , error ) {
2019-05-07 04:12:51 +03:00
var (
downloader base . Downloader
2020-12-27 06:34:19 +03:00
err error
2019-05-07 04:12:51 +03:00
)
for _ , factory := range factories {
2020-08-28 04:36:37 +03:00
if factory . GitServiceType ( ) == opts . GitServiceType {
2020-09-02 20:49:25 +03:00
downloader , err = factory . New ( ctx , opts )
2019-05-07 04:12:51 +03:00
if err != nil {
return nil , err
}
break
}
}
if downloader == nil {
opts . Wiki = true
opts . Milestones = false
opts . Labels = false
opts . Releases = false
opts . Comments = false
opts . Issues = false
opts . PullRequests = false
2019-10-13 16:23:14 +03:00
downloader = NewPlainGitDownloader ( ownerName , opts . RepoName , opts . CloneAddr )
2019-12-19 00:49:56 +03:00
log . Trace ( "Will migrate from git: %s" , opts . OriginalURL )
2019-05-07 04:12:51 +03:00
}
2019-11-16 11:30:06 +03:00
if setting . Migrations . MaxAttempts > 1 {
2020-09-02 20:49:25 +03:00
downloader = base . NewRetryDownloader ( ctx , downloader , setting . Migrations . MaxAttempts , setting . Migrations . RetryBackoff )
2019-11-16 11:30:06 +03:00
}
2020-12-27 06:34:19 +03:00
return downloader , nil
2019-05-07 04:12:51 +03:00
}
2020-04-20 15:30:46 +03:00
// migrateRepository will download information and then upload it to Uploader, this is a simple
2019-05-07 04:12:51 +03:00
// process for small repository. For a big repository, save all the data to disk
// before upload is better
2021-06-17 01:02:24 +03:00
func migrateRepository ( downloader base . Downloader , uploader base . Uploader , opts base . MigrateOptions , messenger base . Messenger ) error {
if messenger == nil {
messenger = base . NilMessenger
}
2019-05-07 04:12:51 +03:00
repo , err := downloader . GetRepoInfo ( )
if err != nil {
2021-01-21 22:33:58 +03:00
if ! base . IsErrNotSupported ( err ) {
return err
}
log . Info ( "migrating repo infos is not supported, ignored" )
2019-05-07 04:12:51 +03:00
}
repo . IsPrivate = opts . Private
repo . IsMirror = opts . Mirror
2019-05-20 15:43:43 +03:00
if opts . Description != "" {
repo . Description = opts . Description
}
2021-01-21 22:33:58 +03:00
if repo . CloneURL , err = downloader . FormatCloneURL ( opts , repo . CloneURL ) ; err != nil {
return err
}
2021-06-04 16:14:20 +03:00
log . Trace ( "migrating git data from %s" , repo . CloneURL )
2021-06-17 01:02:24 +03:00
messenger ( "repo.migrate.migrating_git" )
2021-01-21 22:33:58 +03:00
if err = uploader . CreateRepo ( repo , opts ) ; err != nil {
2019-05-07 04:12:51 +03:00
return err
}
2019-11-13 10:01:19 +03:00
defer uploader . Close ( )
2019-05-07 04:12:51 +03:00
2019-08-14 09:16:12 +03:00
log . Trace ( "migrating topics" )
2021-06-17 01:02:24 +03:00
messenger ( "repo.migrate.migrating_topics" )
2019-08-14 09:16:12 +03:00
topics , err := downloader . GetTopics ( )
if err != nil {
2021-01-21 22:33:58 +03:00
if ! base . IsErrNotSupported ( err ) {
return err
}
log . Warn ( "migrating topics is not supported, ignored" )
2019-08-14 09:16:12 +03:00
}
2021-01-21 22:33:58 +03:00
if len ( topics ) != 0 {
if err = uploader . CreateTopics ( topics ... ) ; err != nil {
2019-08-14 09:16:12 +03:00
return err
}
}
2019-05-07 04:12:51 +03:00
if opts . Milestones {
log . Trace ( "migrating milestones" )
2021-06-17 01:02:24 +03:00
messenger ( "repo.migrate.migrating_milestones" )
2019-05-07 04:12:51 +03:00
milestones , err := downloader . GetMilestones ( )
if err != nil {
2021-01-21 22:33:58 +03:00
if ! base . IsErrNotSupported ( err ) {
return err
}
log . Warn ( "migrating milestones is not supported, ignored" )
2019-05-07 04:12:51 +03:00
}
2019-07-06 22:24:50 +03:00
msBatchSize := uploader . MaxBatchInsertSize ( "milestone" )
for len ( milestones ) > 0 {
if len ( milestones ) < msBatchSize {
msBatchSize = len ( milestones )
}
if err := uploader . CreateMilestones ( milestones ... ) ; err != nil {
return err
}
milestones = milestones [ msBatchSize : ]
2019-05-07 04:12:51 +03:00
}
}
if opts . Labels {
log . Trace ( "migrating labels" )
2021-06-17 01:02:24 +03:00
messenger ( "repo.migrate.migrating_labels" )
2019-05-07 04:12:51 +03:00
labels , err := downloader . GetLabels ( )
if err != nil {
2021-01-21 22:33:58 +03:00
if ! base . IsErrNotSupported ( err ) {
return err
}
log . Warn ( "migrating labels is not supported, ignored" )
2019-05-07 04:12:51 +03:00
}
2019-07-06 22:24:50 +03:00
lbBatchSize := uploader . MaxBatchInsertSize ( "label" )
for len ( labels ) > 0 {
if len ( labels ) < lbBatchSize {
lbBatchSize = len ( labels )
}
if err := uploader . CreateLabels ( labels ... ) ; err != nil {
return err
}
labels = labels [ lbBatchSize : ]
2019-05-07 04:12:51 +03:00
}
}
if opts . Releases {
log . Trace ( "migrating releases" )
2021-06-17 01:02:24 +03:00
messenger ( "repo.migrate.migrating_releases" )
2019-05-07 04:12:51 +03:00
releases , err := downloader . GetReleases ( )
if err != nil {
2021-01-21 22:33:58 +03:00
if ! base . IsErrNotSupported ( err ) {
return err
}
log . Warn ( "migrating releases is not supported, ignored" )
2019-05-07 04:12:51 +03:00
}
2019-07-06 22:24:50 +03:00
relBatchSize := uploader . MaxBatchInsertSize ( "release" )
for len ( releases ) > 0 {
2019-12-12 03:20:11 +03:00
if len ( releases ) < relBatchSize {
relBatchSize = len ( releases )
2019-07-06 22:24:50 +03:00
}
2021-01-21 22:33:58 +03:00
if err = uploader . CreateReleases ( releases [ : relBatchSize ] ... ) ; err != nil {
2019-07-06 22:24:50 +03:00
return err
}
releases = releases [ relBatchSize : ]
2019-05-07 04:12:51 +03:00
}
2019-12-12 03:20:11 +03:00
// Once all releases (if any) are inserted, sync any remaining non-release tags
2021-01-21 22:33:58 +03:00
if err = uploader . SyncTags ( ) ; err != nil {
2019-12-12 03:20:11 +03:00
return err
}
2019-05-07 04:12:51 +03:00
}
2020-01-23 20:28:15 +03:00
var (
commentBatchSize = uploader . MaxBatchInsertSize ( "comment" )
reviewBatchSize = uploader . MaxBatchInsertSize ( "review" )
)
2019-07-06 22:24:50 +03:00
2021-06-30 10:23:49 +03:00
supportAllComments := downloader . SupportGetRepoComments ( )
2019-05-07 04:12:51 +03:00
if opts . Issues {
log . Trace ( "migrating issues and comments" )
2021-06-17 01:02:24 +03:00
messenger ( "repo.migrate.migrating_issues" )
2019-07-06 22:24:50 +03:00
var issueBatchSize = uploader . MaxBatchInsertSize ( "issue" )
2019-05-30 23:26:57 +03:00
for i := 1 ; ; i ++ {
2019-07-06 22:24:50 +03:00
issues , isEnd , err := downloader . GetIssues ( i , issueBatchSize )
2019-05-07 04:12:51 +03:00
if err != nil {
2021-01-21 22:33:58 +03:00
if ! base . IsErrNotSupported ( err ) {
return err
}
log . Warn ( "migrating issues is not supported, ignored" )
break
2019-05-07 04:12:51 +03:00
}
2019-06-29 16:38:22 +03:00
if err := uploader . CreateIssues ( issues ... ) ; err != nil {
return err
}
2019-05-07 04:12:51 +03:00
2021-06-30 10:23:49 +03:00
if opts . Comments && ! supportAllComments {
2020-12-27 06:34:19 +03:00
var allComments = make ( [ ] * base . Comment , 0 , commentBatchSize )
for _ , issue := range issues {
log . Trace ( "migrating issue %d's comments" , issue . Number )
2021-06-30 10:23:49 +03:00
comments , _ , err := downloader . GetComments ( base . GetCommentOptions {
2021-08-22 01:47:45 +03:00
Context : issue . Context ,
2021-06-30 10:23:49 +03:00
} )
2020-12-27 06:34:19 +03:00
if err != nil {
2021-01-21 22:33:58 +03:00
if ! base . IsErrNotSupported ( err ) {
return err
}
log . Warn ( "migrating comments is not supported, ignored" )
2020-12-27 06:34:19 +03:00
}
2019-05-07 04:12:51 +03:00
2020-12-27 06:34:19 +03:00
allComments = append ( allComments , comments ... )
2019-07-08 05:14:12 +03:00
2020-12-27 06:34:19 +03:00
if len ( allComments ) >= commentBatchSize {
2021-01-21 22:33:58 +03:00
if err = uploader . CreateComments ( allComments [ : commentBatchSize ] ... ) ; err != nil {
2020-12-27 06:34:19 +03:00
return err
}
2019-06-29 16:38:22 +03:00
2020-12-27 06:34:19 +03:00
allComments = allComments [ commentBatchSize : ]
2019-05-07 04:12:51 +03:00
}
2019-06-29 16:38:22 +03:00
}
2020-12-27 06:34:19 +03:00
if len ( allComments ) > 0 {
2021-01-21 22:33:58 +03:00
if err = uploader . CreateComments ( allComments ... ) ; err != nil {
2020-12-27 06:34:19 +03:00
return err
}
2019-05-07 04:12:51 +03:00
}
}
2019-05-30 23:26:57 +03:00
if isEnd {
2019-05-07 04:12:51 +03:00
break
}
}
}
if opts . PullRequests {
log . Trace ( "migrating pull requests and comments" )
2021-06-17 01:02:24 +03:00
messenger ( "repo.migrate.migrating_pulls" )
2019-07-06 23:32:15 +03:00
var prBatchSize = uploader . MaxBatchInsertSize ( "pullrequest" )
2019-05-30 23:26:57 +03:00
for i := 1 ; ; i ++ {
2020-10-14 07:06:00 +03:00
prs , isEnd , err := downloader . GetPullRequests ( i , prBatchSize )
2019-05-07 04:12:51 +03:00
if err != nil {
2021-01-21 22:33:58 +03:00
if ! base . IsErrNotSupported ( err ) {
return err
}
log . Warn ( "migrating pull requests is not supported, ignored" )
break
2019-05-07 04:12:51 +03:00
}
2019-06-29 16:38:22 +03:00
if err := uploader . CreatePullRequests ( prs ... ) ; err != nil {
return err
}
2019-05-07 04:12:51 +03:00
2020-12-27 06:34:19 +03:00
if opts . Comments {
2021-06-30 10:23:49 +03:00
if ! supportAllComments {
// plain comments
var allComments = make ( [ ] * base . Comment , 0 , commentBatchSize )
for _ , pr := range prs {
log . Trace ( "migrating pull request %d's comments" , pr . Number )
comments , _ , err := downloader . GetComments ( base . GetCommentOptions {
2021-08-22 01:47:45 +03:00
Context : pr . Context ,
2021-06-30 10:23:49 +03:00
} )
if err != nil {
if ! base . IsErrNotSupported ( err ) {
return err
}
log . Warn ( "migrating comments is not supported, ignored" )
2021-01-21 22:33:58 +03:00
}
2019-06-29 16:38:22 +03:00
2021-06-30 10:23:49 +03:00
allComments = append ( allComments , comments ... )
2019-06-29 16:38:22 +03:00
2021-06-30 10:23:49 +03:00
if len ( allComments ) >= commentBatchSize {
if err = uploader . CreateComments ( allComments [ : commentBatchSize ] ... ) ; err != nil {
return err
}
allComments = allComments [ commentBatchSize : ]
2020-12-27 06:34:19 +03:00
}
2019-05-07 04:12:51 +03:00
}
2021-06-30 10:23:49 +03:00
if len ( allComments ) > 0 {
if err = uploader . CreateComments ( allComments ... ) ; err != nil {
return err
}
2020-12-27 06:34:19 +03:00
}
2019-06-29 16:38:22 +03:00
}
2020-12-27 06:34:19 +03:00
// migrate reviews
var allReviews = make ( [ ] * base . Review , 0 , reviewBatchSize )
for _ , pr := range prs {
2021-08-22 01:47:45 +03:00
reviews , err := downloader . GetReviews ( pr . Context )
2021-01-21 22:33:58 +03:00
if err != nil {
if ! base . IsErrNotSupported ( err ) {
return err
}
log . Warn ( "migrating reviews is not supported, ignored" )
break
}
2020-01-23 20:28:15 +03:00
2020-12-27 06:34:19 +03:00
allReviews = append ( allReviews , reviews ... )
2020-01-23 20:28:15 +03:00
2020-12-27 06:34:19 +03:00
if len ( allReviews ) >= reviewBatchSize {
2021-01-21 22:33:58 +03:00
if err = uploader . CreateReviews ( allReviews [ : reviewBatchSize ] ... ) ; err != nil {
2020-12-27 06:34:19 +03:00
return err
}
allReviews = allReviews [ reviewBatchSize : ]
2020-01-23 20:28:15 +03:00
}
}
2020-12-27 06:34:19 +03:00
if len ( allReviews ) > 0 {
2021-01-21 22:33:58 +03:00
if err = uploader . CreateReviews ( allReviews ... ) ; err != nil {
2020-12-27 06:34:19 +03:00
return err
}
2020-01-23 20:28:15 +03:00
}
}
2020-10-14 07:06:00 +03:00
if isEnd {
2019-05-07 04:12:51 +03:00
break
}
}
}
2021-06-30 10:23:49 +03:00
if opts . Comments && supportAllComments {
log . Trace ( "migrating comments" )
for i := 1 ; ; i ++ {
comments , isEnd , err := downloader . GetComments ( base . GetCommentOptions {
Page : i ,
PageSize : commentBatchSize ,
} )
if err != nil {
return err
}
if err := uploader . CreateComments ( comments ... ) ; err != nil {
return err
}
if isEnd {
break
}
}
}
2020-12-27 06:34:19 +03:00
return uploader . Finish ( )
2019-05-07 04:12:51 +03:00
}
2020-11-29 03:37:58 +03:00
// Init migrations service
func Init ( ) error {
var err error
allowList , err = matchlist . NewMatchlist ( setting . Migrations . AllowedDomains ... )
if err != nil {
return fmt . Errorf ( "init migration allowList domains failed: %v" , err )
}
blockList , err = matchlist . NewMatchlist ( setting . Migrations . BlockedDomains ... )
if err != nil {
return fmt . Errorf ( "init migration blockList domains failed: %v" , err )
}
return nil
}