2019-12-14 20:30:01 +03:00
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package repository
import (
2020-12-02 21:36:06 +03:00
"context"
2019-12-14 20:30:01 +03:00
"fmt"
2021-06-14 20:20:43 +03:00
"io"
2021-11-20 12:34:05 +03:00
"net/http"
2019-12-14 20:30:01 +03:00
"path"
"strings"
"time"
"code.gitea.io/gitea/models"
2021-09-19 14:49:59 +03:00
"code.gitea.io/gitea/models/db"
2021-12-10 04:27:50 +03:00
repo_model "code.gitea.io/gitea/models/repo"
2021-11-24 12:49:20 +03:00
user_model "code.gitea.io/gitea/models/user"
2019-12-14 20:30:01 +03:00
"code.gitea.io/gitea/modules/git"
2021-04-09 01:25:57 +03:00
"code.gitea.io/gitea/modules/lfs"
2019-12-14 20:30:01 +03:00
"code.gitea.io/gitea/modules/log"
2021-11-16 18:25:33 +03:00
"code.gitea.io/gitea/modules/migration"
2019-12-14 20:30:01 +03:00
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/timeutil"
2020-08-11 23:05:34 +03:00
"code.gitea.io/gitea/modules/util"
2020-01-20 23:01:19 +03:00
2019-12-14 20:30:01 +03:00
"gopkg.in/ini.v1"
)
/ *
GitHub , GitLab , Gogs : * . wiki . git
BitBucket : * . git / wiki
* /
var commonWikiURLSuffixes = [ ] string { ".wiki.git" , ".git/wiki" }
2020-07-06 05:08:32 +03:00
// WikiRemoteURL returns accessible repository URL for wiki if exists.
2019-12-14 20:30:01 +03:00
// Otherwise, it returns an empty string.
2022-01-20 02:26:57 +03:00
func WikiRemoteURL ( ctx context . Context , remote string ) string {
2019-12-14 20:30:01 +03:00
remote = strings . TrimSuffix ( remote , ".git" )
for _ , suffix := range commonWikiURLSuffixes {
wikiURL := remote + suffix
2022-01-20 02:26:57 +03:00
if git . IsRepoURLAccessible ( ctx , wikiURL ) {
2019-12-14 20:30:01 +03:00
return wikiURL
}
}
return ""
}
// MigrateRepositoryGitData starts migrating git related data after created migrating repository
2021-11-24 12:49:20 +03:00
func MigrateRepositoryGitData ( ctx context . Context , u * user_model . User ,
2021-12-10 04:27:50 +03:00
repo * repo_model . Repository , opts migration . MigrateOptions ,
2021-11-20 12:34:05 +03:00
httpTransport * http . Transport ,
2021-12-10 04:27:50 +03:00
) ( * repo_model . Repository , error ) {
repoPath := repo_model . RepoPath ( u . Name , opts . RepoName )
2019-12-14 20:30:01 +03:00
if u . IsOrganization ( ) {
2021-11-19 14:41:40 +03:00
t , err := models . OrgFromUser ( u ) . GetOwnerTeam ( )
2019-12-14 20:30:01 +03:00
if err != nil {
return nil , err
}
repo . NumWatches = t . NumMembers
} else {
repo . NumWatches = 1
}
migrateTimeout := time . Duration ( setting . Git . Timeout . Migrate ) * time . Second
var err error
2020-08-11 23:05:34 +03:00
if err = util . RemoveAll ( repoPath ) ; err != nil {
2019-12-14 20:30:01 +03:00
return repo , fmt . Errorf ( "Failed to remove %s: %v" , repoPath , err )
}
2022-01-20 02:26:57 +03:00
if err = git . Clone ( ctx , opts . CloneAddr , repoPath , git . CloneRepoOptions {
2022-03-19 17:16:38 +03:00
Mirror : true ,
Quiet : true ,
Timeout : migrateTimeout ,
SkipTLSVerify : setting . Migrations . SkipTLSVerify ,
2019-12-14 20:30:01 +03:00
} ) ; err != nil {
return repo , fmt . Errorf ( "Clone: %v" , err )
}
if opts . Wiki {
2021-12-10 04:27:50 +03:00
wikiPath := repo_model . WikiPath ( u . Name , opts . RepoName )
2022-01-20 02:26:57 +03:00
wikiRemotePath := WikiRemoteURL ( ctx , opts . CloneAddr )
2019-12-14 20:30:01 +03:00
if len ( wikiRemotePath ) > 0 {
2020-08-11 23:05:34 +03:00
if err := util . RemoveAll ( wikiPath ) ; err != nil {
2019-12-14 20:30:01 +03:00
return repo , fmt . Errorf ( "Failed to remove %s: %v" , wikiPath , err )
}
2022-01-20 02:26:57 +03:00
if err = git . Clone ( ctx , wikiRemotePath , wikiPath , git . CloneRepoOptions {
2022-03-19 17:16:38 +03:00
Mirror : true ,
Quiet : true ,
Timeout : migrateTimeout ,
Branch : "master" ,
SkipTLSVerify : setting . Migrations . SkipTLSVerify ,
2019-12-14 20:30:01 +03:00
} ) ; err != nil {
log . Warn ( "Clone wiki: %v" , err )
2020-08-11 23:05:34 +03:00
if err := util . RemoveAll ( wikiPath ) ; err != nil {
2019-12-14 20:30:01 +03:00
return repo , fmt . Errorf ( "Failed to remove %s: %v" , wikiPath , err )
}
}
}
}
2021-10-13 22:47:02 +03:00
if repo . OwnerID == u . ID {
repo . Owner = u
}
2021-12-10 04:27:50 +03:00
if err = models . CheckDaemonExportOK ( ctx , repo ) ; err != nil {
2021-10-13 22:47:02 +03:00
return repo , fmt . Errorf ( "checkDaemonExportOK: %v" , err )
}
2022-02-06 22:01:47 +03:00
if stdout , err := git . NewCommand ( ctx , "update-server-info" ) .
2021-10-13 22:47:02 +03:00
SetDescription ( fmt . Sprintf ( "MigrateRepositoryGitData(git update-server-info): %s" , repoPath ) ) .
RunInDir ( repoPath ) ; err != nil {
log . Error ( "MigrateRepositoryGitData(git update-server-info) in %v: Stdout: %s\nError: %v" , repo , stdout , err )
return repo , fmt . Errorf ( "error in MigrateRepositoryGitData(git update-server-info): %v" , err )
}
2022-01-20 02:26:57 +03:00
gitRepo , err := git . OpenRepositoryCtx ( ctx , repoPath )
2019-12-14 20:30:01 +03:00
if err != nil {
return repo , fmt . Errorf ( "OpenRepository: %v" , err )
}
defer gitRepo . Close ( )
repo . IsEmpty , err = gitRepo . IsEmpty ( )
if err != nil {
return repo , fmt . Errorf ( "git.IsEmpty: %v" , err )
}
2020-09-15 17:37:44 +03:00
if ! repo . IsEmpty {
if len ( repo . DefaultBranch ) == 0 {
// Try to get HEAD branch and set it as default branch.
headBranch , err := gitRepo . GetHEADBranch ( )
if err != nil {
return repo , fmt . Errorf ( "GetHEADBranch: %v" , err )
}
if headBranch != nil {
repo . DefaultBranch = headBranch . Name
}
2019-12-14 20:30:01 +03:00
}
2020-09-15 17:37:44 +03:00
if ! opts . Releases {
if err = SyncReleasesWithTags ( repo , gitRepo ) ; err != nil {
log . Error ( "Failed to synchronize tags to releases for repository: %v" , err )
}
2019-12-14 20:30:01 +03:00
}
2021-04-09 01:25:57 +03:00
if opts . LFS {
2021-11-20 12:34:05 +03:00
endpoint := lfs . DetermineEndpoint ( opts . CloneAddr , opts . LFSEndpoint )
lfsClient := lfs . NewClient ( endpoint , httpTransport )
if err = StoreMissingLfsObjectsInRepository ( ctx , repo , gitRepo , lfsClient ) ; err != nil {
2021-04-09 01:25:57 +03:00
log . Error ( "Failed to store missing LFS objects for repository: %v" , err )
}
}
2019-12-14 20:30:01 +03:00
}
2022-03-22 18:22:54 +03:00
if err = models . UpdateRepoSize ( ctx , repo ) ; err != nil {
2019-12-14 20:30:01 +03:00
log . Error ( "Failed to update size for repository: %v" , err )
}
if opts . Mirror {
2021-12-10 04:27:50 +03:00
mirrorModel := repo_model . Mirror {
2019-12-14 20:30:01 +03:00
RepoID : repo . ID ,
Interval : setting . Mirror . DefaultInterval ,
EnablePrune : true ,
NextUpdateUnix : timeutil . TimeStampNow ( ) . AddDuration ( setting . Mirror . DefaultInterval ) ,
2021-04-09 01:25:57 +03:00
LFS : opts . LFS ,
}
if opts . LFS {
mirrorModel . LFSEndpoint = opts . LFSEndpoint
2021-01-03 02:47:47 +03:00
}
if opts . MirrorInterval != "" {
parsedInterval , err := time . ParseDuration ( opts . MirrorInterval )
if err != nil {
log . Error ( "Failed to set Interval: %v" , err )
return repo , err
}
if parsedInterval == 0 {
mirrorModel . Interval = 0
mirrorModel . NextUpdateUnix = 0
} else if parsedInterval < setting . Mirror . MinInterval {
err := fmt . Errorf ( "Interval %s is set below Minimum Interval of %s" , parsedInterval , setting . Mirror . MinInterval )
log . Error ( "Interval: %s is too frequent" , opts . MirrorInterval )
return repo , err
} else {
mirrorModel . Interval = parsedInterval
mirrorModel . NextUpdateUnix = timeutil . TimeStampNow ( ) . AddDuration ( parsedInterval )
}
}
2021-12-10 04:27:50 +03:00
if err = repo_model . InsertMirror ( & mirrorModel ) ; err != nil {
2019-12-14 20:30:01 +03:00
return repo , fmt . Errorf ( "InsertOne: %v" , err )
}
repo . IsMirror = true
err = models . UpdateRepository ( repo , false )
} else {
2022-01-20 02:26:57 +03:00
repo , err = CleanUpMigrateInfo ( ctx , repo )
2019-12-14 20:30:01 +03:00
}
return repo , err
}
// cleanUpMigrateGitConfig removes mirror info which prevents "push --all".
// This also removes possible user credentials.
func cleanUpMigrateGitConfig ( configPath string ) error {
cfg , err := ini . Load ( configPath )
if err != nil {
return fmt . Errorf ( "open config file: %v" , err )
}
cfg . DeleteSection ( "remote \"origin\"" )
if err = cfg . SaveToIndent ( configPath , "\t" ) ; err != nil {
return fmt . Errorf ( "save config file: %v" , err )
}
return nil
}
// CleanUpMigrateInfo finishes migrating repository and/or wiki with things that don't need to be done for mirrors.
2022-01-20 02:26:57 +03:00
func CleanUpMigrateInfo ( ctx context . Context , repo * repo_model . Repository ) ( * repo_model . Repository , error ) {
2019-12-14 20:30:01 +03:00
repoPath := repo . RepoPath ( )
2020-01-20 23:01:19 +03:00
if err := createDelegateHooks ( repoPath ) ; err != nil {
2019-12-14 20:30:01 +03:00
return repo , fmt . Errorf ( "createDelegateHooks: %v" , err )
}
if repo . HasWiki ( ) {
2020-01-20 23:01:19 +03:00
if err := createDelegateHooks ( repo . WikiPath ( ) ) ; err != nil {
2019-12-14 20:30:01 +03:00
return repo , fmt . Errorf ( "createDelegateHooks.(wiki): %v" , err )
}
}
2022-02-06 22:01:47 +03:00
_ , err := git . NewCommand ( ctx , "remote" , "rm" , "origin" ) . RunInDir ( repoPath )
2019-12-14 20:30:01 +03:00
if err != nil && ! strings . HasPrefix ( err . Error ( ) , "exit status 128 - fatal: No such remote " ) {
return repo , fmt . Errorf ( "CleanUpMigrateInfo: %v" , err )
}
if repo . HasWiki ( ) {
if err := cleanUpMigrateGitConfig ( path . Join ( repo . WikiPath ( ) , "config" ) ) ; err != nil {
return repo , fmt . Errorf ( "cleanUpMigrateGitConfig (wiki): %v" , err )
}
}
return repo , models . UpdateRepository ( repo , false )
}
// SyncReleasesWithTags synchronizes release table with repository tags
2021-12-10 04:27:50 +03:00
func SyncReleasesWithTags ( repo * repo_model . Repository , gitRepo * git . Repository ) error {
2019-12-14 20:30:01 +03:00
existingRelTags := make ( map [ string ] struct { } )
2021-09-24 14:32:56 +03:00
opts := models . FindReleasesOptions {
IncludeDrafts : true ,
IncludeTags : true ,
ListOptions : db . ListOptions { PageSize : 50 } ,
}
2019-12-14 20:30:01 +03:00
for page := 1 ; ; page ++ {
2020-01-24 22:00:29 +03:00
opts . Page = page
rels , err := models . GetReleasesByRepoID ( repo . ID , opts )
2019-12-14 20:30:01 +03:00
if err != nil {
2022-03-10 13:09:48 +03:00
return fmt . Errorf ( "unable to GetReleasesByRepoID in Repo[%d:%s/%s]: %w" , repo . ID , repo . OwnerName , repo . Name , err )
2019-12-14 20:30:01 +03:00
}
if len ( rels ) == 0 {
break
}
for _ , rel := range rels {
if rel . IsDraft {
continue
}
commitID , err := gitRepo . GetTagCommitID ( rel . TagName )
if err != nil && ! git . IsErrNotExist ( err ) {
2022-03-10 13:09:48 +03:00
return fmt . Errorf ( "unable to GetTagCommitID for %q in Repo[%d:%s/%s]: %w" , rel . TagName , repo . ID , repo . OwnerName , repo . Name , err )
2019-12-14 20:30:01 +03:00
}
if git . IsErrNotExist ( err ) || commitID != rel . Sha1 {
if err := models . PushUpdateDeleteTag ( repo , rel . TagName ) ; err != nil {
2022-03-10 13:09:48 +03:00
return fmt . Errorf ( "unable to PushUpdateDeleteTag: %q in Repo[%d:%s/%s]: %w" , rel . TagName , repo . ID , repo . OwnerName , repo . Name , err )
2019-12-14 20:30:01 +03:00
}
} else {
existingRelTags [ strings . ToLower ( rel . TagName ) ] = struct { } { }
}
}
}
2021-09-10 20:30:37 +03:00
tags , err := gitRepo . GetTags ( 0 , 0 )
2019-12-14 20:30:01 +03:00
if err != nil {
2022-03-10 13:09:48 +03:00
return fmt . Errorf ( "unable to GetTags in Repo[%d:%s/%s]: %w" , repo . ID , repo . OwnerName , repo . Name , err )
2019-12-14 20:30:01 +03:00
}
for _ , tagName := range tags {
if _ , ok := existingRelTags [ strings . ToLower ( tagName ) ] ; ! ok {
2020-01-10 12:34:21 +03:00
if err := PushUpdateAddTag ( repo , gitRepo , tagName ) ; err != nil {
2022-03-10 13:09:48 +03:00
return fmt . Errorf ( "unable to PushUpdateAddTag: %q to Repo[%d:%s/%s]: %w" , tagName , repo . ID , repo . OwnerName , repo . Name , err )
2019-12-14 20:30:01 +03:00
}
}
}
return nil
}
2020-01-10 12:34:21 +03:00
// PushUpdateAddTag must be called for any push actions to add tag
2021-12-10 04:27:50 +03:00
func PushUpdateAddTag ( repo * repo_model . Repository , gitRepo * git . Repository , tagName string ) error {
2020-01-10 12:34:21 +03:00
tag , err := gitRepo . GetTag ( tagName )
if err != nil {
2022-03-10 13:09:48 +03:00
return fmt . Errorf ( "unable to GetTag: %w" , err )
2020-01-10 12:34:21 +03:00
}
2022-01-12 23:37:46 +03:00
commit , err := tag . Commit ( gitRepo )
2020-01-10 12:34:21 +03:00
if err != nil {
2022-03-10 13:09:48 +03:00
return fmt . Errorf ( "unable to get tag Commit: %w" , err )
2020-01-10 12:34:21 +03:00
}
sig := tag . Tagger
if sig == nil {
sig = commit . Author
}
if sig == nil {
sig = commit . Committer
}
2021-11-24 12:49:20 +03:00
var author * user_model . User
2022-01-20 20:46:10 +03:00
createdAt := time . Unix ( 1 , 0 )
2020-01-10 12:34:21 +03:00
if sig != nil {
2021-11-24 12:49:20 +03:00
author , err = user_model . GetUserByEmail ( sig . Email )
if err != nil && ! user_model . IsErrUserNotExist ( err ) {
2022-03-10 13:09:48 +03:00
return fmt . Errorf ( "unable to GetUserByEmail for %q: %w" , sig . Email , err )
2020-01-10 12:34:21 +03:00
}
createdAt = sig . When
}
commitsCount , err := commit . CommitsCount ( )
if err != nil {
2022-03-10 13:09:48 +03:00
return fmt . Errorf ( "unable to get CommitsCount: %w" , err )
2020-01-10 12:34:21 +03:00
}
2022-01-20 20:46:10 +03:00
rel := models . Release {
2020-01-10 12:34:21 +03:00
RepoID : repo . ID ,
TagName : tagName ,
LowerTagName : strings . ToLower ( tagName ) ,
Sha1 : commit . ID . String ( ) ,
NumCommits : commitsCount ,
CreatedUnix : timeutil . TimeStamp ( createdAt . Unix ( ) ) ,
IsTag : true ,
}
if author != nil {
rel . PublisherID = author . ID
}
return models . SaveOrUpdateTag ( repo , & rel )
}
2021-04-09 01:25:57 +03:00
// StoreMissingLfsObjectsInRepository downloads missing LFS objects
2021-12-10 04:27:50 +03:00
func StoreMissingLfsObjectsInRepository ( ctx context . Context , repo * repo_model . Repository , gitRepo * git . Repository , lfsClient lfs . Client ) error {
2021-04-09 01:25:57 +03:00
contentStore := lfs . NewContentStore ( )
pointerChan := make ( chan lfs . PointerBlob )
errChan := make ( chan error , 1 )
go lfs . SearchPointerBlobs ( ctx , gitRepo , pointerChan , errChan )
2021-06-14 20:20:43 +03:00
downloadObjects := func ( pointers [ ] lfs . Pointer ) error {
2021-11-20 12:34:05 +03:00
err := lfsClient . Download ( ctx , pointers , func ( p lfs . Pointer , content io . ReadCloser , objectError error ) error {
2021-06-14 20:20:43 +03:00
if objectError != nil {
return objectError
2021-04-09 01:25:57 +03:00
}
2021-06-14 20:20:43 +03:00
defer content . Close ( )
2021-04-09 01:25:57 +03:00
2021-06-14 20:20:43 +03:00
_ , err := models . NewLFSMetaObject ( & models . LFSMetaObject { Pointer : p , RepositoryID : repo . ID } )
2021-04-09 01:25:57 +03:00
if err != nil {
2022-03-10 13:09:48 +03:00
log . Error ( "Repo[%-v]: Error creating LFS meta object %-v: %v" , repo , p , err )
2021-06-14 20:20:43 +03:00
return err
}
2021-04-09 01:25:57 +03:00
2021-06-14 20:20:43 +03:00
if err := contentStore . Put ( p , content ) ; err != nil {
2022-03-10 13:09:48 +03:00
log . Error ( "Repo[%-v]: Error storing content for LFS meta object %-v: %v" , repo , p , err )
2021-12-10 04:27:50 +03:00
if _ , err2 := models . RemoveLFSMetaObjectByOid ( repo . ID , p . Oid ) ; err2 != nil {
2022-03-10 13:09:48 +03:00
log . Error ( "Repo[%-v]: Error removing LFS meta object %-v: %v" , repo , p , err2 )
2021-04-09 01:25:57 +03:00
}
return err
}
2021-06-14 20:20:43 +03:00
return nil
} )
if err != nil {
select {
case <- ctx . Done ( ) :
return nil
default :
}
2021-04-09 01:25:57 +03:00
}
return err
}
2021-06-14 20:20:43 +03:00
var batch [ ] lfs . Pointer
for pointerBlob := range pointerChan {
2021-12-10 04:27:50 +03:00
meta , err := models . GetLFSMetaObjectByOid ( repo . ID , pointerBlob . Oid )
2021-06-14 20:20:43 +03:00
if err != nil && err != models . ErrLFSObjectNotExist {
2022-03-10 13:09:48 +03:00
log . Error ( "Repo[%-v]: Error querying LFS meta object %-v: %v" , repo , pointerBlob . Pointer , err )
2021-06-14 20:20:43 +03:00
return err
}
if meta != nil {
2022-03-10 13:09:48 +03:00
log . Trace ( "Repo[%-v]: Skipping unknown LFS meta object %-v" , repo , pointerBlob . Pointer )
2021-06-14 20:20:43 +03:00
continue
}
2022-03-10 13:09:48 +03:00
log . Trace ( "Repo[%-v]: LFS object %-v not present in repository" , repo , pointerBlob . Pointer )
2021-06-14 20:20:43 +03:00
exist , err := contentStore . Exists ( pointerBlob . Pointer )
if err != nil {
2022-03-10 13:09:48 +03:00
log . Error ( "Repo[%-v]: Error checking if LFS object %-v exists: %v" , repo , pointerBlob . Pointer , err )
2021-06-14 20:20:43 +03:00
return err
}
if exist {
2022-03-10 13:09:48 +03:00
log . Trace ( "Repo[%-v]: LFS object %-v already present; creating meta object" , repo , pointerBlob . Pointer )
2021-06-14 20:20:43 +03:00
_ , err := models . NewLFSMetaObject ( & models . LFSMetaObject { Pointer : pointerBlob . Pointer , RepositoryID : repo . ID } )
if err != nil {
2022-03-10 13:09:48 +03:00
log . Error ( "Repo[%-v]: Error creating LFS meta object %-v: %v" , repo , pointerBlob . Pointer , err )
2021-06-14 20:20:43 +03:00
return err
}
} else {
if setting . LFS . MaxFileSize > 0 && pointerBlob . Size > setting . LFS . MaxFileSize {
2022-03-10 13:09:48 +03:00
log . Info ( "Repo[%-v]: LFS object %-v download denied because of LFS_MAX_FILE_SIZE=%d < size %d" , repo , pointerBlob . Pointer , setting . LFS . MaxFileSize , pointerBlob . Size )
2021-06-14 20:20:43 +03:00
continue
}
batch = append ( batch , pointerBlob . Pointer )
2021-11-20 12:34:05 +03:00
if len ( batch ) >= lfsClient . BatchSize ( ) {
2021-06-14 20:20:43 +03:00
if err := downloadObjects ( batch ) ; err != nil {
return err
}
batch = nil
}
}
}
if len ( batch ) > 0 {
if err := downloadObjects ( batch ) ; err != nil {
return err
}
}
2021-04-09 01:25:57 +03:00
err , has := <- errChan
if has {
2022-03-10 13:09:48 +03:00
log . Error ( "Repo[%-v]: Error enumerating LFS objects for repository: %v" , repo , err )
2021-04-09 01:25:57 +03:00
return err
}
return nil
}