2021-06-14 20:20:43 +03:00
// Copyright 2021 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package mirror
import (
"context"
"errors"
2021-11-30 23:06:32 +03:00
"fmt"
2021-06-14 20:20:43 +03:00
"io"
"regexp"
2022-03-27 14:54:09 +03:00
"strings"
2021-06-14 20:20:43 +03:00
"time"
2021-12-10 04:27:50 +03:00
repo_model "code.gitea.io/gitea/models/repo"
2021-06-14 20:20:43 +03:00
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/lfs"
"code.gitea.io/gitea/modules/log"
2021-11-30 23:06:32 +03:00
"code.gitea.io/gitea/modules/process"
2021-06-14 20:20:43 +03:00
"code.gitea.io/gitea/modules/repository"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/timeutil"
"code.gitea.io/gitea/modules/util"
)
var stripExitStatus = regexp . MustCompile ( ` exit status \d+ - ` )
// AddPushMirrorRemote registers the push mirror remote.
2022-01-20 02:26:57 +03:00
func AddPushMirrorRemote ( ctx context . Context , m * repo_model . PushMirror , addr string ) error {
2021-06-14 20:20:43 +03:00
addRemoteAndConfig := func ( addr , path string ) error {
2022-03-27 14:54:09 +03:00
cmd := git . NewCommand ( ctx , "remote" , "add" , "--mirror=push" , m . RemoteName , addr )
if strings . Contains ( addr , "://" ) && strings . Contains ( addr , "@" ) {
2022-03-31 05:25:40 +03:00
cmd . SetDescription ( fmt . Sprintf ( "remote add %s --mirror=push %s [repo_path: %s]" , m . RemoteName , util . SanitizeCredentialURLs ( addr ) , path ) )
2022-03-27 14:54:09 +03:00
} else {
cmd . SetDescription ( fmt . Sprintf ( "remote add %s --mirror=push %s [repo_path: %s]" , m . RemoteName , addr , path ) )
}
2022-04-01 05:55:30 +03:00
if _ , _ , err := cmd . RunStdString ( & git . RunOpts { Dir : path } ) ; err != nil {
2021-06-14 20:20:43 +03:00
return err
}
2022-04-01 05:55:30 +03:00
if _ , _ , err := git . NewCommand ( ctx , "config" , "--add" , "remote." + m . RemoteName + ".push" , "+refs/heads/*:refs/heads/*" ) . RunStdString ( & git . RunOpts { Dir : path } ) ; err != nil {
2021-06-14 20:20:43 +03:00
return err
}
2022-04-01 05:55:30 +03:00
if _ , _ , err := git . NewCommand ( ctx , "config" , "--add" , "remote." + m . RemoteName + ".push" , "+refs/tags/*:refs/tags/*" ) . RunStdString ( & git . RunOpts { Dir : path } ) ; err != nil {
2021-06-14 20:20:43 +03:00
return err
}
return nil
}
if err := addRemoteAndConfig ( addr , m . Repo . RepoPath ( ) ) ; err != nil {
return err
}
if m . Repo . HasWiki ( ) {
2022-01-20 02:26:57 +03:00
wikiRemoteURL := repository . WikiRemoteURL ( ctx , addr )
2021-06-14 20:20:43 +03:00
if len ( wikiRemoteURL ) > 0 {
if err := addRemoteAndConfig ( wikiRemoteURL , m . Repo . WikiPath ( ) ) ; err != nil {
return err
}
}
}
return nil
}
// RemovePushMirrorRemote removes the push mirror remote.
2022-01-20 02:26:57 +03:00
func RemovePushMirrorRemote ( ctx context . Context , m * repo_model . PushMirror ) error {
2022-02-06 22:01:47 +03:00
cmd := git . NewCommand ( ctx , "remote" , "rm" , m . RemoteName )
2022-05-20 17:08:52 +03:00
_ = m . GetRepository ( )
2021-06-14 20:20:43 +03:00
2022-04-01 05:55:30 +03:00
if _ , _ , err := cmd . RunStdString ( & git . RunOpts { Dir : m . Repo . RepoPath ( ) } ) ; err != nil {
2021-06-14 20:20:43 +03:00
return err
}
if m . Repo . HasWiki ( ) {
2022-04-01 05:55:30 +03:00
if _ , _ , err := cmd . RunStdString ( & git . RunOpts { Dir : m . Repo . WikiPath ( ) } ) ; err != nil {
2021-06-14 20:20:43 +03:00
// The wiki remote may not exist
log . Warn ( "Wiki Remote[%d] could not be removed: %v" , m . ID , err )
}
}
return nil
}
// SyncPushMirror starts the sync of the push mirror and schedules the next run.
func SyncPushMirror ( ctx context . Context , mirrorID int64 ) bool {
log . Trace ( "SyncPushMirror [mirror: %d]" , mirrorID )
defer func ( ) {
err := recover ( )
if err == nil {
return
}
// There was a panic whilst syncPushMirror...
log . Error ( "PANIC whilst syncPushMirror[%d] Panic: %v\nStacktrace: %s" , mirrorID , err , log . Stack ( 2 ) )
} ( )
2021-12-10 04:27:50 +03:00
m , err := repo_model . GetPushMirrorByID ( mirrorID )
2021-06-14 20:20:43 +03:00
if err != nil {
log . Error ( "GetPushMirrorByID [%d]: %v" , mirrorID , err )
return false
}
2022-05-20 17:08:52 +03:00
_ = m . GetRepository ( )
2021-06-14 20:20:43 +03:00
m . LastError = ""
2021-11-30 23:06:32 +03:00
ctx , _ , finished := process . GetManager ( ) . AddContext ( ctx , fmt . Sprintf ( "Syncing PushMirror %s/%s to %s" , m . Repo . OwnerName , m . Repo . Name , m . RemoteName ) )
defer finished ( )
2021-06-14 20:20:43 +03:00
log . Trace ( "SyncPushMirror [mirror: %d][repo: %-v]: Running Sync" , m . ID , m . Repo )
err = runPushSync ( ctx , m )
if err != nil {
log . Error ( "SyncPushMirror [mirror: %d][repo: %-v]: %v" , m . ID , m . Repo , err )
m . LastError = stripExitStatus . ReplaceAllLiteralString ( err . Error ( ) , "" )
}
m . LastUpdateUnix = timeutil . TimeStampNow ( )
2021-12-10 04:27:50 +03:00
if err := repo_model . UpdatePushMirror ( m ) ; err != nil {
2021-06-14 20:20:43 +03:00
log . Error ( "UpdatePushMirror [%d]: %v" , m . ID , err )
return false
}
log . Trace ( "SyncPushMirror [mirror: %d][repo: %-v]: Finished" , m . ID , m . Repo )
return err == nil
}
2021-12-10 04:27:50 +03:00
func runPushSync ( ctx context . Context , m * repo_model . PushMirror ) error {
2021-06-14 20:20:43 +03:00
timeout := time . Duration ( setting . Git . Timeout . Mirror ) * time . Second
performPush := func ( path string ) error {
2021-11-30 23:06:32 +03:00
remoteAddr , err := git . GetRemoteAddress ( ctx , path , m . RemoteName )
2021-06-14 20:20:43 +03:00
if err != nil {
log . Error ( "GetRemoteAddress(%s) Error %v" , path , err )
return errors . New ( "Unexpected error" )
}
if setting . LFS . StartServer {
log . Trace ( "SyncMirrors [repo: %-v]: syncing LFS objects..." , m . Repo )
2022-03-29 22:13:41 +03:00
gitRepo , err := git . OpenRepository ( ctx , path )
2021-06-14 20:20:43 +03:00
if err != nil {
log . Error ( "OpenRepository: %v" , err )
return errors . New ( "Unexpected error" )
}
defer gitRepo . Close ( )
2021-11-20 12:34:05 +03:00
endpoint := lfs . DetermineEndpoint ( remoteAddr . String ( ) , "" )
lfsClient := lfs . NewClient ( endpoint , nil )
if err := pushAllLFSObjects ( ctx , gitRepo , lfsClient ) ; err != nil {
2022-03-31 05:25:40 +03:00
return util . SanitizeErrorCredentialURLs ( err )
2021-06-14 20:20:43 +03:00
}
}
log . Trace ( "Pushing %s mirror[%d] remote %s" , path , m . ID , m . RemoteName )
2021-11-30 23:06:32 +03:00
if err := git . Push ( ctx , path , git . PushOptions {
2021-06-14 20:20:43 +03:00
Remote : m . RemoteName ,
Force : true ,
Mirror : true ,
Timeout : timeout ,
} ) ; err != nil {
log . Error ( "Error pushing %s mirror[%d] remote %s: %v" , path , m . ID , m . RemoteName , err )
2022-03-31 05:25:40 +03:00
return util . SanitizeErrorCredentialURLs ( err )
2021-06-14 20:20:43 +03:00
}
return nil
}
err := performPush ( m . Repo . RepoPath ( ) )
if err != nil {
return err
}
if m . Repo . HasWiki ( ) {
wikiPath := m . Repo . WikiPath ( )
2021-11-30 23:06:32 +03:00
_ , err := git . GetRemoteAddress ( ctx , wikiPath , m . RemoteName )
2021-06-14 20:20:43 +03:00
if err == nil {
err := performPush ( wikiPath )
if err != nil {
return err
}
} else {
log . Trace ( "Skipping wiki: No remote configured" )
}
}
return nil
}
2021-11-20 12:34:05 +03:00
func pushAllLFSObjects ( ctx context . Context , gitRepo * git . Repository , lfsClient lfs . Client ) error {
2021-06-14 20:20:43 +03:00
contentStore := lfs . NewContentStore ( )
pointerChan := make ( chan lfs . PointerBlob )
errChan := make ( chan error , 1 )
go lfs . SearchPointerBlobs ( ctx , gitRepo , pointerChan , errChan )
uploadObjects := func ( pointers [ ] lfs . Pointer ) error {
2021-11-20 12:34:05 +03:00
err := lfsClient . Upload ( ctx , pointers , func ( p lfs . Pointer , objectError error ) ( io . ReadCloser , error ) {
2021-06-14 20:20:43 +03:00
if objectError != nil {
return nil , objectError
}
content , err := contentStore . Get ( p )
if err != nil {
log . Error ( "Error reading LFS object %v: %v" , p , err )
}
return content , err
} )
if err != nil {
select {
case <- ctx . Done ( ) :
return nil
default :
}
}
return err
}
var batch [ ] lfs . Pointer
for pointerBlob := range pointerChan {
exists , err := contentStore . Exists ( pointerBlob . Pointer )
if err != nil {
log . Error ( "Error checking if LFS object %v exists: %v" , pointerBlob . Pointer , err )
return err
}
if ! exists {
log . Trace ( "Skipping missing LFS object %v" , pointerBlob . Pointer )
continue
}
batch = append ( batch , pointerBlob . Pointer )
2021-11-20 12:34:05 +03:00
if len ( batch ) >= lfsClient . BatchSize ( ) {
2021-06-14 20:20:43 +03:00
if err := uploadObjects ( batch ) ; err != nil {
return err
}
batch = nil
}
}
if len ( batch ) > 0 {
if err := uploadObjects ( batch ) ; err != nil {
return err
}
}
err , has := <- errChan
if has {
log . Error ( "Error enumerating LFS objects for repository: %v" , err )
return err
}
return nil
}