2016-08-31 02:18:33 +03:00
// Copyright 2016 The Gogs Authors. All rights reserved.
2018-09-07 05:06:09 +03:00
// Copyright 2018 The Gitea Authors. All rights reserved.
2016-08-31 02:18:33 +03:00
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package models
import (
"fmt"
2019-08-17 00:56:57 +03:00
"net/url"
2018-09-07 05:06:09 +03:00
"strings"
2016-08-31 02:18:33 +03:00
"time"
2018-08-23 09:15:07 +03:00
"code.gitea.io/gitea/modules/cache"
2019-03-27 12:33:00 +03:00
"code.gitea.io/gitea/modules/git"
2016-11-10 19:24:48 +03:00
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/process"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/sync"
2019-08-15 17:46:21 +03:00
"code.gitea.io/gitea/modules/timeutil"
2017-12-04 04:48:03 +03:00
"code.gitea.io/gitea/modules/util"
"github.com/Unknwon/com"
"github.com/go-xorm/xorm"
2019-05-29 08:49:08 +03:00
"github.com/mcuadros/go-version"
2016-08-31 02:18:33 +03:00
)
2016-11-26 03:30:21 +03:00
// MirrorQueue holds an UniqueQueue object of the mirror
2016-08-31 02:18:33 +03:00
var MirrorQueue = sync . NewUniqueQueue ( setting . Repository . MirrorQueueLength )
// Mirror represents mirror information of a repository.
type Mirror struct {
2017-01-06 18:14:33 +03:00
ID int64 ` xorm:"pk autoincr" `
RepoID int64 ` xorm:"INDEX" `
2016-08-31 02:18:33 +03:00
Repo * Repository ` xorm:"-" `
2017-04-08 18:27:26 +03:00
Interval time . Duration
EnablePrune bool ` xorm:"NOT NULL DEFAULT true" `
2016-08-31 02:18:33 +03:00
2019-08-15 17:46:21 +03:00
UpdatedUnix timeutil . TimeStamp ` xorm:"INDEX" `
NextUpdateUnix timeutil . TimeStamp ` xorm:"INDEX" `
2016-08-31 02:18:33 +03:00
address string ` xorm:"-" `
}
2016-11-26 03:30:21 +03:00
// BeforeInsert will be invoked by XORM before inserting a record
2016-08-31 02:18:33 +03:00
func ( m * Mirror ) BeforeInsert ( ) {
2017-09-13 08:18:22 +03:00
if m != nil {
2019-08-15 17:46:21 +03:00
m . UpdatedUnix = timeutil . TimeStampNow ( )
m . NextUpdateUnix = timeutil . TimeStampNow ( )
2017-09-13 08:18:22 +03:00
}
2016-08-31 02:18:33 +03:00
}
2017-10-01 19:52:35 +03:00
// AfterLoad is invoked from XORM after setting the values of all fields of this object.
func ( m * Mirror ) AfterLoad ( session * xorm . Session ) {
2017-09-13 08:18:22 +03:00
if m == nil {
return
}
2016-08-31 02:18:33 +03:00
var err error
2017-10-01 19:52:35 +03:00
m . Repo , err = getRepositoryByID ( session , m . RepoID )
if err != nil {
2019-04-02 10:48:31 +03:00
log . Error ( "getRepositoryByID[%d]: %v" , m . ID , err )
2016-08-31 02:18:33 +03:00
}
}
// ScheduleNextUpdate calculates and sets next update time.
func ( m * Mirror ) ScheduleNextUpdate ( ) {
2018-11-09 02:58:02 +03:00
if m . Interval != 0 {
2019-08-15 17:46:21 +03:00
m . NextUpdateUnix = timeutil . TimeStampNow ( ) . AddDuration ( m . Interval )
2018-11-09 02:58:02 +03:00
} else {
m . NextUpdateUnix = 0
}
2016-08-31 02:18:33 +03:00
}
2017-12-03 08:29:41 +03:00
func remoteAddress ( repoPath string ) ( string , error ) {
2019-05-29 08:49:08 +03:00
var cmd * git . Command
binVersion , err := git . BinVersion ( )
if err != nil {
return "" , err
}
if version . Compare ( binVersion , "2.7" , ">=" ) {
cmd = git . NewCommand ( "remote" , "get-url" , "origin" )
} else {
cmd = git . NewCommand ( "config" , "--get" , "remote.origin.url" )
}
2019-04-12 23:52:57 +03:00
result , err := cmd . RunInDir ( repoPath )
2017-12-03 08:29:41 +03:00
if err != nil {
2019-04-12 23:52:57 +03:00
if strings . HasPrefix ( err . Error ( ) , "exit status 128 - fatal: No such remote " ) {
return "" , nil
}
2017-12-03 08:29:41 +03:00
return "" , err
}
2019-04-12 23:52:57 +03:00
if len ( result ) > 0 {
return result [ : len ( result ) - 1 ] , nil
}
return "" , nil
2017-12-03 08:29:41 +03:00
}
2016-08-31 02:18:33 +03:00
func ( m * Mirror ) readAddress ( ) {
if len ( m . address ) > 0 {
return
}
2017-12-03 08:29:41 +03:00
var err error
m . address , err = remoteAddress ( m . Repo . RepoPath ( ) )
2016-08-31 02:18:33 +03:00
if err != nil {
2019-04-02 10:48:31 +03:00
log . Error ( "remoteAddress: %v" , err )
2016-08-31 02:18:33 +03:00
}
}
2017-12-03 08:29:41 +03:00
// sanitizeOutput sanitizes output of a command, replacing occurrences of the
// repository's remote address with a sanitized version.
func sanitizeOutput ( output , repoPath string ) ( string , error ) {
remoteAddr , err := remoteAddress ( repoPath )
if err != nil {
// if we're unable to load the remote address, then we're unable to
// sanitize.
return "" , err
}
2017-12-04 04:48:03 +03:00
return util . SanitizeMessage ( output , remoteAddr ) , nil
2017-12-03 08:29:41 +03:00
}
2019-08-17 00:56:57 +03:00
// Address returns mirror address from Git repository config with credentials censored.
2016-08-31 02:18:33 +03:00
func ( m * Mirror ) Address ( ) string {
m . readAddress ( )
2017-12-04 04:48:03 +03:00
return util . SanitizeURLCredentials ( m . address , false )
2016-08-31 02:18:33 +03:00
}
// FullAddress returns mirror address from Git repository config.
func ( m * Mirror ) FullAddress ( ) string {
m . readAddress ( )
return m . address
}
2019-08-17 00:56:57 +03:00
// AddressNoCredentials returns mirror address from Git repository config without credentials.
func ( m * Mirror ) AddressNoCredentials ( ) string {
m . readAddress ( )
u , err := url . Parse ( m . address )
if err != nil {
// this shouldn't happen but just return it unsanitised
return m . address
}
u . User = nil
return u . String ( )
}
// Username returns the mirror address username
func ( m * Mirror ) Username ( ) string {
m . readAddress ( )
u , err := url . Parse ( m . address )
if err != nil {
// this shouldn't happen but if it does return ""
return ""
}
return u . User . Username ( )
}
// Password returns the mirror address password
func ( m * Mirror ) Password ( ) string {
m . readAddress ( )
u , err := url . Parse ( m . address )
if err != nil {
// this shouldn't happen but if it does return ""
return ""
}
password , _ := u . User . Password ( )
return password
}
2016-08-31 02:18:33 +03:00
// SaveAddress writes new address to Git repository config.
func ( m * Mirror ) SaveAddress ( addr string ) error {
2019-04-12 23:52:57 +03:00
repoPath := m . Repo . RepoPath ( )
// Remove old origin
_ , err := git . NewCommand ( "remote" , "remove" , "origin" ) . RunInDir ( repoPath )
if err != nil && ! strings . HasPrefix ( err . Error ( ) , "exit status 128 - fatal: No such remote " ) {
return err
2016-08-31 02:18:33 +03:00
}
2019-05-02 09:43:20 +03:00
_ , err = git . NewCommand ( "remote" , "add" , "origin" , "--mirror=fetch" , addr ) . RunInDir ( repoPath )
2019-04-12 23:52:57 +03:00
return err
2016-08-31 02:18:33 +03:00
}
2018-09-07 05:06:09 +03:00
// gitShortEmptySha Git short empty SHA
const gitShortEmptySha = "0000000"
// mirrorSyncResult contains information of a updated reference.
// If the oldCommitID is "0000000", it means a new reference, the value of newCommitID is empty.
// If the newCommitID is "0000000", it means the reference is deleted, the value of oldCommitID is empty.
type mirrorSyncResult struct {
refName string
oldCommitID string
newCommitID string
}
// parseRemoteUpdateOutput detects create, update and delete operations of references from upstream.
func parseRemoteUpdateOutput ( output string ) [ ] * mirrorSyncResult {
results := make ( [ ] * mirrorSyncResult , 0 , 3 )
lines := strings . Split ( output , "\n" )
for i := range lines {
// Make sure reference name is presented before continue
idx := strings . Index ( lines [ i ] , "-> " )
if idx == - 1 {
continue
}
refName := lines [ i ] [ idx + 3 : ]
switch {
case strings . HasPrefix ( lines [ i ] , " * " ) : // New reference
results = append ( results , & mirrorSyncResult {
refName : refName ,
oldCommitID : gitShortEmptySha ,
} )
case strings . HasPrefix ( lines [ i ] , " - " ) : // Delete reference
results = append ( results , & mirrorSyncResult {
refName : refName ,
newCommitID : gitShortEmptySha ,
} )
case strings . HasPrefix ( lines [ i ] , " " ) : // New commits of a reference
delimIdx := strings . Index ( lines [ i ] [ 3 : ] , " " )
if delimIdx == - 1 {
2019-04-02 10:48:31 +03:00
log . Error ( "SHA delimiter not found: %q" , lines [ i ] )
2018-09-07 05:06:09 +03:00
continue
}
shas := strings . Split ( lines [ i ] [ 3 : delimIdx + 3 ] , ".." )
if len ( shas ) != 2 {
2019-04-02 10:48:31 +03:00
log . Error ( "Expect two SHAs but not what found: %q" , lines [ i ] )
2018-09-07 05:06:09 +03:00
continue
}
results = append ( results , & mirrorSyncResult {
refName : refName ,
oldCommitID : shas [ 0 ] ,
newCommitID : shas [ 1 ] ,
} )
default :
log . Warn ( "parseRemoteUpdateOutput: unexpected update line %q" , lines [ i ] )
}
}
return results
}
2016-08-31 02:18:33 +03:00
// runSync returns true if sync finished without error.
2018-09-07 05:06:09 +03:00
func ( m * Mirror ) runSync ( ) ( [ ] * mirrorSyncResult , bool ) {
2016-08-31 02:18:33 +03:00
repoPath := m . Repo . RepoPath ( )
wikiPath := m . Repo . WikiPath ( )
timeout := time . Duration ( setting . Git . Timeout . Mirror ) * time . Second
gitArgs := [ ] string { "remote" , "update" }
if m . EnablePrune {
gitArgs = append ( gitArgs , "--prune" )
}
2018-09-07 05:06:09 +03:00
_ , stderr , err := process . GetManager ( ) . ExecDir (
2016-08-31 14:31:53 +03:00
timeout , repoPath , fmt . Sprintf ( "Mirror.runSync: %s" , repoPath ) ,
2019-06-26 21:15:26 +03:00
git . GitExecutable , gitArgs ... )
2018-09-07 05:06:09 +03:00
if err != nil {
2017-12-03 08:29:41 +03:00
// sanitize the output, since it may contain the remote address, which may
// contain a password
message , err := sanitizeOutput ( stderr , repoPath )
if err != nil {
2019-04-02 10:48:31 +03:00
log . Error ( "sanitizeOutput: %v" , err )
2018-09-07 05:06:09 +03:00
return nil , false
2017-12-03 08:29:41 +03:00
}
desc := fmt . Sprintf ( "Failed to update mirror repository '%s': %s" , repoPath , message )
2019-04-02 10:48:31 +03:00
log . Error ( desc )
2016-08-31 02:18:33 +03:00
if err = CreateRepositoryNotice ( desc ) ; err != nil {
2019-04-02 10:48:31 +03:00
log . Error ( "CreateRepositoryNotice: %v" , err )
2016-08-31 02:18:33 +03:00
}
2018-09-07 05:06:09 +03:00
return nil , false
2016-08-31 02:18:33 +03:00
}
2018-09-07 05:06:09 +03:00
output := stderr
2017-04-11 16:30:15 +03:00
2017-09-20 08:26:49 +03:00
gitRepo , err := git . OpenRepository ( repoPath )
if err != nil {
2019-04-02 10:48:31 +03:00
log . Error ( "OpenRepository: %v" , err )
2018-09-07 05:06:09 +03:00
return nil , false
2017-09-20 08:26:49 +03:00
}
if err = SyncReleasesWithTags ( m . Repo , gitRepo ) ; err != nil {
2019-04-02 10:48:31 +03:00
log . Error ( "Failed to synchronize tags to releases for repository: %v" , err )
2017-09-20 08:26:49 +03:00
}
2017-04-11 16:30:15 +03:00
if err := m . Repo . UpdateSize ( ) ; err != nil {
2019-04-02 10:48:31 +03:00
log . Error ( "Failed to update size for mirror repository: %v" , err )
2017-04-11 16:30:15 +03:00
}
2016-08-31 02:18:33 +03:00
if m . Repo . HasWiki ( ) {
2017-01-17 08:58:58 +03:00
if _ , stderr , err := process . GetManager ( ) . ExecDir (
2016-08-31 14:31:53 +03:00
timeout , wikiPath , fmt . Sprintf ( "Mirror.runSync: %s" , wikiPath ) ,
2019-06-26 21:15:26 +03:00
git . GitExecutable , "remote" , "update" , "--prune" ) ; err != nil {
2017-12-03 08:29:41 +03:00
// sanitize the output, since it may contain the remote address, which may
// contain a password
message , err := sanitizeOutput ( stderr , wikiPath )
if err != nil {
2019-04-02 10:48:31 +03:00
log . Error ( "sanitizeOutput: %v" , err )
2018-09-07 05:06:09 +03:00
return nil , false
2017-12-03 08:29:41 +03:00
}
desc := fmt . Sprintf ( "Failed to update mirror wiki repository '%s': %s" , wikiPath , message )
2019-04-02 10:48:31 +03:00
log . Error ( desc )
2016-08-31 02:18:33 +03:00
if err = CreateRepositoryNotice ( desc ) ; err != nil {
2019-04-02 10:48:31 +03:00
log . Error ( "CreateRepositoryNotice: %v" , err )
2016-08-31 02:18:33 +03:00
}
2018-09-07 05:06:09 +03:00
return nil , false
2016-08-31 02:18:33 +03:00
}
}
2018-08-23 09:15:07 +03:00
branches , err := m . Repo . GetBranches ( )
if err != nil {
2019-04-02 10:48:31 +03:00
log . Error ( "GetBranches: %v" , err )
2018-09-07 05:06:09 +03:00
return nil , false
2018-08-23 09:15:07 +03:00
}
for i := range branches {
cache . Remove ( m . Repo . GetCommitsCountCacheKey ( branches [ i ] . Name , true ) )
}
2019-08-15 17:46:21 +03:00
m . UpdatedUnix = timeutil . TimeStampNow ( )
2018-09-07 05:06:09 +03:00
return parseRemoteUpdateOutput ( output ) , true
2016-08-31 02:18:33 +03:00
}
func getMirrorByRepoID ( e Engine , repoID int64 ) ( * Mirror , error ) {
m := & Mirror { RepoID : repoID }
has , err := e . Get ( m )
if err != nil {
return nil , err
} else if ! has {
return nil , ErrMirrorNotExist
}
return m , nil
}
// GetMirrorByRepoID returns mirror information of a repository.
func GetMirrorByRepoID ( repoID int64 ) ( * Mirror , error ) {
return getMirrorByRepoID ( x , repoID )
}
func updateMirror ( e Engine , m * Mirror ) error {
2017-10-05 07:43:04 +03:00
_ , err := e . ID ( m . ID ) . AllCols ( ) . Update ( m )
2016-08-31 02:18:33 +03:00
return err
}
2016-11-26 03:30:21 +03:00
// UpdateMirror updates the mirror
2016-08-31 02:18:33 +03:00
func UpdateMirror ( m * Mirror ) error {
return updateMirror ( x , m )
}
2016-11-26 03:30:21 +03:00
// DeleteMirrorByRepoID deletes a mirror by repoID
2016-08-31 02:18:33 +03:00
func DeleteMirrorByRepoID ( repoID int64 ) error {
_ , err := x . Delete ( & Mirror { RepoID : repoID } )
return err
}
// MirrorUpdate checks and updates mirror repositories.
func MirrorUpdate ( ) {
log . Trace ( "Doing: MirrorUpdate" )
2016-11-10 18:16:32 +03:00
if err := x .
Where ( "next_update_unix<=?" , time . Now ( ) . Unix ( ) ) .
2018-11-09 02:58:02 +03:00
And ( "next_update_unix!=0" ) .
2016-11-10 18:16:32 +03:00
Iterate ( new ( Mirror ) , func ( idx int , bean interface { } ) error {
m := bean . ( * Mirror )
if m . Repo == nil {
2019-04-02 10:48:31 +03:00
log . Error ( "Disconnected mirror repository found: %d" , m . ID )
2016-11-10 18:16:32 +03:00
return nil
}
2016-08-31 02:18:33 +03:00
2016-11-10 18:16:32 +03:00
MirrorQueue . Add ( m . RepoID )
return nil
} ) ; err != nil {
2019-04-02 10:48:31 +03:00
log . Error ( "MirrorUpdate: %v" , err )
2016-08-31 02:18:33 +03:00
}
}
// SyncMirrors checks and syncs mirrors.
// TODO: sync more mirrors at same time.
func SyncMirrors ( ) {
2018-02-23 03:27:09 +03:00
sess := x . NewSession ( )
defer sess . Close ( )
2016-08-31 02:18:33 +03:00
// Start listening on new sync requests.
for repoID := range MirrorQueue . Queue ( ) {
log . Trace ( "SyncMirrors [repo_id: %v]" , repoID )
MirrorQueue . Remove ( repoID )
m , err := GetMirrorByRepoID ( com . StrTo ( repoID ) . MustInt64 ( ) )
if err != nil {
2019-04-02 10:48:31 +03:00
log . Error ( "GetMirrorByRepoID [%s]: %v" , repoID , err )
2016-08-31 02:18:33 +03:00
continue
}
2018-09-07 05:06:09 +03:00
results , ok := m . runSync ( )
if ! ok {
2016-08-31 02:18:33 +03:00
continue
}
m . ScheduleNextUpdate ( )
2018-02-23 03:27:09 +03:00
if err = updateMirror ( sess , m ) ; err != nil {
2019-04-02 10:48:31 +03:00
log . Error ( "UpdateMirror [%s]: %v" , repoID , err )
2016-08-31 02:18:33 +03:00
continue
}
2018-02-23 03:27:09 +03:00
2018-09-07 05:06:09 +03:00
var gitRepo * git . Repository
if len ( results ) == 0 {
log . Trace ( "SyncMirrors [repo_id: %d]: no commits fetched" , m . RepoID )
} else {
gitRepo , err = git . OpenRepository ( m . Repo . RepoPath ( ) )
if err != nil {
2019-04-02 10:48:31 +03:00
log . Error ( "OpenRepository [%d]: %v" , m . RepoID , err )
2018-09-07 05:06:09 +03:00
continue
}
}
for _ , result := range results {
// Discard GitHub pull requests, i.e. refs/pull/*
if strings . HasPrefix ( result . refName , "refs/pull/" ) {
continue
}
// Create reference
if result . oldCommitID == gitShortEmptySha {
if err = MirrorSyncCreateAction ( m . Repo , result . refName ) ; err != nil {
2019-04-02 10:48:31 +03:00
log . Error ( "MirrorSyncCreateAction [repo_id: %d]: %v" , m . RepoID , err )
2018-09-07 05:06:09 +03:00
}
continue
}
// Delete reference
if result . newCommitID == gitShortEmptySha {
if err = MirrorSyncDeleteAction ( m . Repo , result . refName ) ; err != nil {
2019-04-02 10:48:31 +03:00
log . Error ( "MirrorSyncDeleteAction [repo_id: %d]: %v" , m . RepoID , err )
2018-09-07 05:06:09 +03:00
}
continue
}
// Push commits
oldCommitID , err := git . GetFullCommitID ( gitRepo . Path , result . oldCommitID )
if err != nil {
2019-04-02 10:48:31 +03:00
log . Error ( "GetFullCommitID [%d]: %v" , m . RepoID , err )
2018-09-07 05:06:09 +03:00
continue
}
newCommitID , err := git . GetFullCommitID ( gitRepo . Path , result . newCommitID )
if err != nil {
2019-04-02 10:48:31 +03:00
log . Error ( "GetFullCommitID [%d]: %v" , m . RepoID , err )
2018-09-07 05:06:09 +03:00
continue
}
commits , err := gitRepo . CommitsBetweenIDs ( newCommitID , oldCommitID )
if err != nil {
2019-04-02 10:48:31 +03:00
log . Error ( "CommitsBetweenIDs [repo_id: %d, new_commit_id: %s, old_commit_id: %s]: %v" , m . RepoID , newCommitID , oldCommitID , err )
2018-09-07 05:06:09 +03:00
continue
}
if err = MirrorSyncPushAction ( m . Repo , MirrorSyncPushActionOptions {
RefName : result . refName ,
OldCommitID : oldCommitID ,
NewCommitID : newCommitID ,
Commits : ListToPushCommits ( commits ) ,
} ) ; err != nil {
2019-04-02 10:48:31 +03:00
log . Error ( "MirrorSyncPushAction [repo_id: %d]: %v" , m . RepoID , err )
2018-09-07 05:06:09 +03:00
continue
}
}
2018-02-23 03:27:09 +03:00
// Get latest commit date and update to current repository updated time
commitDate , err := git . GetLatestCommitTime ( m . Repo . RepoPath ( ) )
if err != nil {
2019-04-02 10:48:31 +03:00
log . Error ( "GetLatestCommitDate [%d]: %v" , m . RepoID , err )
2018-02-23 03:27:09 +03:00
continue
}
if _ , err = sess . Exec ( "UPDATE repository SET updated_unix = ? WHERE id = ?" , commitDate . Unix ( ) , m . RepoID ) ; err != nil {
2019-04-02 10:48:31 +03:00
log . Error ( "Update repository 'updated_unix' [%d]: %v" , m . RepoID , err )
2018-02-23 03:27:09 +03:00
continue
}
2016-08-31 02:18:33 +03:00
}
}
2017-01-05 03:50:34 +03:00
// InitSyncMirrors initializes a go routine to sync the mirrors
2016-08-31 02:18:33 +03:00
func InitSyncMirrors ( ) {
go SyncMirrors ( )
}