alt/zot-wo-auth
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(sync): fixed broken logic to get tags for repo (#900)

Browse Source 
Signed-off-by: Petu Eusebiu <peusebiu@cisco.com>
This commit is contained in:
peusebiu 2022-10-22 10:26:14 +03:00 committed by GitHub
parent 26d982becb
commit 5f99f9a445
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 364 additions and 410 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

70
pkg/extensions/sync/on_demand.go
View File

@ -3,7 +3,6 @@ package sync
import (
"context"
"fmt"
"net/url"
"os"
"sync"
"time"
@ -13,21 +12,16 @@ import (
"github.com/containers/image/v5/docker"
"github.com/containers/image/v5/signature"
"github.com/containers/image/v5/types"
"gopkg.in/resty.v1"
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/storage"
)
type syncContextUtils struct {
imageStore storage.ImageStore
policyCtx *signature.PolicyContext
localCtx *types.SystemContext
upstreamCtx *types.SystemContext
client *resty.Client
url *url.URL
upstreamAddr string
retryOptions *retry.RetryOptions
copyOptions copy.Options
}
@ -115,8 +109,6 @@ func syncOneImage(imageChannel chan error, cfg Config, storeController storage.S
return
}
imageStore := storeController.GetImageStore(localRepo)
for _, registryCfg := range cfg.Registries {
regCfg := registryCfg
if !regCfg.OnDemand {
@ -125,7 +117,7 @@ func syncOneImage(imageChannel chan error, cfg Config, storeController storage.S
continue
}
remoteRepo := localRepo
upstreamRepo := localRepo
// if content config is not specified, then don't filter, just sync demanded image
if len(regCfg.Content) != 0 {
@ -137,7 +129,7 @@ func syncOneImage(imageChannel chan error, cfg Config, storeController storage.S
continue
}
remoteRepo = getRepoSource(localRepo, regCfg.Content[contentID])
upstreamRepo = getRepoSource(localRepo, regCfg.Content[contentID])
}
retryOptions := &retry.RetryOptions{}
@ -163,7 +155,8 @@ func syncOneImage(imageChannel chan error, cfg Config, storeController storage.S
return
}
// it's an image
sig := newSignaturesCopier(httpClient, *registryURL, storeController, log)
upstreamCtx := getUpstreamContext(&regCfg, credentialsFile[upstreamAddr])
options := getCopyOptions(upstreamCtx, localCtx)
@ -171,18 +164,18 @@ func syncOneImage(imageChannel chan error, cfg Config, storeController storage.S
if isCosignTag(tag) {
// at tis point we should already have images synced, but not their signatures.
// is cosign signature
cosignManifest, err := getCosignManifest(httpClient, *registryURL, remoteRepo, tag, log)
cosignManifest, err := sig.getCosignManifest(upstreamRepo, tag)
if err != nil {
log.Error().Str("errorType", TypeOf(err)).
Err(err).Msgf("couldn't get upstream image %s:%s:%s cosign manifest", upstreamURL, remoteRepo, tag)
Err(err).Msgf("couldn't get upstream image %s:%s:%s cosign manifest", upstreamURL, upstreamRepo, tag)
continue
}
err = syncCosignSignature(httpClient, imageStore, *registryURL, localRepo, remoteRepo, tag, cosignManifest, log)
err = sig.syncCosignSignature(localRepo, upstreamRepo, tag, cosignManifest)
if err != nil {
log.Error().Str("errorType", TypeOf(err)).
Err(err).Msgf("couldn't copy upstream image cosign signature %s/%s:%s", upstreamURL, remoteRepo, tag)
Err(err).Msgf("couldn't copy upstream image cosign signature %s/%s:%s", upstreamURL, upstreamRepo, tag)
continue
}
@ -192,18 +185,18 @@ func syncOneImage(imageChannel chan error, cfg Config, storeController storage.S
return
} else if isArtifact {
// is notary signature
refs, err := getNotaryRefs(httpClient, *registryURL, remoteRepo, tag, log)
refs, err := sig.getNotaryRefs(upstreamRepo, tag)
if err != nil {
log.Error().Str("errorType", TypeOf(err)).
Err(err).Msgf("couldn't get upstream image %s/%s:%s notary references", upstreamURL, remoteRepo, tag)
Err(err).Msgf("couldn't get upstream image %s/%s:%s notary references", upstreamURL, upstreamRepo, tag)
continue
}
err = syncNotarySignature(httpClient, imageStore, *registryURL, localRepo, remoteRepo, tag, refs, log)
err = sig.syncNotarySignature(localRepo, upstreamRepo, tag, refs)
if err != nil {
log.Error().Str("errorType", TypeOf(err)).
Err(err).Msgf("couldn't copy image signature %s/%s:%s", upstreamURL, remoteRepo, tag)
Err(err).Msgf("couldn't copy image signature %s/%s:%s", upstreamURL, upstreamRepo, tag)
continue
}
@ -214,24 +207,20 @@ func syncOneImage(imageChannel chan error, cfg Config, storeController storage.S
}
syncContextUtils := syncContextUtils{
imageStore: imageStore,
policyCtx: policyCtx,
localCtx: localCtx,
upstreamCtx: upstreamCtx,
client: httpClient,
url: registryURL,
upstreamAddr: upstreamAddr,
retryOptions: retryOptions,
copyOptions: options,
}
skipped, copyErr := syncRun(regCfg, localRepo, remoteRepo, tag, syncContextUtils, log)
skipped, copyErr := syncRun(regCfg, localRepo, upstreamRepo, tag, syncContextUtils, sig, log)
if skipped {
continue
}
// key used to check if we already have a go routine syncing this image
demandedImageRef := fmt.Sprintf("%s/%s:%s", upstreamAddr, remoteRepo, tag)
demandedImageRef := fmt.Sprintf("%s/%s:%s", upstreamAddr, upstreamRepo, tag)
if copyErr != nil {
// don't retry in background if maxretry is 0
@ -260,7 +249,7 @@ func syncOneImage(imageChannel chan error, cfg Config, storeController storage.S
time.Sleep(retryOptions.Delay)
if err = retry.RetryIfNecessary(context.Background(), func() error {
_, err := syncRun(regCfg, localRepo, remoteRepo, tag, syncContextUtils, log)
_, err := syncRun(regCfg, localRepo, upstreamRepo, tag, syncContextUtils, sig, log)
return err
}, retryOptions); err != nil {
@ -276,14 +265,14 @@ func syncOneImage(imageChannel chan error, cfg Config, storeController storage.S
}
func syncRun(regCfg RegistryConfig,
localRepo, remoteRepo, tag string, utils syncContextUtils,
localRepo, upstreamRepo, tag string, utils syncContextUtils, sig *signaturesCopier,
log log.Logger,
) (bool, error) {
upstreamImageRef, err := getImageRef(utils.upstreamAddr, remoteRepo, tag)
upstreamImageRef, err := getImageRef(utils.upstreamAddr, upstreamRepo, tag)
if err != nil {
log.Error().Str("errorType", TypeOf(err)).
Err(err).Msgf("error creating docker reference for repository %s/%s:%s",
utils.upstreamAddr, remoteRepo, tag)
utils.upstreamAddr, upstreamRepo, tag)
return false, err
}
@ -297,14 +286,13 @@ func syncRun(regCfg RegistryConfig,
}
// get upstream signatures
cosignManifest, err := getCosignManifest(utils.client, *utils.url, remoteRepo,
upstreamImageDigest.String(), log)
cosignManifest, err := sig.getCosignManifest(upstreamRepo, upstreamImageDigest.String())
if err != nil {
log.Error().Str("errorType", TypeOf(err)).
Err(err).Msgf("couldn't get upstream image %s cosign manifest", upstreamImageRef.DockerReference())
}
refs, err := getNotaryRefs(utils.client, *utils.url, remoteRepo, upstreamImageDigest.String(), log)
refs, err := sig.getNotaryRefs(upstreamRepo, upstreamImageDigest.String())
if err != nil {
log.Error().Str("errorType", TypeOf(err)).
Err(err).Msgf("couldn't get upstream image %s notary references", upstreamImageRef.DockerReference())
@ -321,7 +309,9 @@ func syncRun(regCfg RegistryConfig,
}
}
localCachePath, err := getLocalCachePath(utils.imageStore, localRepo)
imageStore := sig.storeController.GetImageStore(localRepo)
localCachePath, err := getLocalCachePath(imageStore, localRepo)
if err != nil {
log.Error().Err(err).Msgf("couldn't get localCachePath for %s", localRepo)
}
@ -348,7 +338,7 @@ func syncRun(regCfg RegistryConfig,
return false, err
}
err = pushSyncedLocalImage(localRepo, tag, localCachePath, utils.imageStore, log)
err = pushSyncedLocalImage(localRepo, tag, localCachePath, imageStore, log)
if err != nil {
log.Error().Str("errorType", TypeOf(err)).
Err(err).Msgf("error while pushing synced cached image %s",
@ -357,25 +347,23 @@ func syncRun(regCfg RegistryConfig,
return false, err
}
err = syncCosignSignature(utils.client, utils.imageStore, *utils.url, localRepo, remoteRepo,
upstreamImageDigest.String(), cosignManifest, log)
err = sig.syncCosignSignature(localRepo, upstreamRepo, upstreamImageDigest.String(), cosignManifest)
if err != nil {
log.Error().Str("errorType", TypeOf(err)).
Err(err).Msgf("couldn't copy image cosign signature %s/%s:%s", utils.upstreamAddr, remoteRepo, tag)
Err(err).Msgf("couldn't copy image cosign signature %s/%s:%s", utils.upstreamAddr, upstreamRepo, tag)
return false, err
}
err = syncNotarySignature(utils.client, utils.imageStore, *utils.url, localRepo, remoteRepo,
upstreamImageDigest.String(), refs, log)
err = sig.syncNotarySignature(localRepo, upstreamRepo, upstreamImageDigest.String(), refs)
if err != nil {
log.Error().Str("errorType", TypeOf(err)).
Err(err).Msgf("couldn't copy image notary signature %s/%s:%s", utils.upstreamAddr, remoteRepo, tag)
Err(err).Msgf("couldn't copy image notary signature %s/%s:%s", utils.upstreamAddr, upstreamRepo, tag)
return false, err
}
log.Info().Msgf("successfully synced %s/%s:%s", utils.upstreamAddr, remoteRepo, tag)
log.Info().Msgf("successfully synced %s/%s:%s", utils.upstreamAddr, upstreamRepo, tag)
return false, nil
}

164
pkg/extensions/sync/signatures.go
View File

@ -20,24 +20,40 @@ import (
"zotregistry.io/zot/pkg/storage"
)
func getCosignManifest(client *resty.Client, regURL url.URL, repo, digest string,
log log.Logger,
) (*ispec.Manifest, error) {
type signaturesCopier struct {
client *resty.Client
upstreamURL url.URL
storeController storage.StoreController
log log.Logger
}
func newSignaturesCopier(httpClient *resty.Client, upstreamURL url.URL,
storeController storage.StoreController, log log.Logger,
) *signaturesCopier {
return &signaturesCopier{
client: httpClient,
upstreamURL: upstreamURL,
storeController: storeController,
log: log,
}
}
func (sig *signaturesCopier) getCosignManifest(repo, digest string) (*ispec.Manifest, error) {
var cosignManifest ispec.Manifest
cosignTag := getCosignTagFromImageDigest(digest)
getCosignManifestURL := regURL
getCosignManifestURL := sig.upstreamURL
getCosignManifestURL.Path = path.Join(getCosignManifestURL.Path, "v2", repo, "manifests", cosignTag)
getCosignManifestURL.RawQuery = getCosignManifestURL.Query().Encode()
resp, err := client.R().
resp, err := sig.client.R().
SetHeader("Content-Type", "application/vnd.oci.image.manifest.v1+json").
Get(getCosignManifestURL.String())
if err != nil {
log.Error().Str("errorType", TypeOf(err)).
sig.log.Error().Str("errorType", TypeOf(err)).
Err(err).Str("url", getCosignManifestURL.String()).
Msgf("couldn't get cosign manifest: %s", cosignTag)
@ -45,12 +61,12 @@ func getCosignManifest(client *resty.Client, regURL url.URL, repo, digest string
}
if resp.StatusCode() == http.StatusNotFound {
log.Info().Msgf("couldn't find any cosign signature from %s, status code: %d skipping",
sig.log.Info().Msgf("couldn't find any cosign signature from %s, status code: %d skipping",
getCosignManifestURL.String(), resp.StatusCode())
return nil, zerr.ErrSyncSignatureNotFound
} else if resp.IsError() {
log.Error().Str("errorType", TypeOf(zerr.ErrSyncSignature)).
sig.log.Error().Str("errorType", TypeOf(zerr.ErrSyncSignature)).
Err(zerr.ErrSyncSignature).Msgf("couldn't get cosign signature from %s, status code: %d skipping",
getCosignManifestURL.String(), resp.StatusCode())
@ -59,7 +75,7 @@ func getCosignManifest(client *resty.Client, regURL url.URL, repo, digest string
err = json.Unmarshal(resp.Body(), &cosignManifest)
if err != nil {
log.Error().Str("errorType", TypeOf(err)).
sig.log.Error().Str("errorType", TypeOf(err)).
Err(err).Str("url", getCosignManifestURL.String()).
Msgf("couldn't unmarshal cosign manifest %s", cosignTag)
@ -69,10 +85,10 @@ func getCosignManifest(client *resty.Client, regURL url.URL, repo, digest string
return &cosignManifest, nil
}
func getNotaryRefs(client *resty.Client, regURL url.URL, repo, digest string, log log.Logger) (ReferenceList, error) {
func (sig *signaturesCopier) getNotaryRefs(repo, digest string) (ReferenceList, error) {
var referrers ReferenceList
getReferrersURL := regURL
getReferrersURL := sig.upstreamURL
// based on manifest digest get referrers
getReferrersURL.Path = path.Join(getReferrersURL.Path, constants.ArtifactSpecRoutePrefix,
@ -80,24 +96,24 @@ func getNotaryRefs(client *resty.Client, regURL url.URL, repo, digest string, lo
getReferrersURL.RawQuery = getReferrersURL.Query().Encode()
resp, err := client.R().
resp, err := sig.client.R().
SetHeader("Content-Type", "application/json").
SetQueryParam("artifactType", notreg.ArtifactTypeNotation).
Get(getReferrersURL.String())
if err != nil {
log.Error().Str("errorType", TypeOf(err)).
sig.log.Error().Str("errorType", TypeOf(err)).
Err(err).Str("url", getReferrersURL.String()).Msg("couldn't get referrers")
return referrers, err
}
if resp.StatusCode() == http.StatusNotFound || resp.StatusCode() == http.StatusBadRequest {
log.Info().Msgf("couldn't find any notary signature from %s, status code: %d, skipping",
sig.log.Info().Msgf("couldn't find any notary signature from %s, status code: %d, skipping",
getReferrersURL.String(), resp.StatusCode())
return ReferenceList{}, zerr.ErrSyncSignatureNotFound
} else if resp.IsError() {
log.Error().Str("errorType", TypeOf(zerr.ErrSyncSignature)).
sig.log.Error().Str("errorType", TypeOf(zerr.ErrSyncSignature)).
Err(zerr.ErrSyncSignature).Msgf("couldn't get notary signature from %s, status code: %d skipping",
getReferrersURL.String(), resp.StatusCode())
@ -106,7 +122,7 @@ func getNotaryRefs(client *resty.Client, regURL url.URL, repo, digest string, lo
err = json.Unmarshal(resp.Body(), &referrers)
if err != nil {
log.Error().Str("errorType", TypeOf(err)).
sig.log.Error().Str("errorType", TypeOf(err)).
Err(err).Str("url", getReferrersURL.String()).
Msgf("couldn't unmarshal notary signature")
@ -116,8 +132,7 @@ func getNotaryRefs(client *resty.Client, regURL url.URL, repo, digest string, lo
return referrers, nil
}
func syncCosignSignature(client *resty.Client, imageStore storage.ImageStore,
regURL url.URL, localRepo, remoteRepo, digest string, cosignManifest *ispec.Manifest, log log.Logger,
func (sig *signaturesCopier) syncCosignSignature(localRepo, remoteRepo, digest string, cosignManifest *ispec.Manifest,
) error {
cosignTag := getCosignTagFromImageDigest(digest)
@ -126,24 +141,36 @@ func syncCosignSignature(client *resty.Client, imageStore storage.ImageStore,
return nil
}
log.Info().Msg("syncing cosign signatures")
skipCosignSig, err := sig.canSkipCosignSignature(localRepo, digest, cosignManifest)
if err != nil {
sig.log.Error().Err(err).Msgf("couldn't check if the upstream image %s:%s cosign signature can be skipped",
remoteRepo, digest)
}
if skipCosignSig {
return nil
}
imageStore := sig.storeController.GetImageStore(localRepo)
sig.log.Info().Msg("syncing cosign signatures")
for _, blob := range cosignManifest.Layers {
// get blob
getBlobURL := regURL
getBlobURL := sig.upstreamURL
getBlobURL.Path = path.Join(getBlobURL.Path, "v2", remoteRepo, "blobs", blob.Digest.String())
getBlobURL.RawQuery = getBlobURL.Query().Encode()
resp, err := client.R().SetDoNotParseResponse(true).Get(getBlobURL.String())
resp, err := sig.client.R().SetDoNotParseResponse(true).Get(getBlobURL.String())
if err != nil {
log.Error().Str("errorType", TypeOf(err)).
sig.log.Error().Str("errorType", TypeOf(err)).
Err(err).Msgf("couldn't get cosign blob: %s", blob.Digest.String())
return err
}
if resp.IsError() {
log.Info().Msgf("couldn't find cosign blob from %s, status code: %d", getBlobURL.String(), resp.StatusCode())
sig.log.Info().Msgf("couldn't find cosign blob from %s, status code: %d", getBlobURL.String(), resp.StatusCode())
return zerr.ErrSyncSignature
}
@ -153,7 +180,7 @@ func syncCosignSignature(client *resty.Client, imageStore storage.ImageStore,
// push blob
_, _, err = imageStore.FullBlobUpload(localRepo, resp.RawBody(), blob.Digest.String())
if err != nil {
log.Error().Str("errorType", TypeOf(err)).
sig.log.Error().Str("errorType", TypeOf(err)).
Err(err).Msg("couldn't upload cosign blob")
return err
@ -161,20 +188,21 @@ func syncCosignSignature(client *resty.Client, imageStore storage.ImageStore,
}
// get config blob
getBlobURL := regURL
getBlobURL := sig.upstreamURL
getBlobURL.Path = path.Join(getBlobURL.Path, "v2", remoteRepo, "blobs", cosignManifest.Config.Digest.String())
getBlobURL.RawQuery = getBlobURL.Query().Encode()
resp, err := client.R().SetDoNotParseResponse(true).Get(getBlobURL.String())
resp, err := sig.client.R().SetDoNotParseResponse(true).Get(getBlobURL.String())
if err != nil {
log.Error().Str("errorType", TypeOf(err)).
sig.log.Error().Str("errorType", TypeOf(err)).
Err(err).Msgf("couldn't get cosign config blob: %s", getBlobURL.String())
return err
}
if resp.IsError() {
log.Info().Msgf("couldn't find cosign config blob from %s, status code: %d", getBlobURL.String(), resp.StatusCode())
sig.log.Info().Msgf("couldn't find cosign config blob from %s, status code: %d",
getBlobURL.String(), resp.StatusCode())
return zerr.ErrSyncSignature
}
@ -184,7 +212,7 @@ func syncCosignSignature(client *resty.Client, imageStore storage.ImageStore,
// push config blob
_, _, err = imageStore.FullBlobUpload(localRepo, resp.RawBody(), cosignManifest.Config.Digest.String())
if err != nil {
log.Error().Str("errorType", TypeOf(err)).
sig.log.Error().Str("errorType", TypeOf(err)).
Err(err).Msg("couldn't upload cosign config blob")
return err
@ -192,7 +220,7 @@ func syncCosignSignature(client *resty.Client, imageStore storage.ImageStore,
cosignManifestBuf, err := json.Marshal(cosignManifest)
if err != nil {
log.Error().Str("errorType", TypeOf(err)).
sig.log.Error().Str("errorType", TypeOf(err)).
Err(err).Msg("couldn't marshal cosign manifest")
}
@ -200,36 +228,47 @@ func syncCosignSignature(client *resty.Client, imageStore storage.ImageStore,
_, err = imageStore.PutImageManifest(localRepo, cosignTag,
ispec.MediaTypeImageManifest, cosignManifestBuf)
if err != nil {
log.Error().Str("errorType", TypeOf(err)).
sig.log.Error().Str("errorType", TypeOf(err)).
Err(err).Msg("couldn't upload cosign manifest")
return err
}
log.Info().Msgf("successfully synced cosign signature for repo %s digest %s", localRepo, digest)
sig.log.Info().Msgf("successfully synced cosign signature for repo %s digest %s", localRepo, digest)
return nil
}
func syncNotarySignature(client *resty.Client, imageStore storage.ImageStore,
regURL url.URL, localRepo, remoteRepo, digest string, referrers ReferenceList, log log.Logger,
func (sig *signaturesCopier) syncNotarySignature(localRepo, remoteRepo, digest string, referrers ReferenceList,
) error {
if len(referrers.References) == 0 {
return nil
}
log.Info().Msg("syncing notary signatures")
skipNotarySig, err := sig.canSkipNotarySignature(localRepo, digest, referrers)
if skipNotarySig || err != nil {
sig.log.Error().Err(err).Msgf("couldn't check if the upstream image %s:%s notary signature can be skipped",
remoteRepo, digest)
}
if skipNotarySig {
return nil
}
imageStore := sig.storeController.GetImageStore(localRepo)
sig.log.Info().Msg("syncing notary signatures")
for _, ref := range referrers.References {
// get referrer manifest
getRefManifestURL := regURL
getRefManifestURL := sig.upstreamURL
getRefManifestURL.Path = path.Join(getRefManifestURL.Path, "v2", remoteRepo, "manifests", ref.Digest.String())
getRefManifestURL.RawQuery = getRefManifestURL.Query().Encode()
resp, err := client.R().
resp, err := sig.client.R().
Get(getRefManifestURL.String())
if err != nil {
log.Error().Str("errorType", TypeOf(err)).
sig.log.Error().Str("errorType", TypeOf(err)).
Err(err).Msgf("couldn't get notary manifest: %s", getRefManifestURL.String())
return err
@ -240,20 +279,20 @@ func syncNotarySignature(client *resty.Client, imageStore storage.ImageStore,
err = json.Unmarshal(resp.Body(), &artifactManifest)
if err != nil {
log.Error().Str("errorType", TypeOf(err)).
sig.log.Error().Str("errorType", TypeOf(err)).
Err(err).Msgf("couldn't unmarshal notary manifest: %s", getRefManifestURL.String())
return err
}
for _, blob := range artifactManifest.Blobs {
getBlobURL := regURL
getBlobURL := sig.upstreamURL
getBlobURL.Path = path.Join(getBlobURL.Path, "v2", remoteRepo, "blobs", blob.Digest.String())
getBlobURL.RawQuery = getBlobURL.Query().Encode()
resp, err := client.R().SetDoNotParseResponse(true).Get(getBlobURL.String())
resp, err := sig.client.R().SetDoNotParseResponse(true).Get(getBlobURL.String())
if err != nil {
log.Error().Str("errorType", TypeOf(err)).
sig.log.Error().Str("errorType", TypeOf(err)).
Err(err).Msgf("couldn't get notary blob: %s", getBlobURL.String())
return err
@ -262,7 +301,7 @@ func syncNotarySignature(client *resty.Client, imageStore storage.ImageStore,
defer resp.RawBody().Close()
if resp.IsError() {
log.Info().Msgf("couldn't find notary blob from %s, status code: %d",
sig.log.Info().Msgf("couldn't find notary blob from %s, status code: %d",
getBlobURL.String(), resp.StatusCode())
return zerr.ErrSyncSignature
@ -270,7 +309,7 @@ func syncNotarySignature(client *resty.Client, imageStore storage.ImageStore,
_, _, err = imageStore.FullBlobUpload(localRepo, resp.RawBody(), blob.Digest.String())
if err != nil {
log.Error().Str("errorType", TypeOf(err)).
sig.log.Error().Str("errorType", TypeOf(err)).
Err(err).Msg("couldn't upload notary sig blob")
return err
@ -280,50 +319,51 @@ func syncNotarySignature(client *resty.Client, imageStore storage.ImageStore,
_, err = imageStore.PutImageManifest(localRepo, ref.Digest.String(),
artifactspec.MediaTypeArtifactManifest, resp.Body())
if err != nil {
log.Error().Str("errorType", TypeOf(err)).
sig.log.Error().Str("errorType", TypeOf(err)).
Err(err).Msg("couldn't upload notary sig manifest")
return err
}
}
log.Info().Msgf("successfully synced notary signature for repo %s digest %s", localRepo, digest)
sig.log.Info().Msgf("successfully synced notary signature for repo %s digest %s", localRepo, digest)
return nil
}
func canSkipNotarySignature(repo, tag, digest string, refs ReferenceList, imageStore storage.ImageStore,
log log.Logger,
func (sig *signaturesCopier) canSkipNotarySignature(localRepo, digest string, refs ReferenceList,
) (bool, error) {
imageStore := sig.storeController.GetImageStore(localRepo)
// check notary signature already synced
if len(refs.References) > 0 {
localRefs, err := imageStore.GetReferrers(repo, digest, notreg.ArtifactTypeNotation)
localRefs, err := imageStore.GetReferrers(localRepo, digest, notreg.ArtifactTypeNotation)
if err != nil {
if errors.Is(err, zerr.ErrManifestNotFound) {
return false, nil
}
log.Error().Str("errorType", TypeOf(err)).
Err(err).Msgf("couldn't get local notary signature %s:%s manifest", repo, tag)
sig.log.Error().Str("errorType", TypeOf(err)).
Err(err).Msgf("couldn't get local notary signature %s:%s manifest", localRepo, digest)
return false, err
}
if !artifactDescriptorsEqual(localRefs, refs.References) {
log.Info().Msgf("upstream notary signatures %s:%s changed, syncing again", repo, tag)
sig.log.Info().Msgf("upstream notary signatures %s:%s changed, syncing again", localRepo, digest)
return false, nil
}
}
log.Info().Msgf("skipping notary signature %s:%s, already synced", repo, tag)
sig.log.Info().Msgf("skipping notary signature %s:%s, already synced", localRepo, digest)
return true, nil
}
func canSkipCosignSignature(repo, tag, digest string, cosignManifest *ispec.Manifest, imageStore storage.ImageStore,
log log.Logger,
func (sig *signaturesCopier) canSkipCosignSignature(localRepo, digest string, cosignManifest *ispec.Manifest,
) (bool, error) {
imageStore := sig.storeController.GetImageStore(localRepo)
// check cosign signature already synced
if cosignManifest != nil {
var localCosignManifest ispec.Manifest
@ -332,34 +372,34 @@ func canSkipCosignSignature(repo, tag, digest string, cosignManifest *ispec.Mani
because of an issue where cosign digests differs between upstream and downstream */
cosignManifestTag := getCosignTagFromImageDigest(digest)
localCosignManifestBuf, _, _, err := imageStore.GetImageManifest(repo, cosignManifestTag)
localCosignManifestBuf, _, _, err := imageStore.GetImageManifest(localRepo, cosignManifestTag)
if err != nil {
if errors.Is(err, zerr.ErrManifestNotFound) {
return false, nil
}
log.Error().Str("errorType", TypeOf(err)).
Err(err).Msgf("couldn't get local cosign %s:%s manifest", repo, tag)
sig.log.Error().Str("errorType", TypeOf(err)).
Err(err).Msgf("couldn't get local cosign %s:%s manifest", localRepo, digest)
return false, err
}
err = json.Unmarshal(localCosignManifestBuf, &localCosignManifest)
if err != nil {
log.Error().Str("errorType", TypeOf(err)).
Err(err).Msgf("couldn't unmarshal local cosign signature %s:%s manifest", repo, tag)
sig.log.Error().Str("errorType", TypeOf(err)).
Err(err).Msgf("couldn't unmarshal local cosign signature %s:%s manifest", localRepo, digest)
return false, err
}
if !manifestsEqual(localCosignManifest, *cosignManifest) {
log.Info().Msgf("upstream cosign signatures %s:%s changed, syncing again", repo, tag)
sig.log.Info().Msgf("upstream cosign signatures %s:%s changed, syncing again", localRepo, digest)
return false, nil
}
}
log.Info().Msgf("skipping cosign signature %s:%s, already synced", repo, tag)
sig.log.Info().Msgf("skipping cosign signature %s:%s, already synced", localRepo, digest)
return true, nil
}

440
pkg/extensions/sync/sync.go
View File

@ -7,11 +7,9 @@ import (
"fmt"
"io"
"os"
"regexp"
goSync "sync"
"time"
"github.com/Masterminds/semver"
"github.com/containers/common/pkg/retry"
"github.com/containers/image/v5/copy"
"github.com/containers/image/v5/docker"
@ -76,6 +74,12 @@ type Tags struct {
Semver *bool
}
type RepoReferences struct {
contentID int // matched registry config content
name string // repo name
imageReferences []types.ImageReference // contained images(tags)
}
// getUpstreamCatalog gets all repos from a registry.
func getUpstreamCatalog(client *resty.Client, upstreamURL string, log log.Logger) (catalog, error) {
var catalog catalog
@ -106,159 +110,70 @@ func getUpstreamCatalog(client *resty.Client, upstreamURL string, log log.Logger
return catalog, nil
}
// getImageTags lists all tags in a repository.
// It returns a string slice of tags and any error encountered.
func getImageTags(ctx context.Context, sysCtx *types.SystemContext, repoRef reference.Named) ([]string, error) {
dockerRef, err := docker.NewReference(reference.TagNameOnly(repoRef))
// hard to reach test case, injected error, see pkg/test/dev.go
if err = test.Error(err); err != nil {
return nil, err // Should never happen for a reference with tag and no digest
}
tags, err := docker.GetRepositoryTags(ctx, sysCtx, dockerRef)
if err != nil {
return nil, err
}
return tags, nil
}
// filterImagesByTagRegex filters images by tag regex given in the config.
func filterImagesByTagRegex(upstreamReferences *[]types.ImageReference, content Content, log log.Logger) error {
refs := *upstreamReferences
if content.Tags == nil {
// no need to filter anything
return nil
}
if content.Tags.Regex != nil {
log.Info().Msgf("start filtering using the regular expression: %s", *content.Tags.Regex)
tagReg, err := regexp.Compile(*content.Tags.Regex)
if err != nil {
return err
}
numTags := 0
for _, ref := range refs {
tagged := getTagFromRef(ref, log)
if tagged != nil {
if tagReg.MatchString(tagged.Tag()) {
refs[numTags] = ref
numTags++
}
}
}
refs = refs[:numTags]
}
*upstreamReferences = refs
return nil
}
// filterImagesBySemver filters images by checking if their tags are semver compliant.
func filterImagesBySemver(upstreamReferences *[]types.ImageReference, content Content, log log.Logger) {
refs := *upstreamReferences
if content.Tags == nil {
return
}
if content.Tags.Semver != nil && *content.Tags.Semver {
log.Info().Msg("start filtering using semver compliant rule")
numTags := 0
for _, ref := range refs {
tagged := getTagFromRef(ref, log)
if tagged != nil {
_, ok := semver.NewVersion(tagged.Tag())
if ok == nil {
refs[numTags] = ref
numTags++
}
}
}
refs = refs[:numTags]
}
*upstreamReferences = refs
}
// imagesToCopyFromRepos lists all images given a registry name and its repos.
func imagesToCopyFromUpstream(ctx context.Context, registryName string, repos []string,
func imagesToCopyFromUpstream(ctx context.Context, registryName string, repoName string,
upstreamCtx *types.SystemContext, content Content, log log.Logger,
) (map[string][]types.ImageReference, error) {
upstreamReferences := make(map[string][]types.ImageReference)
) ([]types.ImageReference, error) {
imageRefs := []types.ImageReference{}
for _, repoName := range repos {
repoUpstreamReferences := make([]types.ImageReference, 0)
repoRef, err := parseRepositoryReference(fmt.Sprintf("%s/%s", registryName, repoName))
if err != nil {
log.Error().Str("errorType", TypeOf(err)).
Err(err).Msgf("couldn't parse repository reference: %s", repoRef)
repoRef, err := parseRepositoryReference(fmt.Sprintf("%s/%s", registryName, repoName))
if err != nil {
log.Error().Str("errorType", TypeOf(err)).
Err(err).Msgf("couldn't parse repository reference: %s", repoRef)
return nil, err
}
tags, err := getImageTags(ctx, upstreamCtx, repoRef)
if err != nil {
log.Error().Str("errorType", TypeOf(err)).
Err(err).Msgf("couldn't fetch tags for %s", repoRef)
return nil, err
}
for _, tag := range tags {
// don't copy cosign signature, containers/image doesn't support it
// we will copy it manually later
if isCosignTag(tag) {
continue
}
taggedRef, err := reference.WithTag(repoRef, tag)
if err != nil {
log.Err(err).Msgf("error creating a reference for repository %s and tag %q", repoRef.Name(), tag)
return nil, err
}
ref, err := docker.NewReference(taggedRef)
if err != nil {
log.Err(err).Msgf("cannot obtain a valid image reference for transport %q and reference %s",
docker.Transport.Name(), taggedRef.String())
return nil, err
}
repoUpstreamReferences = append(repoUpstreamReferences, ref)
}
upstreamReferences[repoName] = repoUpstreamReferences
log.Debug().Msgf("repo: %s - upstream refs to be copied: %v", repoName, upstreamReferences)
err = filterImagesByTagRegex(&repoUpstreamReferences, content, log)
if err != nil {
return map[string][]types.ImageReference{}, err
}
log.Debug().Msgf("repo: %s - remaining upstream refs to be copied: %v", repoName, repoUpstreamReferences)
filterImagesBySemver(&repoUpstreamReferences, content, log)
log.Debug().Msgf("repo: %s - remaining upstream refs to be copied: %v", repoName, repoUpstreamReferences)
upstreamReferences[repoName] = repoUpstreamReferences
return imageRefs, err
}
return upstreamReferences, nil
tags, err := getImageTags(ctx, upstreamCtx, repoRef)
if err != nil {
log.Error().Str("errorType", TypeOf(err)).
Err(err).Msgf("couldn't fetch tags for %s", repoRef)
return imageRefs, err
}
// filter based on tags rules
if content.Tags != nil {
if content.Tags.Regex != nil {
tags, err = filterTagsByRegex(tags, *content.Tags.Regex, log)
if err != nil {
return imageRefs, err
}
}
if content.Tags.Semver != nil && *content.Tags.Semver {
tags = filterTagsBySemver(tags, log)
}
}
log.Debug().Msgf("repo: %s - upstream tags to be copied: %v", repoName, tags)
for _, tag := range tags {
// don't copy cosign signature, containers/image doesn't support it
// we will copy it manually later
if isCosignTag(tag) {
continue
}
taggedRef, err := reference.WithTag(repoRef, tag)
if err != nil {
log.Err(err).Msgf("error creating a reference for repository %s and tag %q", repoRef.Name(), tag)
return imageRefs, err
}
ref, err := docker.NewReference(taggedRef)
if err != nil {
log.Err(err).Msgf("cannot obtain a valid image reference for transport %q and reference %s",
docker.Transport.Name(), taggedRef.String())
return imageRefs, err
}
imageRefs = append(imageRefs, ref)
}
return imageRefs, nil
}
func getCopyOptions(upstreamCtx, localCtx *types.SystemContext) copy.Options {
@ -339,69 +254,53 @@ func syncRegistry(ctx context.Context, regCfg RegistryConfig,
upstreamAddr := StripRegistryTransport(upstreamURL)
reposWithContentID := make(map[string][]struct {
ref types.ImageReference
content Content
})
reposReferences := []RepoReferences{}
for contentID, repos := range repos {
r := repos
contentID := contentID
for _, repoName := range repos {
var imageReferences []types.ImageReference
if err = retry.RetryIfNecessary(ctx, func() error {
for _, repo := range r {
refs, err := imagesToCopyFromUpstream(ctx, upstreamAddr, r, upstreamCtx, regCfg.Content[contentID], log)
if err != nil {
return err
}
if err = retry.RetryIfNecessary(ctx, func() error {
imageReferences, err = imagesToCopyFromUpstream(ctx, upstreamAddr, repoName, upstreamCtx,
regCfg.Content[contentID], log)
for _, ref := range refs[repo] {
reposWithContentID[repo] = append(reposWithContentID[repo], struct {
ref types.ImageReference
content Content
}{
ref: ref,
content: regCfg.Content[contentID],
})
}
}
return nil
}, retryOptions); err != nil {
log.Error().Str("errorType", TypeOf(err)).
Err(err).Msg("error while getting images references from upstream, retrying...")
return err
}
}
for remoteRepo, imageList := range reposWithContentID {
select {
case <-ctx.Done():
return ctx.Err()
default:
break
}
remoteRepoCopy := remoteRepo
for _, image := range imageList {
localRepo := getRepoDestination(remoteRepo, image.content)
imageStore := storeController.GetImageStore(localRepo)
localCachePath, err := getLocalCachePath(imageStore, localRepo)
if err != nil {
return err
}, retryOptions); err != nil {
log.Error().Str("errorType", TypeOf(err)).
Err(err).Msgf("couldn't get localCachePath for %s", remoteRepoCopy)
Err(err).Msg("error while getting images references from upstream, retrying...")
return err
}
defer os.RemoveAll(localCachePath)
reposReferences = append(reposReferences, RepoReferences{
contentID: contentID,
name: repoName,
imageReferences: imageReferences,
})
}
}
upstreamImageRef := image.ref
sig := newSignaturesCopier(httpClient, *registryURL, storeController, log)
for _, repoReference := range reposReferences {
upstreamRepo := repoReference.name
content := regCfg.Content[repoReference.contentID]
localRepo := getRepoDestination(upstreamRepo, content)
imageStore := storeController.GetImageStore(localRepo)
localCachePath, err := getLocalCachePath(imageStore, localRepo)
if err != nil {
log.Error().Str("errorType", TypeOf(err)).
Err(err).Msgf("couldn't get localCachePath for %s", localRepo)
return err
}
defer os.RemoveAll(localCachePath)
for _, upstreamImageRef := range repoReference.imageReferences {
upstreamImageDigest, err := docker.GetDigest(ctx, upstreamCtx, upstreamImageRef)
if err != nil {
log.Error().Err(err).Msgf("couldn't get upstream image %s manifest", upstreamImageRef.DockerReference())
@ -409,17 +308,15 @@ func syncRegistry(ctx context.Context, regCfg RegistryConfig,
return err
}
tag := getTagFromRef(upstreamImageRef, log).Tag()
// get upstream signatures
cosignManifest, err := getCosignManifest(httpClient, *registryURL, remoteRepoCopy,
upstreamImageDigest.String(), log)
cosignManifest, err := sig.getCosignManifest(upstreamRepo, upstreamImageDigest.String())
if err != nil && !errors.Is(err, zerr.ErrSyncSignatureNotFound) {
log.Error().Err(err).Msgf("couldn't get upstream image %s cosign manifest", upstreamImageRef.DockerReference())
return err
}
refs, err := getNotaryRefs(httpClient, *registryURL, remoteRepoCopy, upstreamImageDigest.String(), log)
refs, err := sig.getNotaryRefs(upstreamRepo, upstreamImageDigest.String())
if err != nil && !errors.Is(err, zerr.ErrSyncSignatureNotFound) {
log.Error().Err(err).Msgf("couldn't get upstream image %s notary references", upstreamImageRef.DockerReference())
@ -437,6 +334,8 @@ func syncRegistry(ctx context.Context, regCfg RegistryConfig,
}
}
tag := getTagFromRef(upstreamImageRef, log).Tag()
skipImage, err := canSkipImage(localRepo, tag, upstreamImageDigest.String(), imageStore, log)
if err != nil {
log.Error().Err(err).Msgf("couldn't check if the upstream image %s can be skipped",
@ -445,103 +344,60 @@ func syncRegistry(ctx context.Context, regCfg RegistryConfig,
return err
}
// sync only differences
if skipImage {
if !skipImage {
// sync image
localImageRef, err := getLocalImageRef(localCachePath, localRepo, tag)
if err != nil {
log.Error().Str("errorType", TypeOf(err)).
Err(err).Msgf("couldn't obtain a valid image reference for reference %s/%s:%s",
localCachePath, localRepo, tag)
return err
}
log.Info().Msgf("copying image %s to %s", upstreamImageRef.DockerReference(), localCachePath)
if err = retry.RetryIfNecessary(ctx, func() error {
_, err = copy.Image(ctx, policyCtx, localImageRef, upstreamImageRef, &options)
return err
}, retryOptions); err != nil {
log.Error().Str("errorType", TypeOf(err)).
Err(err).Msgf("error while copying image %s to %s",
upstreamImageRef.DockerReference(), localCachePath)
return err
}
// push from cache to repo
err = pushSyncedLocalImage(localRepo, tag, localCachePath, imageStore, log)
if err != nil {
log.Error().Str("errorType", TypeOf(err)).
Err(err).Msgf("error while pushing synced cached image %s",
fmt.Sprintf("%s/%s:%s", localCachePath, localRepo, tag))
return err
}
} else {
log.Info().Msgf("already synced image %s, checking its signatures", upstreamImageRef.DockerReference())
skipNotarySig, err := canSkipNotarySignature(localRepo, tag, upstreamImageDigest.String(),
refs, imageStore, log)
if err != nil {
log.Error().Err(err).Msgf("couldn't check if the upstream image %s notary signature can be skipped",
upstreamImageRef.DockerReference())
}
if !skipNotarySig {
if err = retry.RetryIfNecessary(ctx, func() error {
err = syncNotarySignature(httpClient, imageStore, *registryURL, localRepo, remoteRepoCopy,
upstreamImageDigest.String(), refs, log)
return err
}, retryOptions); err != nil {
log.Error().Err(err).Msgf("couldn't copy notary signature for %s", upstreamImageRef.DockerReference())
}
}
skipCosignSig, err := canSkipCosignSignature(localRepo, tag, upstreamImageDigest.String(),
cosignManifest, imageStore, log)
if err != nil {
log.Error().Err(err).Msgf("couldn't check if the upstream image %s cosign signature can be skipped",
upstreamImageRef.DockerReference())
}
if !skipCosignSig {
if err = retry.RetryIfNecessary(ctx, func() error {
err = syncCosignSignature(httpClient, imageStore, *registryURL, localRepo, remoteRepoCopy,
upstreamImageDigest.String(), cosignManifest, log)
return err
}, retryOptions); err != nil {
log.Error().Err(err).Msgf("couldn't copy cosign signature for %s", upstreamImageRef.DockerReference())
}
}
continue
}
localImageRef, err := getLocalImageRef(localCachePath, localRepo, tag)
if err != nil {
log.Error().Str("errorType", TypeOf(err)).
Err(err).Msgf("couldn't obtain a valid image reference for reference %s/%s:%s",
localCachePath, localRepo, tag)
return err
}
log.Info().Msgf("copying image %s to %s", upstreamImageRef.DockerReference(), localCachePath)
// sync signatures
if err = retry.RetryIfNecessary(ctx, func() error {
_, err = copy.Image(ctx, policyCtx, localImageRef, upstreamImageRef, &options)
err = sig.syncNotarySignature(localRepo, upstreamRepo, upstreamImageDigest.String(), refs)
if err != nil {
return err
}
return err
}, retryOptions); err != nil {
log.Error().Str("errorType", TypeOf(err)).
Err(err).Msgf("error while copying image %s to %s",
upstreamImageRef.DockerReference(), localCachePath)
err = sig.syncCosignSignature(localRepo, upstreamRepo, upstreamImageDigest.String(), cosignManifest)
if err != nil {
return err
}
return err
}
// push from cache to repo
err = pushSyncedLocalImage(localRepo, tag, localCachePath, imageStore, log)
if err != nil {
log.Error().Str("errorType", TypeOf(err)).
Err(err).Msgf("error while pushing synced cached image %s",
fmt.Sprintf("%s/%s:%s", localCachePath, localRepo, tag))
return err
}
refs, err = getNotaryRefs(httpClient, *registryURL, remoteRepoCopy, upstreamImageDigest.String(), log)
if err = retry.RetryIfNecessary(ctx, func() error {
err = syncNotarySignature(httpClient, imageStore, *registryURL, localRepo,
remoteRepoCopy, upstreamImageDigest.String(), refs, log)
return err
return nil
}, retryOptions); err != nil {
log.Error().Str("errorType", TypeOf(err)).
Err(err).Msgf("couldn't copy notary signature for %s", upstreamImageRef.DockerReference())
}
cosignManifest, err = getCosignManifest(httpClient, *registryURL, remoteRepoCopy,
upstreamImageDigest.String(), log)
if err = retry.RetryIfNecessary(ctx, func() error {
err = syncCosignSignature(httpClient, imageStore, *registryURL, localRepo,
remoteRepoCopy, upstreamImageDigest.String(), cosignManifest, log)
return err
}, retryOptions); err != nil {
log.Error().Str("errorType", TypeOf(err)).
Err(err).Msgf("couldn't copy cosign signature for %s", upstreamImageRef.DockerReference())
}
}
}

38
pkg/extensions/sync/sync_internal_test.go
View File

@ -290,14 +290,13 @@ func TestSyncInternal(t *testing.T) {
})
Convey("Test imagesToCopyFromUpstream()", t, func() {
repos := []string{"repo1"}
upstreamCtx := &types.SystemContext{}
_, err := imagesToCopyFromUpstream(context.Background(), "localhost:4566", repos, upstreamCtx,
_, err := imagesToCopyFromUpstream(context.Background(), "localhost:4566", "repo1", upstreamCtx,
Content{}, log.NewLogger("debug", ""))
So(err, ShouldNotBeNil)
_, err = imagesToCopyFromUpstream(context.Background(), "docker://localhost:4566", repos, upstreamCtx,
_, err = imagesToCopyFromUpstream(context.Background(), "docker://localhost:4566", "repo1", upstreamCtx,
Content{}, log.NewLogger("debug", ""))
So(err, ShouldNotBeNil)
})
@ -323,16 +322,15 @@ func TestSyncInternal(t *testing.T) {
Layers: []ispec.Descriptor{desc},
}
err = syncCosignSignature(client, &local.ImageStoreLocal{}, *regURL, testImage, testImage,
testImageTag, &ispec.Manifest{}, log)
sig := newSignaturesCopier(client, *regURL, storage.StoreController{DefaultStore: &local.ImageStoreLocal{}}, log)
err = sig.syncCosignSignature(testImage, testImage, testImageTag, &ispec.Manifest{})
So(err, ShouldNotBeNil)
err = syncCosignSignature(client, &local.ImageStoreLocal{}, *regURL, testImage, testImage,
testImageTag, &manifest, log)
err = sig.syncCosignSignature(testImage, testImage, testImageTag, &manifest)
So(err, ShouldNotBeNil)
err = syncNotarySignature(client, &local.ImageStoreLocal{}, *regURL, testImage, testImage,
"invalidDigest", ReferenceList{[]artifactspec.Descriptor{ref}}, log)
err = sig.syncNotarySignature(testImage, testImage, "invalidDigest", ReferenceList{[]artifactspec.Descriptor{ref}})
So(err, ShouldNotBeNil)
})
@ -370,16 +368,20 @@ func TestSyncInternal(t *testing.T) {
So(err, ShouldBeNil)
So(testImageManifestDigest, ShouldNotBeEmpty)
canBeSkipped, err = canSkipNotarySignature(testImage, testImageTag,
testImageManifestDigest, refs, imageStore, log)
regURL, err := url.Parse("http://zot")
So(err, ShouldBeNil)
So(regURL, ShouldNotBeNil)
sig := newSignaturesCopier(resty.New(), *regURL, storage.StoreController{DefaultStore: imageStore}, log)
canBeSkipped, err = sig.canSkipNotarySignature(testImage, testImageManifestDigest, refs)
So(err, ShouldBeNil)
So(canBeSkipped, ShouldBeFalse)
err = os.Chmod(path.Join(imageStore.RootDir(), testImage, "index.json"), 0o000)
So(err, ShouldBeNil)
canBeSkipped, err = canSkipNotarySignature(testImage, testImageTag,
testImageManifestDigest, refs, imageStore, log)
canBeSkipped, err = sig.canSkipNotarySignature(testImage, testImageManifestDigest, refs)
So(err, ShouldNotBeNil)
So(canBeSkipped, ShouldBeFalse)
@ -390,8 +392,7 @@ func TestSyncInternal(t *testing.T) {
err = os.Chmod(path.Join(imageStore.RootDir(), testImage, "index.json"), 0o755)
So(err, ShouldBeNil)
canBeSkipped, err = canSkipCosignSignature(testImage, testImageTag,
testImageManifestDigest, &cosignManifest, imageStore, log)
canBeSkipped, err = sig.canSkipCosignSignature(testImage, testImageManifestDigest, &cosignManifest)
So(err, ShouldBeNil)
So(canBeSkipped, ShouldBeFalse)
})
@ -424,6 +425,13 @@ func TestSyncInternal(t *testing.T) {
So(len(filteredRepos), ShouldEqual, 0)
})
Convey("Test filterTagsByRegex()", t, func() {
tags := []string{"one"}
filteredTags, err := filterTagsByRegex(tags, ".*", log.NewLogger("", ""))
So(err, ShouldBeNil)
So(filteredTags, ShouldResemble, tags)
})
Convey("Verify pushSyncedLocalImage func", t, func() {
storageDir := t.TempDir()

62
pkg/extensions/sync/utils.go
View File

@ -1,6 +1,7 @@
package sync
import (
"context"
"crypto/tls"
"crypto/x509"
"encoding/json"
@ -9,8 +10,10 @@ import (
"net/url"
"os"
"path"
"regexp"
"strings"
"github.com/Masterminds/semver"
glob "github.com/bmatcuk/doublestar/v4"
"github.com/containers/image/v5/docker"
"github.com/containers/image/v5/docker/reference"
@ -49,6 +52,65 @@ func getTagFromRef(ref types.ImageReference, log log.Logger) reference.Tagged {
return tagged
}
// getImageTags lists all tags in a repository.
// It returns a string slice of tags and any error encountered.
func getImageTags(ctx context.Context, sysCtx *types.SystemContext, repoRef reference.Named) ([]string, error) {
dockerRef, err := docker.NewReference(reference.TagNameOnly(repoRef))
// hard to reach test case, injected error, see pkg/test/dev.go
if err = test.Error(err); err != nil {
return nil, err // Should never happen for a reference with tag and no digest
}
tags, err := docker.GetRepositoryTags(ctx, sysCtx, dockerRef)
if err != nil {
return nil, err
}
return tags, nil
}
// filterTagsByRegex filters images by tag regex given in the config.
func filterTagsByRegex(tags []string, regex string, log log.Logger) ([]string, error) {
filteredTags := []string{}
if len(tags) == 0 || regex == "" {
return filteredTags, nil
}
log.Info().Msgf("start filtering using the regular expression: %s", regex)
tagReg, err := regexp.Compile(regex)
if err != nil {
log.Error().Err(err).Str("regex", regex).Msg("couldn't compile regex")
return filteredTags, err
}
for _, tag := range tags {
if tagReg.MatchString(tag) {
filteredTags = append(filteredTags, tag)
}
}
return filteredTags, nil
}
// filterTagsBySemver filters tags by checking if they are semver compliant.
func filterTagsBySemver(tags []string, log log.Logger) []string {
filteredTags := []string{}
log.Info().Msg("start filtering using semver compliant rule")
for _, tag := range tags {
_, err := semver.NewVersion(tag)
if err == nil {
filteredTags = append(filteredTags, tag)
}
}
return filteredTags
}
// parseRepositoryReference parses input into a reference.Named, and verifies that it names a repository, not an image.
func parseRepositoryReference(input string) (reference.Named, error) {
ref, err := reference.ParseNormalizedNamed(input)