alt/zot-wo-auth
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

sync: Added support for syncing notary/cosign signatures, closes #261

Browse Source 
Signed-off-by: Petu Eusebiu <peusebiu@cisco.com>
This commit is contained in:
Petu Eusebiu 2021-12-07 20:26:26 +02:00 committed by Ramkumar Chinchani
parent e6d6d5a7de
commit 1109bb4dde
7 changed files with 1026 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
pkg/cli/root_test.go
View File

@ -188,6 +188,22 @@ func TestVerify(t *testing.T) {
So(func() { _ = cli.NewServerRootCmd().Execute() }, ShouldPanic)
})
Convey("Test verify w/ sync and w/ filesystem storage", t, func(c C) {
tmpfile, err := ioutil.TempFile("", "zot-test*.json")
So(err, ShouldBeNil)
defer os.Remove(tmpfile.Name()) // clean up
content := []byte(`{"storage":{"rootDirectory":"/tmp/zot"},
"http":{"address":"127.0.0.1","port":"8080","realm":"zot",
"auth":{"htpasswd":{"path":"test/data/htpasswd"},"failDelay":1}},
"extensions":{"sync": {"registries": [{"url":"localhost:9999"}]}}}`)
_, err = tmpfile.Write(content)
So(err, ShouldBeNil)
err = tmpfile.Close()
So(err, ShouldBeNil)
os.Args = []string{"cli_test", "verify", tmpfile.Name()}
So(func() { _ = cli.NewServerRootCmd().Execute() }, ShouldNotPanic)
})
Convey("Test verify with bad sync prefixes", t, func(c C) {
tmpfile, err := ioutil.TempFile("", "zot-test*.json")
So(err, ShouldBeNil)
@ -274,6 +290,12 @@ func TestScrub(t *testing.T) {
So(err, ShouldBeNil)
})
Convey("Test scrub no args", t, func(c C) {
os.Args = []string{"cli_test", "scrub"}
err := cli.NewServerRootCmd().Execute()
So(err, ShouldBeNil)
})
Convey("Test scrub config", t, func(c C) {
Convey("non-existent config", func(c C) {
os.Args = []string{"cli_test", "scrub", path.Join(os.TempDir(), "/x.yaml")}

30
pkg/extensions/sync/on_demand.go
View File

@ -21,6 +21,12 @@ func OneImage(cfg Config, storeController storage.StoreController,
repo, tag string, log log.Logger) error {
var credentialsFile CredentialsFile
/* don't copy cosign signature, containers/image doesn't support it
we will copy it manually later */
if isCosignTag(tag) {
return nil
}
if cfg.CredentialsFile != "" {
var err error
@ -46,7 +52,8 @@ func OneImage(cfg Config, storeController storage.StoreController,
return err
}
for _, regCfg := range cfg.Registries {
for _, registryCfg := range cfg.Registries {
regCfg := registryCfg
if !regCfg.OnDemand {
log.Info().Msgf("skipping syncing on demand from %s, onDemand flag is false", regCfg.URL)
@ -127,9 +134,9 @@ func OneImage(cfg Config, storeController storage.StoreController,
if err = retry.RetryIfNecessary(context.Background(), func() error {
_, copyErr = copy.Image(context.Background(), policyCtx, localRef, upstreamRef, &options)
return err
}, retryOptions); copyErr != nil {
log.Error().Err(copyErr).Msgf("error while copying image %s to %s",
return copyErr
}, retryOptions); err != nil {
log.Error().Err(err).Msgf("error while copying image %s to %s",
upstreamRef.DockerReference().Name(), localTaggedRepo)
} else {
log.Info().Msgf("successfully synced %s", upstreamRef.DockerReference().Name())
@ -142,6 +149,21 @@ func OneImage(cfg Config, storeController storage.StoreController,
return err
}
httpClient, err := getHTTPClient(&regCfg, credentialsFile[upstreamRegistryName], log)
if err != nil {
return err
}
if err = retry.RetryIfNecessary(context.Background(), func() error {
err = syncSignatures(httpClient, storeController, regCfg.URL, imageName, upstreamTaggedRef.Tag(), log)
return err
}, retryOptions); err != nil {
log.Error().Err(err).Msgf("Couldn't copy image signature %s", upstreamRef.DockerReference().Name())
return err
}
return nil
}
}

79
pkg/extensions/sync/sync.go
View File

@ -2,12 +2,9 @@ package sync
import (
"context"
"crypto/tls"
"crypto/x509"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"os"
"path"
"regexp"
@ -75,49 +72,10 @@ type Tags struct {
}
// getUpstreamCatalog gets all repos from a registry.
func getUpstreamCatalog(regCfg *RegistryConfig, credentials Credentials, log log.Logger) (catalog, error) {
func getUpstreamCatalog(client *resty.Client, regCfg *RegistryConfig, log log.Logger) (catalog, error) {
var c catalog
registryCatalogURL := fmt.Sprintf("%s%s", regCfg.URL, "/v2/_catalog")
client := resty.New()
if regCfg.CertDir != "" {
log.Debug().Msgf("sync: using certs directory: %s", regCfg.CertDir)
clientCert := path.Join(regCfg.CertDir, "client.cert")
clientKey := path.Join(regCfg.CertDir, "client.key")
caCertPath := path.Join(regCfg.CertDir, "ca.crt")
caCert, err := ioutil.ReadFile(caCertPath)
if err != nil {
log.Error().Err(err).Msg("couldn't read CA certificate")
return c, err
}
caCertPool := x509.NewCertPool()
caCertPool.AppendCertsFromPEM(caCert)
client.SetTLSClientConfig(&tls.Config{RootCAs: caCertPool, MinVersion: tls.VersionTLS12})
cert, err := tls.LoadX509KeyPair(clientCert, clientKey)
if err != nil {
log.Error().Err(err).Msg("couldn't read certificates key pairs")
return c, err
}
client.SetCertificates(cert)
}
// nolint: gosec
if regCfg.TLSVerify != nil && !*regCfg.TLSVerify {
client.SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true})
}
if credentials.Username != "" && credentials.Password != "" {
log.Debug().Msgf("sync: using basic auth")
client.SetBasicAuth(credentials.Username, credentials.Password)
}
resp, err := client.R().SetHeader("Content-Type", "application/json").Get(registryCatalogURL)
if err != nil {
@ -247,6 +205,12 @@ func imagesToCopyFromUpstream(registryName string, repos []string, upstreamCtx *
}
for _, tag := range tags {
// don't copy cosign signature, containers/image doesn't support it
// we will copy it manually later
if isCosignTag(tag) {
continue
}
taggedRef, err := reference.WithTag(repoRef, tag)
if err != nil {
log.Err(err).Msgf("error creating a reference for repository %s and tag %q", repoRef.Name(), tag)
@ -283,11 +247,10 @@ func imagesToCopyFromUpstream(registryName string, repos []string, upstreamCtx *
func getCopyOptions(upstreamCtx, localCtx *types.SystemContext) copy.Options {
options := copy.Options{
DestinationCtx: localCtx,
SourceCtx: upstreamCtx,
ReportWriter: io.Discard,
// force only oci manifest MIME type
ForceManifestMIMEType: ispec.MediaTypeImageManifest,
DestinationCtx: localCtx,
SourceCtx: upstreamCtx,
ReportWriter: io.Discard,
ForceManifestMIMEType: ispec.MediaTypeImageManifest, // force only oci manifest MIME type
}
return options
@ -295,7 +258,6 @@ func getCopyOptions(upstreamCtx, localCtx *types.SystemContext) copy.Options {
func getUpstreamContext(regCfg *RegistryConfig, credentials Credentials) *types.SystemContext {
upstreamCtx := &types.SystemContext{}
upstreamCtx.DockerCertPath = regCfg.CertDir
upstreamCtx.DockerDaemonCertPath = regCfg.CertDir
@ -336,8 +298,13 @@ func syncRegistry(regCfg RegistryConfig, storeController storage.StoreController
var catalog catalog
httpClient, err := getHTTPClient(&regCfg, credentials, log)
if err != nil {
return err
}
if err = retry.RetryIfNecessary(context.Background(), func() error {
catalog, err = getUpstreamCatalog(&regCfg, credentials, log)
catalog, err = getUpstreamCatalog(httpClient, &regCfg, log)
return err
}, retryOptions); err != nil {
@ -430,6 +397,16 @@ func syncRegistry(regCfg RegistryConfig, storeController storage.StoreController
return err
}
if err = retry.RetryIfNecessary(context.Background(), func() error {
err = syncSignatures(httpClient, storeController, regCfg.URL, imageName, upstreamTaggedRef.Tag(), log)
return err
}, retryOptions); err != nil {
log.Error().Err(err).Msgf("Couldn't copy image signature %s", upstreamRef.DockerReference().Name())
return err
}
}
log.Info().Msgf("finished syncing %s", regCfg.URL)
@ -445,6 +422,8 @@ func getLocalContexts(log log.Logger) (*types.SystemContext, *signature.PolicyCo
var err error
localCtx := &types.SystemContext{}
// preserve compression
localCtx.OCIAcceptUncompressedLayers = true
// accept any image with or without signature
policy = &signature.Policy{Default: []signature.PolicyRequirement{signature.NewPRInsecureAcceptAnything()}}

31
pkg/extensions/sync/sync_internal_test.go
View File

@ -5,6 +5,7 @@ import (
"encoding/json"
"fmt"
"io/ioutil"
"net/url"
"os"
"path"
goSync "sync"
@ -18,6 +19,7 @@ import (
ispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/rs/zerolog"
. "github.com/smartystreets/goconvey/convey"
"gopkg.in/resty.v1"
"zotregistry.io/zot/errors"
"zotregistry.io/zot/pkg/extensions/monitoring"
"zotregistry.io/zot/pkg/log"
@ -123,11 +125,11 @@ func TestSyncInternal(t *testing.T) {
CertDir: "/tmp/missing_certs/a/b/c/d/z",
}
_, err := getUpstreamCatalog(&syncRegistryConfig, Credentials{}, log.NewLogger("debug", ""))
_, err := getUpstreamCatalog(resty.New(), &syncRegistryConfig, log.NewLogger("debug", ""))
So(err, ShouldNotBeNil)
})
Convey("Test getUpstreamCatalog() with bad certs", t, func() {
Convey("Test getHttpClient() with bad certs", t, func() {
badCertsDir, err := ioutil.TempDir("", "bad_certs*")
if err != nil {
panic(err)
@ -153,7 +155,10 @@ func TestSyncInternal(t *testing.T) {
CertDir: badCertsDir,
}
_, err = getUpstreamCatalog(&syncRegistryConfig, Credentials{}, log.NewLogger("debug", ""))
_, err = getHTTPClient(&syncRegistryConfig, Credentials{}, log.NewLogger("debug", ""))
So(err, ShouldNotBeNil)
syncRegistryConfig.CertDir = "/path/to/invalid/cert"
_, err = getHTTPClient(&syncRegistryConfig, Credentials{}, log.NewLogger("debug", ""))
So(err, ShouldNotBeNil)
})
@ -169,6 +174,26 @@ func TestSyncInternal(t *testing.T) {
So(err, ShouldNotBeNil)
})
Convey("Test OneImage() skips cosign signatures", t, func() {
err := OneImage(Config{}, storage.StoreController{}, "repo", "sha256-.sig", log.NewLogger("", ""))
So(err, ShouldBeNil)
})
Convey("Test syncSignatures()", t, func() {
log := log.NewLogger("", "")
err := syncSignatures(resty.New(), storage.StoreController{}, "%", "repo", "tag", log)
So(err, ShouldNotBeNil)
err = syncSignatures(resty.New(), storage.StoreController{}, "http://zot", "repo", "tag", log)
So(err, ShouldNotBeNil)
err = syncSignatures(resty.New(), storage.StoreController{}, "https://google.com", "repo", "tag", log)
So(err, ShouldNotBeNil)
url, _ := url.Parse("invalid")
err = syncCosignSignature(resty.New(), storage.StoreController{}, *url, "repo", "tag", log)
So(err, ShouldNotBeNil)
err = syncNotarySignature(resty.New(), storage.StoreController{}, *url, "repo", "tag", log)
So(err, ShouldNotBeNil)
})
Convey("Test filterRepos()", t, func() {
repos := []string{"repo", "repo1", "repo2", "repo/repo2", "repo/repo2/repo3/repo4"}
contents := []Content{

593
pkg/extensions/sync/sync_test.go
View File

@ -4,6 +4,7 @@
package sync_test
import (
"context"
"crypto/tls"
"crypto/x509"
"encoding/json"
@ -12,13 +13,21 @@ import (
"io"
"io/ioutil"
"os"
"os/exec"
"path"
"reflect"
"strings"
"testing"
"time"
"github.com/notaryproject/notation-go-lib"
godigest "github.com/opencontainers/go-digest"
ispec "github.com/opencontainers/image-spec/specs-go/v1"
artifactspec "github.com/oras-project/artifacts-spec/specs-go/v1"
"github.com/sigstore/cosign/cmd/cosign/cli/generate"
"github.com/sigstore/cosign/cmd/cosign/cli/options"
"github.com/sigstore/cosign/cmd/cosign/cli/sign"
"github.com/sigstore/cosign/cmd/cosign/cli/verify"
. "github.com/smartystreets/goconvey/convey"
"gopkg.in/resty.v1"
"zotregistry.io/zot/pkg/api"
@ -47,6 +56,10 @@ type TagsList struct {
Tags []string
}
type ReferenceList struct {
References []notation.Descriptor `json:"references"`
}
type catalog struct {
Repositories []string `json:"repositories"`
}
@ -199,6 +212,8 @@ func startDownstreamServer(secure bool, syncConfig *sync.Config) (*api.Controlle
}
destConfig.Storage.RootDirectory = destDir
destConfig.Storage.Dedupe = false
destConfig.Storage.GC = false
destConfig.Extensions = &extconf.ExtensionConfig{}
destConfig.Extensions.Search = nil
@ -1632,6 +1647,37 @@ func TestSyncSubPaths(t *testing.T) {
})
}
func TestSyncOnDemandRepoErr(t *testing.T) {
Convey("Verify sync on demand parseRepositoryReference error", t, func() {
tlsVerify := false
syncRegistryConfig := sync.RegistryConfig{
Content: []sync.Content{
{
// will sync on demand, should not be filtered out
Prefix: testImage,
},
},
URL: "docker://invalid",
TLSVerify: &tlsVerify,
CertDir: "",
OnDemand: true,
}
syncConfig := &sync.Config{Registries: []sync.RegistryConfig{syncRegistryConfig}}
dc, destBaseURL, destDir, _ := startDownstreamServer(false, syncConfig)
defer os.RemoveAll(destDir)
defer func() {
dc.Shutdown()
}()
resp, err := resty.R().Get(destBaseURL + "/v2/" + testImage + "/manifests/" + testImageTag)
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 404)
})
}
func TestSyncOnDemandContentFiltering(t *testing.T) {
Convey("Verify sync on demand feature", t, func() {
sc, srcBaseURL, srcDir, _, _ := startUpstreamServer(false, false)
@ -1821,3 +1867,550 @@ func TestSyncConfigRules(t *testing.T) {
})
})
}
func TestSyncSignatures(t *testing.T) {
Convey("Verify sync signatures", t, func() {
updateDuration, _ := time.ParseDuration("30m")
sc, srcBaseURL, srcDir, _, _ := startUpstreamServer(false, false)
defer os.RemoveAll(srcDir)
defer func() {
sc.Shutdown()
}()
// create repo, push and sign it
repoName := "signed-repo"
var digest godigest.Digest
So(func() { digest = pushRepo(srcBaseURL, repoName) }, ShouldNotPanic)
splittedURL := strings.SplitAfter(srcBaseURL, ":")
srcPort := splittedURL[len(splittedURL)-1]
cwd, err := os.Getwd()
So(err, ShouldBeNil)
defer func() { _ = os.Chdir(cwd) }()
tdir, err := ioutil.TempDir("", "sigs")
So(err, ShouldBeNil)
defer os.RemoveAll(tdir)
_ = os.Chdir(tdir)
So(func() { signImage(tdir, srcPort, repoName, digest) }, ShouldNotPanic)
regex := ".*"
var semver bool
var tlsVerify bool
syncRegistryConfig := sync.RegistryConfig{
Content: []sync.Content{
{
// won't match any image on source registry, we will sync on demand
Prefix: repoName,
Tags: &sync.Tags{
Regex: &regex,
Semver: &semver,
},
},
},
URL: srcBaseURL,
PollInterval: updateDuration,
TLSVerify: &tlsVerify,
CertDir: "",
OnDemand: true,
}
syncConfig := &sync.Config{
Registries: []sync.RegistryConfig{syncRegistryConfig},
}
dc, destBaseURL, destDir, destClient := startDownstreamServer(false, syncConfig)
defer os.RemoveAll(destDir)
defer func() {
dc.Shutdown()
}()
// wait for sync
var destTagsList TagsList
for {
resp, err := destClient.R().Get(destBaseURL + "/v2/" + repoName + "/tags/list")
if err != nil {
panic(err)
}
err = json.Unmarshal(resp.Body(), &destTagsList)
if err != nil {
panic(err)
}
if len(destTagsList.Tags) > 0 {
break
}
time.Sleep(500 * time.Millisecond)
}
splittedURL = strings.SplitAfter(destBaseURL, ":")
destPort := splittedURL[len(splittedURL)-1]
a := &options.AnnotationOptions{Annotations: []string{"tag=1.0"}}
amap, err := a.AnnotationsMap()
if err != nil {
panic(err)
}
// notation verify the image
image := fmt.Sprintf("localhost:%s/%s:%s", destPort, repoName, "1.0")
cmd := exec.Command("notation", "verify", "--cert", "good", "--plain-http", image)
out, err := cmd.CombinedOutput()
So(err, ShouldBeNil)
msg := string(out)
So(msg, ShouldNotBeEmpty)
So(strings.Contains(msg, "verification failure"), ShouldBeFalse)
// cosign verify the image
vrfy := verify.VerifyCommand{
RegistryOptions: options.RegistryOptions{AllowInsecure: true},
CheckClaims: true,
KeyRef: path.Join(tdir, "cosign.pub"),
Annotations: amap,
}
err = vrfy.Exec(context.TODO(), []string{fmt.Sprintf("localhost:%s/%s:%s", destPort, repoName, "1.0")})
So(err, ShouldBeNil)
// test negative cases (trigger errors)
// test notary signatures errors
// based on manifest digest get referrers
getReferrersURL := srcBaseURL + path.Join("/oras/artifacts/v1/", repoName, "manifests", digest.String(), "referrers")
resp, err := resty.R().
SetHeader("Content-Type", "application/json").
SetQueryParam("artifactType", "application/vnd.cncf.notary.v2.signature").
Get(getReferrersURL)
So(err, ShouldBeNil)
So(resp, ShouldNotBeEmpty)
var referrers ReferenceList
err = json.Unmarshal(resp.Body(), &referrers)
So(err, ShouldBeNil)
// read manifest
var nm artifactspec.Manifest
for _, ref := range referrers.References {
refPath := path.Join(srcDir, repoName, "blobs", string(ref.Digest.Algorithm()), ref.Digest.Hex())
body, err := ioutil.ReadFile(refPath)
So(err, ShouldBeNil)
err = json.Unmarshal(body, &nm)
So(err, ShouldBeNil)
// triggers perm denied on sig blobs
for _, blob := range nm.Blobs {
blobPath := path.Join(srcDir, repoName, "blobs", string(blob.Digest.Algorithm()), blob.Digest.Hex())
err := os.Chmod(blobPath, 0o000)
So(err, ShouldBeNil)
}
}
// err = os.Chmod(path.Join(srcDir, repoName, "index.json"), 0000)
// if err != nil {
// panic(err)
// }
resp, _ = resty.R().Post(destBaseURL + "/sync")
So(resp, ShouldNotBeNil)
So(resp.StatusCode(), ShouldEqual, 500)
for _, blob := range nm.Blobs {
srcBlobPath := path.Join(srcDir, repoName, "blobs", string(blob.Digest.Algorithm()), blob.Digest.Hex())
err := os.Chmod(srcBlobPath, 0o755)
So(err, ShouldBeNil)
destBlobPath := path.Join(destDir, repoName, "blobs", string(blob.Digest.Algorithm()), blob.Digest.Hex())
err = os.Remove(destBlobPath)
So(err, ShouldBeNil)
err = os.MkdirAll(destBlobPath, 0o000)
So(err, ShouldBeNil)
}
resp, _ = resty.R().Post(destBaseURL + "/sync")
So(resp, ShouldNotBeNil)
So(resp.StatusCode(), ShouldEqual, 500)
// clean
for _, blob := range nm.Blobs {
destBlobPath := path.Join(destDir, repoName, "blobs", string(blob.Digest.Algorithm()), blob.Digest.Hex())
err = os.Chmod(destBlobPath, 0o755)
So(err, ShouldBeNil)
err = os.Remove(destBlobPath)
So(err, ShouldBeNil)
}
// test cosign signatures errors
// based on manifest digest get cosign manifest
cosignEncodedDigest := strings.Replace(digest.String(), ":", "-", 1) + ".sig"
getCosignManifestURL := srcBaseURL + path.Join("/v2", repoName, "manifests", cosignEncodedDigest)
mResp, err := resty.R().Get(getCosignManifestURL)
So(err, ShouldBeNil)
var cm ispec.Manifest
err = json.Unmarshal(mResp.Body(), &cm)
So(err, ShouldBeNil)
for _, blob := range cm.Layers {
blobPath := path.Join(srcDir, repoName, "blobs", string(blob.Digest.Algorithm()), blob.Digest.Hex())
err := os.Chmod(blobPath, 0o000)
So(err, ShouldBeNil)
}
resp, _ = resty.R().Post(destBaseURL + "/sync")
So(resp, ShouldNotBeNil)
So(resp.StatusCode(), ShouldEqual, 500)
for _, blob := range cm.Layers {
srcBlobPath := path.Join(srcDir, repoName, "blobs", string(blob.Digest.Algorithm()), blob.Digest.Hex())
err := os.Chmod(srcBlobPath, 0o755)
So(err, ShouldBeNil)
destBlobPath := path.Join(destDir, repoName, "blobs", string(blob.Digest.Algorithm()), blob.Digest.Hex())
err = os.Remove(destBlobPath)
So(err, ShouldBeNil)
err = os.MkdirAll(destBlobPath, 0o000)
So(err, ShouldBeNil)
}
resp, _ = resty.R().Post(destBaseURL + "/sync")
So(resp, ShouldNotBeNil)
So(resp.StatusCode(), ShouldEqual, 500)
for _, blob := range cm.Layers {
destBlobPath := path.Join(destDir, repoName, "blobs", string(blob.Digest.Algorithm()), blob.Digest.Hex())
err = os.Chmod(destBlobPath, 0o755)
So(err, ShouldBeNil)
err = os.Remove(destBlobPath)
So(err, ShouldBeNil)
}
// trigger error on config blob
srcConfigBlobPath := path.Join(srcDir, repoName, "blobs", string(cm.Config.Digest.Algorithm()),
cm.Config.Digest.Hex())
err = os.Chmod(srcConfigBlobPath, 0o000)
So(err, ShouldBeNil)
resp, _ = resty.R().Post(destBaseURL + "/sync")
So(resp, ShouldNotBeNil)
So(resp.StatusCode(), ShouldEqual, 500)
err = os.Chmod(srcConfigBlobPath, 0o755)
So(err, ShouldBeNil)
destConfigBlobPath := path.Join(destDir, repoName, "blobs", string(cm.Config.Digest.Algorithm()),
cm.Config.Digest.Hex())
err = os.Remove(destConfigBlobPath)
So(err, ShouldBeNil)
err = os.MkdirAll(destConfigBlobPath, 0o000)
So(err, ShouldBeNil)
resp, _ = resty.R().Post(destBaseURL + "/sync")
So(resp, ShouldNotBeNil)
So(resp.StatusCode(), ShouldEqual, 500)
})
}
func TestSyncError(t *testing.T) {
Convey("Verify periodically sync pushSyncedLocalImage() error", t, func() {
updateDuration, _ := time.ParseDuration("30m")
sc, srcBaseURL, srcDir, _, _ := startUpstreamServer(false, false)
defer os.RemoveAll(srcDir)
defer func() {
sc.Shutdown()
}()
regex := ".*"
semver := true
var tlsVerify bool
syncRegistryConfig := sync.RegistryConfig{
Content: []sync.Content{
{
Prefix: testImage,
Tags: &sync.Tags{
Regex: &regex,
Semver: &semver,
},
},
},
URL: srcBaseURL,
PollInterval: updateDuration,
TLSVerify: &tlsVerify,
CertDir: "",
}
syncConfig := &sync.Config{Registries: []sync.RegistryConfig{syncRegistryConfig}}
dc, destBaseURL, destDir, destClient := startDownstreamServer(false, syncConfig)
defer os.RemoveAll(destDir)
defer func() {
dc.Shutdown()
}()
// give permission denied on pushSyncedLocalImage()
localRepoPath := path.Join(destDir, testImage, "blobs")
err := os.MkdirAll(localRepoPath, 0o000)
So(err, ShouldBeNil)
resp, _ := destClient.R().Post(destBaseURL + "/sync")
So(resp, ShouldNotBeNil)
So(resp.StatusCode(), ShouldEqual, 500)
})
}
func TestSyncSignaturesOnDemand(t *testing.T) {
Convey("Verify sync signatures on demand feature", t, func() {
sc, srcBaseURL, srcDir, _, _ := startUpstreamServer(false, false)
defer os.RemoveAll(srcDir)
defer func() {
sc.Shutdown()
}()
// create repo, push and sign it
repoName := "signed-repo"
var digest godigest.Digest
So(func() { digest = pushRepo(srcBaseURL, repoName) }, ShouldNotPanic)
splittedURL := strings.SplitAfter(srcBaseURL, ":")
srcPort := splittedURL[len(splittedURL)-1]
cwd, err := os.Getwd()
So(err, ShouldBeNil)
defer func() { _ = os.Chdir(cwd) }()
tdir, err := ioutil.TempDir("", "sigs")
So(err, ShouldBeNil)
defer os.RemoveAll(tdir)
_ = os.Chdir(tdir)
So(func() { signImage(tdir, srcPort, repoName, digest) }, ShouldNotPanic)
var tlsVerify bool
syncRegistryConfig := sync.RegistryConfig{
URL: srcBaseURL,
TLSVerify: &tlsVerify,
CertDir: "",
OnDemand: true,
}
syncConfig := &sync.Config{Registries: []sync.RegistryConfig{syncRegistryConfig}}
dc, destBaseURL, destDir, _ := startDownstreamServer(false, syncConfig)
defer os.RemoveAll(destDir)
defer func() {
dc.Shutdown()
}()
// sync on demand
resp, err := resty.R().Get(destBaseURL + "/v2/" + repoName + "/manifests/1.0")
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 200)
splittedURL = strings.SplitAfter(destBaseURL, ":")
destPort := splittedURL[len(splittedURL)-1]
a := &options.AnnotationOptions{Annotations: []string{"tag=1.0"}}
amap, err := a.AnnotationsMap()
if err != nil {
panic(err)
}
// notation verify the synced image
image := fmt.Sprintf("localhost:%s/%s:%s", destPort, repoName, "1.0")
cmd := exec.Command("notation", "verify", "--cert", "good", "--plain-http", image)
out, err := cmd.CombinedOutput()
So(err, ShouldBeNil)
msg := string(out)
So(msg, ShouldNotBeEmpty)
So(strings.Contains(msg, "verification failure"), ShouldBeFalse)
// cosign verify the synced image
vrfy := verify.VerifyCommand{
RegistryOptions: options.RegistryOptions{AllowInsecure: true},
CheckClaims: true,
KeyRef: path.Join(tdir, "cosign.pub"),
Annotations: amap,
}
err = vrfy.Exec(context.TODO(), []string{fmt.Sprintf("localhost:%s/%s:%s", destPort, repoName, "1.0")})
So(err, ShouldBeNil)
// test negative case
cosignEncodedDigest := strings.Replace(digest.String(), ":", "-", 1) + ".sig"
getCosignManifestURL := srcBaseURL + path.Join("/v2", repoName, "manifests", cosignEncodedDigest)
mResp, err := resty.R().Get(getCosignManifestURL)
So(err, ShouldBeNil)
var cm ispec.Manifest
err = json.Unmarshal(mResp.Body(), &cm)
So(err, ShouldBeNil)
// trigger error on config blob
srcConfigBlobPath := path.Join(srcDir, repoName, "blobs", string(cm.Config.Digest.Algorithm()),
cm.Config.Digest.Hex())
err = os.Chmod(srcConfigBlobPath, 0o000)
So(err, ShouldBeNil)
// remove already synced image
err = os.RemoveAll(path.Join(destDir, repoName))
So(err, ShouldBeNil)
// sync on demand
resp, err = resty.R().Get(destBaseURL + "/v2/" + repoName + "/manifests/1.0")
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 404)
})
}
func signImage(tdir, port, repoName string, digest godigest.Digest) {
// push signatures to upstream server so that we can sync them later
// generate a keypair
os.Setenv("COSIGN_PASSWORD", "")
err := generate.GenerateKeyPairCmd(context.TODO(), "", nil)
if err != nil {
panic(err)
}
// sign the image
err = sign.SignCmd(context.TODO(),
sign.KeyOpts{KeyRef: path.Join(tdir, "cosign.key"), PassFunc: generate.GetPass},
options.RegistryOptions{AllowInsecure: true},
map[string]interface{}{"tag": "1.0"},
[]string{fmt.Sprintf("localhost:%s/%s@%s", port, repoName, digest.String())},
"", true, "", false, false, "")
if err != nil {
panic(err)
}
// verify the image
a := &options.AnnotationOptions{Annotations: []string{"tag=1.0"}}
amap, err := a.AnnotationsMap()
if err != nil {
panic(err)
}
vrfy := verify.VerifyCommand{
RegistryOptions: options.RegistryOptions{AllowInsecure: true},
CheckClaims: true,
KeyRef: path.Join(tdir, "cosign.pub"),
Annotations: amap,
}
err = vrfy.Exec(context.TODO(), []string{fmt.Sprintf("localhost:%s/%s:%s", port, repoName, "1.0")})
if err != nil {
panic(err)
}
// "notation" (notaryv2) doesn't yet support exported apis, so use the binary instead
_, err = exec.LookPath("notation")
if err != nil {
panic(err)
}
os.Setenv("XDG_CONFIG_HOME", tdir)
// generate a keypair
cmd := exec.Command("notation", "cert", "generate-test", "--trust", "good")
err = cmd.Run()
if err != nil {
panic(err)
}
// sign the image
image := fmt.Sprintf("localhost:%s/%s:%s", port, repoName, "1.0")
cmd = exec.Command("notation", "sign", "--key", "good", "--plain-http", image)
err = cmd.Run()
if err != nil {
panic(err)
}
// verify the image
cmd = exec.Command("notation", "verify", "--cert", "good", "--plain-http", image)
out, err := cmd.CombinedOutput()
So(err, ShouldBeNil)
msg := string(out)
So(msg, ShouldNotBeEmpty)
So(strings.Contains(msg, "verification failure"), ShouldBeFalse)
}
func pushRepo(url, repoName string) godigest.Digest {
// create a blob/layer
resp, err := resty.R().Post(url + fmt.Sprintf("/v2/%s/blobs/uploads/", repoName))
if err != nil {
panic(err)
}
loc := Location(url, resp)
_, err = resty.R().Get(loc)
if err != nil {
panic(err)
}
content := []byte("this is a blob")
digest := godigest.FromBytes(content)
_, err = resty.R().SetQueryParam("digest", digest.String()).
SetHeader("Content-Type", "application/octet-stream").SetBody(content).Put(loc)
if err != nil {
panic(err)
}
// create a manifest
manifest := ispec.Manifest{
Config: ispec.Descriptor{
Digest: digest,
Size: int64(len(content)),
},
Layers: []ispec.Descriptor{
{
MediaType: "application/vnd.oci.image.layer.v1.tar",
Digest: digest,
Size: int64(len(content)),
},
},
}
manifest.SchemaVersion = 2
content, err = json.Marshal(manifest)
if err != nil {
panic(err)
}
digest = godigest.FromBytes(content)
_, err = resty.R().SetHeader("Content-Type", "application/vnd.oci.image.manifest.v1+json").
SetBody(content).Put(url + fmt.Sprintf("/v2/%s/manifests/1.0", repoName))
if err != nil {
panic(err)
}
return digest
}

325
pkg/extensions/sync/utils.go
View File

@ -1,8 +1,11 @@
package sync
import (
"crypto/tls"
"crypto/x509"
"encoding/json"
"io/ioutil"
"net/url"
"os"
"path"
"strings"
@ -10,13 +13,20 @@ import (
glob "github.com/bmatcuk/doublestar/v4"
"github.com/containers/image/v5/docker/reference"
"github.com/containers/image/v5/types"
"github.com/notaryproject/notation-go-lib"
ispec "github.com/opencontainers/image-spec/specs-go/v1"
artifactspec "github.com/oras-project/artifacts-spec/specs-go/v1"
"gopkg.in/resty.v1"
"zotregistry.io/zot/errors"
"zotregistry.io/zot/pkg/extensions/monitoring"
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/storage"
)
type ReferenceList struct {
References []notation.Descriptor `json:"references"`
}
// getTagFromRef returns a tagged reference from an image reference.
func getTagFromRef(ref types.ImageReference, log log.Logger) reference.Tagged {
tagged, isTagged := ref.DockerReference().(reference.Tagged)
@ -91,6 +101,311 @@ func getFileCredentials(filepath string) (CredentialsFile, error) {
return creds, nil
}
func getHTTPClient(regCfg *RegistryConfig, credentials Credentials, log log.Logger) (*resty.Client, error) {
client := resty.New()
if regCfg.CertDir != "" {
log.Debug().Msgf("sync: using certs directory: %s", regCfg.CertDir)
clientCert := path.Join(regCfg.CertDir, "client.cert")
clientKey := path.Join(regCfg.CertDir, "client.key")
caCertPath := path.Join(regCfg.CertDir, "ca.crt")
caCert, err := ioutil.ReadFile(caCertPath)
if err != nil {
log.Error().Err(err).Msg("couldn't read CA certificate")
return nil, err
}
caCertPool := x509.NewCertPool()
caCertPool.AppendCertsFromPEM(caCert)
client.SetTLSClientConfig(&tls.Config{RootCAs: caCertPool, MinVersion: tls.VersionTLS12})
cert, err := tls.LoadX509KeyPair(clientCert, clientKey)
if err != nil {
log.Error().Err(err).Msg("couldn't read certificates key pairs")
return nil, err
}
client.SetCertificates(cert)
}
// nolint: gosec
if regCfg.TLSVerify != nil && !*regCfg.TLSVerify {
client.SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true})
}
if credentials.Username != "" && credentials.Password != "" {
log.Debug().Msgf("sync: using basic auth")
client.SetBasicAuth(credentials.Username, credentials.Password)
}
return client, nil
}
func syncCosignSignature(client *resty.Client, storeController storage.StoreController,
regURL url.URL, repo, digest string, log log.Logger) error {
log.Info().Msg("syncing cosign signatures")
getCosignManifestURL := regURL
cosignEncodedDigest := strings.Replace(digest, ":", "-", 1) + ".sig"
getCosignManifestURL.Path = path.Join(getCosignManifestURL.Path, "v2", repo, "manifests", cosignEncodedDigest)
getCosignManifestURL.RawQuery = getCosignManifestURL.Query().Encode()
mResp, err := client.R().Get(getCosignManifestURL.String())
if err != nil {
log.Error().Err(err).Str("url", getCosignManifestURL.String()).
Msgf("couldn't get cosign manifest: %s", cosignEncodedDigest)
return err
}
if mResp.IsError() {
log.Info().Msgf("couldn't find any cosign signature from %s, status code: %d skipping",
getCosignManifestURL.String(), mResp.StatusCode())
return nil
}
var m ispec.Manifest
err = json.Unmarshal(mResp.Body(), &m)
if err != nil {
log.Error().Err(err).Str("url", getCosignManifestURL.String()).
Msgf("couldn't unmarshal cosign manifest %s", cosignEncodedDigest)
return err
}
imageStore := storeController.GetImageStore(repo)
for _, blob := range m.Layers {
// get blob
getBlobURL := regURL
getBlobURL.Path = path.Join(getBlobURL.Path, "v2", repo, "blobs", blob.Digest.String())
getBlobURL.RawQuery = getBlobURL.Query().Encode()
resp, err := client.R().SetDoNotParseResponse(true).Get(getBlobURL.String())
if err != nil {
log.Error().Err(err).Msgf("couldn't get cosign blob: %s", blob.Digest.String())
return err
}
if resp.IsError() {
log.Info().Msgf("couldn't find cosign blob from %s, status code: %d", getBlobURL.String(), resp.StatusCode())
return errors.ErrBadBlobDigest
}
defer resp.RawBody().Close()
// push blob
_, _, err = imageStore.FullBlobUpload(repo, resp.RawBody(), blob.Digest.String())
if err != nil {
log.Error().Err(err).Msg("couldn't upload cosign blob")
return err
}
}
// get config blob
getBlobURL := regURL
getBlobURL.Path = path.Join(getBlobURL.Path, "v2", repo, "blobs", m.Config.Digest.String())
getBlobURL.RawQuery = getBlobURL.Query().Encode()
resp, err := client.R().SetDoNotParseResponse(true).Get(getBlobURL.String())
if err != nil {
log.Error().Err(err).Msgf("couldn't get cosign config blob: %s", getBlobURL.String())
return err
}
if resp.IsError() {
log.Info().Msgf("couldn't find cosign config blob from %s, status code: %d", getBlobURL.String(), resp.StatusCode())
return errors.ErrBadBlobDigest
}
defer resp.RawBody().Close()
// push config blob
_, _, err = imageStore.FullBlobUpload(repo, resp.RawBody(), m.Config.Digest.String())
if err != nil {
log.Error().Err(err).Msg("couldn't upload cosign blob")
return err
}
// push manifest
_, err = imageStore.PutImageManifest(repo, cosignEncodedDigest, ispec.MediaTypeImageManifest, mResp.Body())
if err != nil {
log.Error().Err(err).Msg("couldn't upload cosing manifest")
return err
}
return nil
}
func syncNotarySignature(client *resty.Client, storeController storage.StoreController,
regURL url.URL, repo, digest string, log log.Logger) error {
log.Info().Msg("syncing notary signatures")
getReferrersURL := regURL
getRefManifestURL := regURL
// based on manifest digest get referrers
getReferrersURL.Path = path.Join(getReferrersURL.Path, "oras/artifacts/v1/", repo, "manifests", digest, "referrers")
getReferrersURL.RawQuery = getReferrersURL.Query().Encode()
resp, err := client.R().
SetHeader("Content-Type", "application/json").
SetQueryParam("artifactType", "application/vnd.cncf.notary.v2.signature").
Get(getReferrersURL.String())
if err != nil {
log.Error().Err(err).Msgf("couldn't get referrers from %s", getReferrersURL.String())
return err
}
if resp.IsError() {
log.Info().Msgf("couldn't find any notary signature from %s, status code: %d, skipping",
getReferrersURL.String(), resp.StatusCode())
return nil
}
var referrers ReferenceList
err = json.Unmarshal(resp.Body(), &referrers)
if err != nil {
log.Error().Err(err).Msgf("couldn't unmarshal notary signature from %s", getReferrersURL.String())
return err
}
imageStore := storeController.GetImageStore(repo)
for _, ref := range referrers.References {
// get referrer manifest
getRefManifestURL.Path = path.Join(getRefManifestURL.Path, "v2", repo, "manifests", ref.Digest.String())
getRefManifestURL.RawQuery = getRefManifestURL.Query().Encode()
resp, err := client.R().
Get(getRefManifestURL.String())
if err != nil {
log.Error().Err(err).Msgf("couldn't get notary manifest: %s", getRefManifestURL.String())
return err
}
// read manifest
var m artifactspec.Manifest
err = json.Unmarshal(resp.Body(), &m)
if err != nil {
log.Error().Err(err).Msgf("couldn't unmarshal notary manifest: %s", getRefManifestURL.String())
return err
}
for _, blob := range m.Blobs {
getBlobURL := regURL
getBlobURL.Path = path.Join(getBlobURL.Path, "v2", repo, "blobs", blob.Digest.String())
getBlobURL.RawQuery = getBlobURL.Query().Encode()
resp, err := client.R().SetDoNotParseResponse(true).Get(getBlobURL.String())
if err != nil {
log.Error().Err(err).Msgf("couldn't get notary blob: %s", getBlobURL.String())
return err
}
defer resp.RawBody().Close()
if resp.IsError() {
log.Info().Msgf("couldn't find notary blob from %s, status code: %d",
getBlobURL.String(), resp.StatusCode())
return errors.ErrBadBlobDigest
}
_, _, err = imageStore.FullBlobUpload(repo, resp.RawBody(), blob.Digest.String())
if err != nil {
log.Error().Err(err).Msg("couldn't upload notary sig blob")
return err
}
}
_, err = imageStore.PutImageManifest(repo, ref.Digest.String(), artifactspec.MediaTypeArtifactManifest, resp.Body())
if err != nil {
log.Error().Err(err).Msg("couldn't upload notary sig manifest")
return err
}
}
return nil
}
func syncSignatures(client *resty.Client, storeController storage.StoreController,
registryURL, repo, tag string, log log.Logger) error {
log.Info().Msg("syncing signatures")
// get manifest and find out its digest
regURL, err := url.Parse(registryURL)
if err != nil {
log.Error().Err(err).Msgf("couldn't parse registry URL: %s", registryURL)
return err
}
getManifestURL := *regURL
getManifestURL.Path = path.Join(getManifestURL.Path, "v2", repo, "manifests", tag)
resp, err := client.R().SetHeader("Content-Type", "application/json").Head(getManifestURL.String())
if err != nil {
log.Error().Err(err).Str("url", getManifestURL.String()).
Msgf("couldn't query %s", registryURL)
return err
}
digests, ok := resp.Header()["Docker-Content-Digest"]
if !ok {
log.Error().Err(errors.ErrBadBlobDigest).Str("url", getManifestURL.String()).
Msgf("couldn't get digest for manifest: %s:%s", repo, tag)
return errors.ErrBadBlobDigest
}
if len(digests) != 1 {
log.Error().Err(errors.ErrBadBlobDigest).Str("url", getManifestURL.String()).
Msgf("multiple digests found for: %s:%s", repo, tag)
return errors.ErrBadBlobDigest
}
err = syncNotarySignature(client, storeController, *regURL, repo, digests[0], log)
if err != nil {
return err
}
err = syncCosignSignature(client, storeController, *regURL, repo, digests[0], log)
if err != nil {
return err
}
log.Info().Msg("successfully synced signatures")
return nil
}
func pushSyncedLocalImage(repo, tag, uuid string,
storeController storage.StoreController, log log.Logger) error {
log.Info().Msgf("pushing synced local image %s:%s to local registry", repo, tag)
@ -165,3 +480,13 @@ func pushSyncedLocalImage(repo, tag, uuid string,
return nil
}
// sync feature will try to pull cosign signature because for sync cosign signature is just an image
// this function will check if tag is a cosign tag.
func isCosignTag(tag string) bool {
if strings.HasPrefix(tag, "sha256-") && strings.HasSuffix(tag, ".sig") {
return true
}
return false
}

3
pkg/storage/cache_test.go
View File

@ -37,6 +37,9 @@ func TestCache(t *testing.T) {
err = cache.PutBlob("key", path.Join(dir, "value"))
So(err, ShouldBeNil)
err = cache.PutBlob("key", "value")
So(err, ShouldNotBeNil)
exists = cache.HasBlob("key", "value")
So(exists, ShouldBeTrue)