alt/zot-wo-auth
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

config: support multiple storage locations

Browse Source 
added support to point multiple storage locations in zot by running multiple instance of zot in background.

see examples/config-multiple.json for more info about config.

Closes #181
This commit is contained in:
Shivam Mishra 2021-04-05 17:40:33 -07:00 committed by Ramkumar Chinchani
parent 9ca6eea940
commit 28974e81dc
21 changed files with 1810 additions and 164 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
README.md
View File

@ -23,6 +23,7 @@ https://anuvu.github.io/zot/
* Storage optimizations:
* Automatic garbage collection of orphaned blobs
* Layer deduplication using hard links when content is identical
* Serve [multiple storage paths (and backends)](./examples/config-multiple.json) using a single zot server
* Swagger based documentation
* Single binary for _all_ the above features
* Released under Apache 2.0 License

2
errors/errors.go
View File

@ -37,4 +37,6 @@ var (
ErrScanNotSupported = errors.New("search: scanning of image media type not supported")
ErrCLITimeout = errors.New("cli: Query timed out while waiting for results")
ErrDuplicateConfigName = errors.New("cli: cli config name already added")
ErrInvalidRoute = errors.New("routes: invalid route prefix")
ErrImgStoreNotFound = errors.New("routes: image store not found corresponding to given route")
)

37
examples/config-multiple-cve.json Normal file
View File

@ -0,0 +1,37 @@
{
"version": "0.1.0-dev",
"storage": {
"rootDirectory": "/tmp/zot",
"dedupe": true,
"gc": true,
"subPaths": {
"/infra": {
"rootDirectory": "/tmp/zot1",
"dedupe": true
},
"/b": {
"rootDirectory": "/tmp/zot2",
"dedupe": true
},
"/c": {
"rootDirectory": "/tmp/zot3",
"dedupe": false
}
}
},
"http": {
"address": "127.0.0.1",
"port": "5000",
"ReadOnly": false
},
"log": {
"level": "debug"
},
"extensions": {
"search": {
"cve": {
"updateInterval": "24h"
}
}
}
}

30
examples/config-multiple.json Normal file
View File

@ -0,0 +1,30 @@
{
"version": "0.1.0-dev",
"storage": {
"rootDirectory": "/tmp/zot",
"dedupe": true,
"gc": true,
"subPaths": {
"/a": {
"rootDirectory": "/tmp/zot1",
"dedupe": true
},
"/b": {
"rootDirectory": "/tmp/zot2",
"dedupe": true
},
"/c": {
"rootDirectory": "/tmp/zot3",
"dedupe": false
}
}
},
"http": {
"address": "127.0.0.1",
"port": "5000",
"ReadOnly": false
},
"log": {
"level": "debug"
}
}

11
go.mod
View File

@ -9,6 +9,7 @@ require (
github.com/aquasecurity/trivy v0.0.0-00010101000000-000000000000
github.com/briandowns/spinner v1.11.1
github.com/chartmuseum/auth v0.4.0
github.com/containers/storage v1.29.0
github.com/dustin/go-humanize v1.0.0
github.com/getlantern/deepcopy v0.0.0-20160317154340-7f45deb8130a
github.com/go-chi/chi v4.0.2+incompatible // indirect
@ -17,7 +18,7 @@ require (
github.com/google/go-containerregistry v0.0.0-20200331213917-3d03ed9b1ca2
github.com/gorilla/handlers v1.4.2
github.com/gorilla/mux v1.7.4
github.com/json-iterator/go v1.1.9
github.com/json-iterator/go v1.1.10
github.com/mitchellh/mapstructure v1.1.2
github.com/nmcclain/asn1-ber v0.0.0-20170104154839-2661553a0484 // indirect
github.com/nmcclain/ldap v0.0.0-20191021200707-3b3b69a7e9e3
@ -31,14 +32,14 @@ require (
github.com/smartystreets/goconvey v1.6.4
github.com/spf13/cobra v0.0.5
github.com/spf13/viper v1.6.1
github.com/stretchr/testify v1.6.1
github.com/stretchr/testify v1.7.0
github.com/swaggo/http-swagger v0.0.0-20190614090009-c2865af9083e
github.com/swaggo/swag v1.6.3
github.com/vektah/gqlparser/v2 v2.0.1
go.etcd.io/bbolt v1.3.4
golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9
go.etcd.io/bbolt v1.3.5
golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0
gopkg.in/resty.v1 v1.12.0
gopkg.in/yaml.v2 v2.2.8
gopkg.in/yaml.v2 v2.4.0
)
replace github.com/aquasecurity/trivy => github.com/anuvu/trivy v0.9.2-0.20200731014147-c5f97b59c172

513
go.sum
View File

File diff suppressed because it is too large Load Diff

11
pkg/api/config.go
View File

@ -72,11 +72,18 @@ type LogConfig struct {
Output string
}
type GlobalStorageConfig struct {
RootDirectory string
Dedupe bool
GC bool
SubPaths map[string]StorageConfig
}
type Config struct {
Version string
Commit string
BinaryType string
Storage StorageConfig
Storage GlobalStorageConfig
HTTP HTTPConfig
Log *LogConfig
Extensions *ext.ExtensionConfig
@ -87,7 +94,7 @@ func NewConfig() *Config {
Version: dspec.Version,
Commit: Commit,
BinaryType: BinaryType,
Storage: StorageConfig{GC: true, Dedupe: true},
Storage: GlobalStorageConfig{GC: true, Dedupe: true},
HTTP: HTTPConfig{Address: "127.0.0.1", Port: "8080"},
Log: &LogConfig{Level: "debug"},
}

84
pkg/api/controller.go
View File

@ -7,7 +7,6 @@ import (
"io/ioutil"
"net"
"net/http"
"os"
"time"
"github.com/anuvu/zot/errors"
@ -23,11 +22,11 @@ const (
)
type Controller struct {
Config *Config
Router *mux.Router
ImageStore *storage.ImageStore
Log log.Logger
Server *http.Server
Config *Config
Router *mux.Router
StoreController storage.StoreController
Log log.Logger
Server *http.Server
}
func NewController(config *Config) *Controller {
@ -63,20 +62,69 @@ func (c *Controller) Run() error {
handlers.RecoveryHandler(handlers.RecoveryLogger(c.Log),
handlers.PrintRecoveryStack(false)))
c.ImageStore = storage.NewImageStore(c.Config.Storage.RootDirectory, c.Config.Storage.GC,
c.Config.Storage.Dedupe, c.Log)
if c.ImageStore == nil {
// we can't proceed without at least a image store
os.Exit(1)
}
// Enable extensions if extension config is provided
if c.Config != nil && c.Config.Extensions != nil {
ext.EnableExtensions(c.Config.Extensions, c.Log, c.Config.Storage.RootDirectory)
}
c.Router = engine
c.Router.UseEncodedPath()
c.StoreController = storage.StoreController{}
if c.Config.Storage.RootDirectory != "" {
if c.Config.Storage.Dedupe {
err := storage.ValidateHardLink(c.Config.Storage.RootDirectory)
if err != nil {
c.Log.Warn().Msg("input storage root directory filesystem does not supports hardlinking," +
"disabling dedupe functionality")
c.Config.Storage.Dedupe = false
}
}
defaultStore := storage.NewImageStore(c.Config.Storage.RootDirectory,
c.Config.Storage.GC, c.Config.Storage.Dedupe, c.Log)
c.StoreController.DefaultStore = defaultStore
// Enable extensions if extension config is provided
if c.Config != nil && c.Config.Extensions != nil {
ext.EnableExtensions(c.Config.Extensions, c.Log, c.Config.Storage.RootDirectory)
}
} else {
// we can't proceed without global storage
c.Log.Error().Err(errors.ErrImgStoreNotFound).Msg("controller: no storage config provided")
return errors.ErrImgStoreNotFound
}
if c.Config.Storage.SubPaths != nil {
if len(c.Config.Storage.SubPaths) > 0 {
subPaths := c.Config.Storage.SubPaths
subImageStore := make(map[string]*storage.ImageStore)
// creating image store per subpaths
for route, storageConfig := range subPaths {
if storageConfig.Dedupe {
err := storage.ValidateHardLink(storageConfig.RootDirectory)
if err != nil {
c.Log.Warn().Msg("input storage root directory filesystem does not supports hardlinking, " +
"disabling dedupe functionality")
storageConfig.Dedupe = false
}
}
subImageStore[route] = storage.NewImageStore(storageConfig.RootDirectory,
storageConfig.GC, storageConfig.Dedupe, c.Log)
// Enable extensions if extension config is provided
if c.Config != nil && c.Config.Extensions != nil {
ext.EnableExtensions(c.Config.Extensions, c.Log, storageConfig.RootDirectory)
}
}
c.StoreController.SubStore = subImageStore
}
}
_ = NewRouteHandler(c)
addr := fmt.Sprintf("%s:%s", c.Config.HTTP.Address, c.Config.HTTP.Port)

243
pkg/api/controller_test.go
View File

@ -371,6 +371,136 @@ func TestBasicAuth(t *testing.T) {
})
}
func TestMultipleInstance(t *testing.T) {
Convey("Negative test zot multiple instance", t, func() {
config := api.NewConfig()
config.HTTP.Port = SecurePort1
htpasswdPath := makeHtpasswdFile()
defer os.Remove(htpasswdPath)
config.HTTP.Auth = &api.AuthConfig{
HTPasswd: api.AuthHTPasswd{
Path: htpasswdPath,
},
}
c := api.NewController(config)
err := c.Run()
So(err, ShouldEqual, errors.ErrImgStoreNotFound)
globalDir, err := ioutil.TempDir("", "oci-repo-test")
if err != nil {
panic(err)
}
defer os.RemoveAll(globalDir)
subDir, err := ioutil.TempDir("/tmp", "oci-sub-test")
if err != nil {
panic(err)
}
defer os.RemoveAll(subDir)
c.Config.Storage.RootDirectory = globalDir
subPathMap := make(map[string]api.StorageConfig)
subPathMap["/a"] = api.StorageConfig{RootDirectory: subDir}
go func() {
if err := c.Run(); err != nil {
return
}
}()
// wait till ready
for {
_, err := resty.R().Get(BaseURL1)
if err == nil {
break
}
time.Sleep(100 * time.Millisecond)
}
defer func() {
ctx := context.Background()
_ = c.Server.Shutdown(ctx)
}()
client := resty.New()
tagResponse, err := client.R().SetBasicAuth(username, passphrase).
Get(BaseURL1 + "/v2/zot-test/tags/list")
So(err, ShouldBeNil)
So(tagResponse.StatusCode(), ShouldEqual, 404)
})
Convey("Test zot multiple instance", t, func() {
config := api.NewConfig()
config.HTTP.Port = SecurePort1
htpasswdPath := makeHtpasswdFile()
defer os.Remove(htpasswdPath)
config.HTTP.Auth = &api.AuthConfig{
HTPasswd: api.AuthHTPasswd{
Path: htpasswdPath,
},
}
c := api.NewController(config)
globalDir, err := ioutil.TempDir("", "oci-repo-test")
if err != nil {
panic(err)
}
defer os.RemoveAll(globalDir)
subDir, err := ioutil.TempDir("/tmp", "oci-sub-test")
if err != nil {
panic(err)
}
defer os.RemoveAll(subDir)
c.Config.Storage.RootDirectory = globalDir
subPathMap := make(map[string]api.StorageConfig)
subPathMap["/a"] = api.StorageConfig{RootDirectory: subDir}
go func() {
// this blocks
if err := c.Run(); err != nil {
return
}
}()
// wait till ready
for {
_, err := resty.R().Get(BaseURL1)
if err == nil {
break
}
time.Sleep(100 * time.Millisecond)
}
defer func() {
ctx := context.Background()
_ = c.Server.Shutdown(ctx)
}()
// without creds, should get access error
resp, err := resty.R().Get(BaseURL1 + "/v2/")
So(err, ShouldBeNil)
So(resp, ShouldNotBeNil)
So(resp.StatusCode(), ShouldEqual, 401)
var e api.Error
err = json.Unmarshal(resp.Body(), &e)
So(err, ShouldBeNil)
// with creds, should get expected status code
resp, _ = resty.R().SetBasicAuth(username, passphrase).Get(BaseURL1)
So(resp, ShouldNotBeNil)
So(resp.StatusCode(), ShouldEqual, 404)
resp, _ = resty.R().SetBasicAuth(username, passphrase).Get(BaseURL1 + "/v2/")
So(resp, ShouldNotBeNil)
So(resp.StatusCode(), ShouldEqual, 200)
})
}
func TestTLSWithBasicAuth(t *testing.T) {
Convey("Make a new controller", t, func() {
caCert, err := ioutil.ReadFile(CACert)
@ -1820,13 +1950,13 @@ func TestParallelRequests(t *testing.T) {
{
srcImageName: "zot-cve-test",
srcImageTag: "0.0.1",
destImageName: "zot-3-test",
destImageName: "a/zot-3-test",
testCaseName: "Request-3",
},
{
srcImageName: "zot-cve-test",
srcImageTag: "0.0.1",
destImageName: "zot-4-test",
destImageName: "b/zot-4-test",
testCaseName: "Request-4",
},
{
@ -1881,13 +2011,13 @@ func TestParallelRequests(t *testing.T) {
{
srcImageName: "zot-cve-test",
srcImageTag: "0.0.1",
destImageName: "zot-3-test",
destImageName: "a/zot-3-test",
testCaseName: "Request-13",
},
{
srcImageName: "zot-cve-test",
srcImageTag: "0.0.1",
destImageName: "zot-4-test",
destImageName: "b/zot-4-test",
testCaseName: "Request-14",
},
}
@ -1896,8 +2026,6 @@ func TestParallelRequests(t *testing.T) {
config.HTTP.Port = SecurePort1
htpasswdPath := makeHtpasswdFileFromString(getCredString(username, passphrase))
// defer os.Remove(htpasswdPath)
config.HTTP.Auth = &api.AuthConfig{
HTPasswd: api.AuthHTPasswd{
Path: htpasswdPath,
@ -1911,11 +2039,23 @@ func TestParallelRequests(t *testing.T) {
panic(err)
}
err = copyFiles("../../test/data", dir)
firstSubDir, err := ioutil.TempDir("", "oci-sub-dir")
if err != nil {
panic(err)
}
//defer os.RemoveAll(dir)
secondSubDir, err := ioutil.TempDir("", "oci-sub-dir")
if err != nil {
panic(err)
}
subPaths := make(map[string]api.StorageConfig)
subPaths["/a"] = api.StorageConfig{RootDirectory: firstSubDir}
subPaths["/b"] = api.StorageConfig{RootDirectory: secondSubDir}
c.Config.Storage.SubPaths = subPaths
c.Config.Storage.RootDirectory = dir
go func() {
@ -1939,7 +2079,7 @@ func TestParallelRequests(t *testing.T) {
for i, testcase := range testCases {
testcase := testcase
j := i
//println(i)
t.Run(testcase.testCaseName, func(t *testing.T) {
t.Parallel()
client := resty.New()
@ -1949,7 +2089,7 @@ func TestParallelRequests(t *testing.T) {
assert.Equal(t, err, nil, "Error should be nil")
assert.NotEqual(t, tagResponse.StatusCode(), 400, "bad request")
manifestList := getAllManifests(path.Join(c.Config.Storage.RootDirectory, testcase.srcImageName))
manifestList := getAllManifests(path.Join("../../test/data", testcase.srcImageName))
for _, manifest := range manifestList {
headResponse, err := client.R().SetBasicAuth(username, passphrase).
@ -1963,7 +2103,7 @@ func TestParallelRequests(t *testing.T) {
assert.Equal(t, getResponse.StatusCode(), 404, "response status code should return 404")
}
blobList := getAllBlobs(path.Join(c.Config.Storage.RootDirectory, testcase.srcImageName))
blobList := getAllBlobs(path.Join("../../test/data", testcase.srcImageName))
for _, blob := range blobList {
// Get request of blob
@ -1981,7 +2121,7 @@ func TestParallelRequests(t *testing.T) {
assert.Equal(t, err, nil, "Should not be nil")
assert.NotEqual(t, getResponse.StatusCode(), 500, "internal server error should not occurred")
blobPath := path.Join(c.Config.Storage.RootDirectory, testcase.srcImageName, "blobs/sha256", blob)
blobPath := path.Join("../../test/data", testcase.srcImageName, "blobs/sha256", blob)
buf, err := ioutil.ReadFile(blobPath)
if err != nil {
@ -2050,9 +2190,6 @@ func TestParallelRequests(t *testing.T) {
SetHeader("Content-Range", fmt.Sprintf("%d", readContent)+"-"+fmt.Sprintf("%d", readContent+n-1)).
SetBasicAuth(username, passphrase).
Patch(BaseURL1 + "/v2/" + testcase.destImageName + "/blobs/uploads/" + sessionID)
if err != nil {
panic(err)
}
assert.Equal(t, err, nil, "Error should be nil")
assert.NotEqual(t, patchResponse.StatusCode(), 500, "response status code should not return 500")
@ -2273,3 +2410,79 @@ func stopServer(ctrl *api.Controller) {
panic(err)
}
}
func TestHardLink(t *testing.T) {
Convey("Validate hard link", t, func() {
config := api.NewConfig()
config.HTTP.Port = SecurePort1
htpasswdPath := makeHtpasswdFileFromString(getCredString(username, passphrase))
config.HTTP.Auth = &api.AuthConfig{
HTPasswd: api.AuthHTPasswd{
Path: htpasswdPath,
},
}
c := api.NewController(config)
dir, err := ioutil.TempDir("", "hard-link-test")
if err != nil {
panic(err)
}
defer os.RemoveAll(dir)
err = os.Chmod(dir, 0400)
if err != nil {
panic(err)
}
subDir, err := ioutil.TempDir("", "sub-hardlink-test")
if err != nil {
panic(err)
}
defer os.RemoveAll(subDir)
err = os.Chmod(subDir, 0400)
if err != nil {
panic(err)
}
c.Config.Storage.RootDirectory = dir
subPaths := make(map[string]api.StorageConfig)
subPaths["/a"] = api.StorageConfig{RootDirectory: subDir, Dedupe: true}
c.Config.Storage.SubPaths = subPaths
go func() {
// this blocks
if err := c.Run(); err != nil {
return
}
}()
time.Sleep(5 * time.Second)
// wait till ready
for {
_, err := resty.R().Get(BaseURL1)
if err == nil {
break
}
time.Sleep(100 * time.Millisecond)
}
err = os.Chmod(dir, 0644)
if err != nil {
panic(err)
}
err = os.Chmod(subDir, 0644)
if err != nil {
panic(err)
}
So(c.Config.Storage.Dedupe, ShouldEqual, false)
})
}

113
pkg/api/routes.go
View File

@ -25,6 +25,7 @@ import (
"github.com/anuvu/zot/errors"
ext "github.com/anuvu/zot/pkg/extensions"
"github.com/anuvu/zot/pkg/log"
"github.com/anuvu/zot/pkg/storage"
"github.com/gorilla/mux"
jsoniter "github.com/json-iterator/go"
ispec "github.com/opencontainers/image-spec/specs-go/v1"
@ -90,7 +91,7 @@ func (rh *RouteHandler) SetupRoutes() {
rh.c.Router.PathPrefix("/swagger/v2/").Methods("GET").Handler(httpSwagger.WrapHandler)
// Setup Extensions Routes
if rh.c.Config != nil && rh.c.Config.Extensions != nil {
ext.SetupRoutes(rh.c.Router, rh.c.Config.Storage.RootDirectory, rh.c.ImageStore, rh.c.Log)
ext.SetupRoutes(rh.c.Router, rh.c.StoreController, rh.c.Log)
}
}
@ -139,6 +140,7 @@ type ImageTags struct {
// @Failure 400 {string} string "bad request".
func (rh *RouteHandler) ListTags(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
name, ok := vars["name"]
if !ok || name == "" {
@ -146,11 +148,11 @@ func (rh *RouteHandler) ListTags(w http.ResponseWriter, r *http.Request) {
return
}
is := rh.getImageStore(name)
paginate := false
n := -1
var err error
nQuery, ok := r.URL.Query()["n"]
if ok {
@ -161,6 +163,8 @@ func (rh *RouteHandler) ListTags(w http.ResponseWriter, r *http.Request) {
var n1 int64
var err error
if n1, err = strconv.ParseInt(nQuery[0], 10, 0); err != nil {
w.WriteHeader(http.StatusBadRequest)
return
@ -182,7 +186,7 @@ func (rh *RouteHandler) ListTags(w http.ResponseWriter, r *http.Request) {
last = lastQuery[0]
}
tags, err := rh.c.ImageStore.GetImageTags(name)
tags, err := is.GetImageTags(name)
if err != nil {
WriteJSON(w, http.StatusNotFound, NewErrorList(NewError(NAME_UNKNOWN, map[string]string{"name": name})))
return
@ -255,13 +259,15 @@ func (rh *RouteHandler) CheckManifest(w http.ResponseWriter, r *http.Request) {
return
}
is := rh.getImageStore(name)
reference, ok := vars["reference"]
if !ok || reference == "" {
WriteJSON(w, http.StatusNotFound, NewErrorList(NewError(MANIFEST_INVALID, map[string]string{"reference": reference})))
return
}
_, digest, _, err := rh.c.ImageStore.GetImageManifest(name, reference)
_, digest, _, err := is.GetImageManifest(name, reference)
if err != nil {
switch err {
case errors.ErrRepoNotFound:
@ -310,13 +316,15 @@ func (rh *RouteHandler) GetManifest(w http.ResponseWriter, r *http.Request) {
return
}
is := rh.getImageStore(name)
reference, ok := vars["reference"]
if !ok || reference == "" {
WriteJSON(w, http.StatusNotFound, NewErrorList(NewError(MANIFEST_UNKNOWN, map[string]string{"reference": reference})))
return
}
content, digest, mediaType, err := rh.c.ImageStore.GetImageManifest(name, reference)
content, digest, mediaType, err := is.GetImageManifest(name, reference)
if err != nil {
switch err {
case errors.ErrRepoNotFound:
@ -362,6 +370,8 @@ func (rh *RouteHandler) UpdateManifest(w http.ResponseWriter, r *http.Request) {
return
}
is := rh.getImageStore(name)
reference, ok := vars["reference"]
if !ok || reference == "" {
WriteJSON(w, http.StatusNotFound, NewErrorList(NewError(MANIFEST_INVALID, map[string]string{"reference": reference})))
@ -382,7 +392,7 @@ func (rh *RouteHandler) UpdateManifest(w http.ResponseWriter, r *http.Request) {
return
}
digest, err := rh.c.ImageStore.PutImageManifest(name, reference, mediaType, body)
digest, err := is.PutImageManifest(name, reference, mediaType, body)
if err != nil {
switch err {
case errors.ErrRepoNotFound:
@ -428,13 +438,16 @@ func (rh *RouteHandler) DeleteManifest(w http.ResponseWriter, r *http.Request) {
return
}
is := rh.getImageStore(name)
reference, ok := vars["reference"]
if !ok || reference == "" {
w.WriteHeader(http.StatusNotFound)
return
}
err := rh.c.ImageStore.DeleteImageManifest(name, reference)
err := is.DeleteImageManifest(name, reference)
if err != nil {
switch err {
case errors.ErrRepoNotFound:
@ -476,6 +489,8 @@ func (rh *RouteHandler) CheckBlob(w http.ResponseWriter, r *http.Request) {
return
}
is := rh.getImageStore(name)
digest, ok := vars["digest"]
if !ok || digest == "" {
w.WriteHeader(http.StatusNotFound)
@ -484,7 +499,7 @@ func (rh *RouteHandler) CheckBlob(w http.ResponseWriter, r *http.Request) {
mediaType := r.Header.Get("Accept")
ok, blen, err := rh.c.ImageStore.CheckBlob(name, digest, mediaType)
ok, blen, err := is.CheckBlob(name, digest, mediaType)
if err != nil {
switch err {
case errors.ErrBadBlobDigest:
@ -530,6 +545,8 @@ func (rh *RouteHandler) GetBlob(w http.ResponseWriter, r *http.Request) {
return
}
is := rh.getImageStore(name)
digest, ok := vars["digest"]
if !ok || digest == "" {
w.WriteHeader(http.StatusNotFound)
@ -538,7 +555,7 @@ func (rh *RouteHandler) GetBlob(w http.ResponseWriter, r *http.Request) {
mediaType := r.Header.Get("Accept")
br, blen, err := rh.c.ImageStore.GetBlob(name, digest, mediaType)
br, blen, err := is.GetBlob(name, digest, mediaType)
if err != nil {
switch err {
case errors.ErrBadBlobDigest:
@ -576,16 +593,20 @@ func (rh *RouteHandler) DeleteBlob(w http.ResponseWriter, r *http.Request) {
if !ok || name == "" {
w.WriteHeader(http.StatusNotFound)
return
}
digest, ok := vars["digest"]
if !ok || digest == "" {
w.WriteHeader(http.StatusNotFound)
return
}
err := rh.c.ImageStore.DeleteBlob(name, digest)
is := rh.getImageStore(name)
err := is.DeleteBlob(name, digest)
if err != nil {
switch err {
case errors.ErrBadBlobDigest:
@ -626,6 +647,9 @@ func (rh *RouteHandler) CreateBlobUpload(w http.ResponseWriter, r *http.Request)
return
}
is := rh.getImageStore(name)
// currently zot does not support cross-repository mounting, following dist-spec and returning 202
if mountDigests, ok := r.URL.Query()["mount"]; ok {
if len(mountDigests) != 1 {
w.WriteHeader(http.StatusBadRequest)
@ -639,9 +663,9 @@ func (rh *RouteHandler) CreateBlobUpload(w http.ResponseWriter, r *http.Request)
}
// zot does not support cross mounting directly and do a workaround by copying blob using hard link
err := rh.c.ImageStore.MountBlob(name, from[0], mountDigests[0])
err := is.MountBlob(name, from[0], mountDigests[0])
if err != nil {
u, err := rh.c.ImageStore.NewBlobUpload(name)
u, err := is.NewBlobUpload(name)
if err != nil {
switch err {
case errors.ErrRepoNotFound:
@ -703,7 +727,7 @@ func (rh *RouteHandler) CreateBlobUpload(w http.ResponseWriter, r *http.Request)
return
}
sessionID, size, err := rh.c.ImageStore.FullBlobUpload(name, r.Body, digest)
sessionID, size, err := is.FullBlobUpload(name, r.Body, digest)
if err != nil {
rh.c.Log.Error().Err(err).Int64("actual", size).Int64("expected", contentLength).Msg("failed full upload")
w.WriteHeader(http.StatusInternalServerError)
@ -725,7 +749,7 @@ func (rh *RouteHandler) CreateBlobUpload(w http.ResponseWriter, r *http.Request)
return
}
u, err := rh.c.ImageStore.NewBlobUpload(name)
u, err := is.NewBlobUpload(name)
if err != nil {
switch err {
case errors.ErrRepoNotFound:
@ -765,13 +789,16 @@ func (rh *RouteHandler) GetBlobUpload(w http.ResponseWriter, r *http.Request) {
return
}
is := rh.getImageStore(name)
sessionID, ok := vars["session_id"]
if !ok || sessionID == "" {
w.WriteHeader(http.StatusNotFound)
return
}
size, err := rh.c.ImageStore.GetBlobUpload(name, sessionID)
size, err := is.GetBlobUpload(name, sessionID)
if err != nil {
switch err {
case errors.ErrBadUploadRange:
@ -824,19 +851,21 @@ func (rh *RouteHandler) PatchBlobUpload(w http.ResponseWriter, r *http.Request)
return
}
is := rh.getImageStore(name)
sessionID, ok := vars["session_id"]
if !ok || sessionID == "" {
w.WriteHeader(http.StatusNotFound)
return
}
var err error
var clen int64
var err error
if r.Header.Get("Content-Length") == "" || r.Header.Get("Content-Range") == "" {
// streamed blob upload
clen, err = rh.c.ImageStore.PutBlobChunkStreamed(name, sessionID, r.Body)
clen, err = is.PutBlobChunkStreamed(name, sessionID, r.Body)
} else {
// chunked blob upload
@ -863,7 +892,7 @@ func (rh *RouteHandler) PatchBlobUpload(w http.ResponseWriter, r *http.Request)
return
}
clen, err = rh.c.ImageStore.PutBlobChunk(name, sessionID, from, to, r.Body)
clen, err = is.PutBlobChunk(name, sessionID, from, to, r.Body)
}
if err != nil {
@ -916,6 +945,8 @@ func (rh *RouteHandler) UpdateBlobUpload(w http.ResponseWriter, r *http.Request)
return
}
is := rh.getImageStore(name)
sessionID, ok := vars["session_id"]
if !ok || sessionID == "" {
w.WriteHeader(http.StatusNotFound)
@ -969,7 +1000,7 @@ func (rh *RouteHandler) UpdateBlobUpload(w http.ResponseWriter, r *http.Request)
return
}
_, err = rh.c.ImageStore.PutBlobChunk(name, sessionID, from, to, r.Body)
_, err = is.PutBlobChunk(name, sessionID, from, to, r.Body)
if err != nil {
switch err {
case errors.ErrBadUploadRange:
@ -992,7 +1023,7 @@ func (rh *RouteHandler) UpdateBlobUpload(w http.ResponseWriter, r *http.Request)
finish:
// blob chunks already transferred, just finish
if err := rh.c.ImageStore.FinishBlobUpload(name, sessionID, r.Body, digest); err != nil {
if err := is.FinishBlobUpload(name, sessionID, r.Body, digest); err != nil {
switch err {
case errors.ErrBadBlobDigest:
WriteJSON(w, http.StatusBadRequest,
@ -1040,13 +1071,15 @@ func (rh *RouteHandler) DeleteBlobUpload(w http.ResponseWriter, r *http.Request)
return
}
is := rh.getImageStore(name)
sessionID, ok := vars["session_id"]
if !ok || sessionID == "" {
w.WriteHeader(http.StatusNotFound)
return
}
if err := rh.c.ImageStore.DeleteBlobUpload(name, sessionID); err != nil {
if err := is.DeleteBlobUpload(name, sessionID); err != nil {
switch err {
case errors.ErrRepoNotFound:
WriteJSON(w, http.StatusNotFound,
@ -1078,13 +1111,32 @@ type RepositoryList struct {
// @Failure 500 {string} string "internal server error"
// @Router /v2/_catalog [get].
func (rh *RouteHandler) ListRepositories(w http.ResponseWriter, r *http.Request) {
repos, err := rh.c.ImageStore.GetRepositories()
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
return
combineRepoList := make([]string, 0)
subStore := rh.c.StoreController.SubStore
for _, imgStore := range subStore {
repos, err := imgStore.GetRepositories()
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
return
}
combineRepoList = append(combineRepoList, repos...)
}
is := RepositoryList{Repositories: repos}
singleStore := rh.c.StoreController.DefaultStore
if singleStore != nil {
repos, err := singleStore.GetRepositories()
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
return
}
combineRepoList = append(combineRepoList, repos...)
}
is := RepositoryList{Repositories: combineRepoList}
WriteJSON(w, http.StatusOK, is)
}
@ -1148,3 +1200,8 @@ func WriteDataFromReader(w http.ResponseWriter, status int, length int64, mediaT
}
}
}
// will return image storage corresponding to subpath provided in config.
func (rh *RouteHandler) getImageStore(name string) *storage.ImageStore {
return rh.c.StoreController.GetImageStore(name)
}

11
pkg/compliance/config.go
View File

@ -1,11 +1,12 @@
package compliance
type Config struct {
Address string
Port string
Version string
OutputJSON bool
Compliance bool
Address string
Port string
Version string
StorageInfo []string
OutputJSON bool
Compliance bool
}
func NewConfig() *Config {

278
pkg/compliance/v1_0_0/check.go
View File

@ -7,6 +7,7 @@ import (
"fmt"
"io"
"os"
"path"
"strings"
"testing"
@ -46,6 +47,8 @@ func CheckWorkflows(t *testing.T, config *compliance.Config) {
baseURL := fmt.Sprintf("http://%s:%s", config.Address, config.Port)
storageInfo := config.StorageInfo
fmt.Println("------------------------------")
fmt.Println("Checking for v1.0.0 compliance")
fmt.Println("------------------------------")
@ -459,6 +462,11 @@ func CheckWorkflows(t *testing.T, config *compliance.Config) {
loc := Location(baseURL, resp)
So(loc, ShouldNotBeEmpty)
// since we are not specifying any prefix i.e provided in config while starting server,
// so it should store repo7 to global root dir
_, err = os.Stat(path.Join(storageInfo[0], "repo7"))
So(err, ShouldBeNil)
resp, err = resty.R().Get(loc)
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 204)
@ -686,6 +694,276 @@ func CheckWorkflows(t *testing.T, config *compliance.Config) {
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 202)
})
Convey("Multiple Storage", func() {
// test APIS on subpath routes, default storage already tested above
// subpath route firsttest
resp, err := resty.R().Post(baseURL + "/v2/firsttest/first/blobs/uploads/")
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 202)
firstloc := Location(baseURL, resp)
So(firstloc, ShouldNotBeEmpty)
resp, err = resty.R().Get(firstloc)
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 204)
// if firsttest route is used as prefix in url that means repo should be stored in subpaths["firsttest"] rootdir
_, err = os.Stat(path.Join(storageInfo[1], "firsttest/first"))
So(err, ShouldBeNil)
// subpath route secondtest
resp, err = resty.R().Post(baseURL + "/v2/secondtest/second/blobs/uploads/")
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 202)
secondloc := Location(baseURL, resp)
So(secondloc, ShouldNotBeEmpty)
resp, err = resty.R().Get(secondloc)
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 204)
// if secondtest route is used as prefix in url that means repo should be stored in subpaths["secondtest"] rootdir
_, err = os.Stat(path.Join(storageInfo[2], "secondtest/second"))
So(err, ShouldBeNil)
content := []byte("this is a blob5")
digest := godigest.FromBytes(content)
So(digest, ShouldNotBeNil)
// monolithic blob upload: success
// first test
resp, err = resty.R().SetQueryParam("digest", digest.String()).
SetHeader("Content-Type", "application/octet-stream").SetBody(content).Put(firstloc)
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 201)
firstblobLoc := resp.Header().Get("Location")
So(firstblobLoc, ShouldNotBeEmpty)
So(resp.Header().Get("Content-Length"), ShouldEqual, "0")
So(resp.Header().Get(api.DistContentDigestKey), ShouldNotBeEmpty)
// second test
resp, err = resty.R().SetQueryParam("digest", digest.String()).
SetHeader("Content-Type", "application/octet-stream").SetBody(content).Put(secondloc)
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 201)
secondblobLoc := resp.Header().Get("Location")
So(secondblobLoc, ShouldNotBeEmpty)
So(resp.Header().Get("Content-Length"), ShouldEqual, "0")
So(resp.Header().Get(api.DistContentDigestKey), ShouldNotBeEmpty)
// check a non-existent manifest
resp, err = resty.R().SetHeader("Content-Type", "application/vnd.oci.image.manifest.v1+json").
SetBody(content).Head(baseURL + "/v2/unknown/manifests/test:1.0")
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 404)
resp, err = resty.R().SetHeader("Content-Type", "application/vnd.oci.image.manifest.v1+json").
SetBody(content).Head(baseURL + "/v2/firsttest/unknown/manifests/test:1.0")
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 404)
resp, err = resty.R().SetHeader("Content-Type", "application/vnd.oci.image.manifest.v1+json").
SetBody(content).Head(baseURL + "/v2/secondtest/unknown/manifests/test:1.0")
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 404)
// create a manifest
m := ispec.Manifest{
Config: ispec.Descriptor{
Digest: digest,
Size: int64(len(content)),
},
Layers: []ispec.Descriptor{
{
MediaType: "application/vnd.oci.image.layer.v1.tar",
Digest: digest,
Size: int64(len(content)),
},
},
}
m.SchemaVersion = 2
content, err = json.Marshal(m)
So(err, ShouldBeNil)
digest = godigest.FromBytes(content)
So(digest, ShouldNotBeNil)
// subpath firsttest
resp, err = resty.R().SetHeader("Content-Type", "application/vnd.oci.image.manifest.v1+json").
SetBody(content).Put(baseURL + "/v2/firsttest/first/manifests/test:1.0")
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 201)
d := resp.Header().Get(api.DistContentDigestKey)
So(d, ShouldNotBeEmpty)
So(d, ShouldEqual, digest.String())
// subpath secondtest
resp, err = resty.R().SetHeader("Content-Type", "application/vnd.oci.image.manifest.v1+json").
SetBody(content).Put(baseURL + "/v2/secondtest/second/manifests/test:1.0")
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 201)
d = resp.Header().Get(api.DistContentDigestKey)
So(d, ShouldNotBeEmpty)
So(d, ShouldEqual, digest.String())
content = []byte("this is a blob5")
digest = godigest.FromBytes(content)
So(digest, ShouldNotBeNil)
// create a manifest with same blob but a different tag
m = ispec.Manifest{
Config: ispec.Descriptor{
Digest: digest,
Size: int64(len(content)),
},
Layers: []ispec.Descriptor{
{
MediaType: "application/vnd.oci.image.layer.v1.tar",
Digest: digest,
Size: int64(len(content)),
},
},
}
m.SchemaVersion = 2
content, err = json.Marshal(m)
So(err, ShouldBeNil)
digest = godigest.FromBytes(content)
So(digest, ShouldNotBeNil)
// subpath firsttest
resp, err = resty.R().SetHeader("Content-Type", "application/vnd.oci.image.manifest.v1+json").
SetBody(content).Put(baseURL + "/v2/firsttest/first/manifests/test:2.0")
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 201)
d = resp.Header().Get(api.DistContentDigestKey)
So(d, ShouldNotBeEmpty)
So(d, ShouldEqual, digest.String())
// subpath secondtest
resp, err = resty.R().SetHeader("Content-Type", "application/vnd.oci.image.manifest.v1+json").
SetBody(content).Put(baseURL + "/v2/secondtest/second/manifests/test:2.0")
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 201)
d = resp.Header().Get(api.DistContentDigestKey)
So(d, ShouldNotBeEmpty)
So(d, ShouldEqual, digest.String())
// check/get by tag
resp, err = resty.R().Head(baseURL + "/v2/firsttest/first/manifests/test:1.0")
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 200)
resp, err = resty.R().Get(baseURL + "/v2/firsttest/first/manifests/test:1.0")
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 200)
So(resp.Body(), ShouldNotBeEmpty)
resp, err = resty.R().Head(baseURL + "/v2/secondtest/second/manifests/test:1.0")
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 200)
resp, err = resty.R().Get(baseURL + "/v2/secondtest/second/manifests/test:1.0")
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 200)
So(resp.Body(), ShouldNotBeEmpty)
// check/get by reference
resp, err = resty.R().Head(baseURL + "/v2/firsttest/first/manifests/" + digest.String())
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 200)
resp, err = resty.R().Get(baseURL + "/v2/firsttest/first/manifests/" + digest.String())
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 200)
So(resp.Body(), ShouldNotBeEmpty)
resp, err = resty.R().Head(baseURL + "/v2/secondtest/second/manifests/" + digest.String())
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 200)
resp, err = resty.R().Get(baseURL + "/v2/secondtest/second/manifests/" + digest.String())
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 200)
So(resp.Body(), ShouldNotBeEmpty)
// delete manifest by tag should fail
resp, err = resty.R().Delete(baseURL + "/v2/firsttest/first/manifests/test:1.0")
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 400)
resp, err = resty.R().Delete(baseURL + "/v2/secondtest/second/manifests/test:1.0")
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 400)
// delete manifest by digest
resp, err = resty.R().Delete(baseURL + "/v2/firsttest/first/manifests/" + digest.String())
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 202)
resp, err = resty.R().Delete(baseURL + "/v2/secondtest/second/manifests/" + digest.String())
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 202)
// delete manifest by digest
resp, err = resty.R().Delete(baseURL + "/v2/firsttest/first/manifests/" + digest.String())
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 404)
resp, err = resty.R().Delete(baseURL + "/v2/secondtest/second/manifests/" + digest.String())
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 404)
// delete again should fail
resp, err = resty.R().Delete(baseURL + "/v2/firsttest/first/manifests/" + digest.String())
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 404)
resp, err = resty.R().Delete(baseURL + "/v2/secondtest/second/manifests/" + digest.String())
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 404)
// check/get by tag
resp, err = resty.R().Head(baseURL + "/v2/firsttest/first/manifests/test:1.0")
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 404)
resp, err = resty.R().Get(baseURL + "/v2/firsttest/first/manifests/test:1.0")
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 404)
So(resp.Body(), ShouldNotBeEmpty)
resp, err = resty.R().Head(baseURL + "/v2/secondtest/second/manifests/test:1.0")
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 404)
resp, err = resty.R().Get(baseURL + "/v2/secondtest/second/manifests/test:1.0")
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 404)
So(resp.Body(), ShouldNotBeEmpty)
resp, err = resty.R().Head(baseURL + "/v2/firsttest/first/repo7/manifests/test:2.0")
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 404)
resp, err = resty.R().Get(baseURL + "/v2/firsttest/first/manifests/test:2.0")
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 404)
So(resp.Body(), ShouldNotBeEmpty)
resp, err = resty.R().Head(baseURL + "/v2/secondtest/second/manifests/test:2.0")
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 404)
resp, err = resty.R().Get(baseURL + "/v2/secondtest/second/manifests/test:2.0")
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 404)
So(resp.Body(), ShouldNotBeEmpty)
// check/get by reference
resp, err = resty.R().Head(baseURL + "/v2/firsttest/first/manifests/" + digest.String())
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 404)
resp, err = resty.R().Get(baseURL + "/v2/firsttest/first/manifests/" + digest.String())
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 404)
So(resp.Body(), ShouldNotBeEmpty)
resp, err = resty.R().Head(baseURL + "/v2/secondtest/second/manifests/" + digest.String())
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 404)
resp, err = resty.R().Get(baseURL + "/v2/secondtest/second/manifests/" + digest.String())
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 404)
So(resp.Body(), ShouldNotBeEmpty)
})
})
}

44
pkg/compliance/v1_0_0/check_test.go
View File

@ -18,24 +18,35 @@ import (
// nolint: gochecknoglobals
var (
listenAddress = "127.0.0.1"
defaultDir = ""
firstDir = ""
secondDir = ""
)
func TestWorkflows(t *testing.T) {
ctrl, randomPort := startServer()
defer stopServer(ctrl)
storageInfo := []string{defaultDir, firstDir, secondDir}
v1_0_0.CheckWorkflows(t, &compliance.Config{
Address: listenAddress,
Port: randomPort,
Address: listenAddress,
Port: randomPort,
StorageInfo: storageInfo,
})
}
func TestWorkflowsOutputJSON(t *testing.T) {
ctrl, randomPort := startServer()
defer stopServer(ctrl)
storageInfo := []string{defaultDir, firstDir, secondDir}
v1_0_0.CheckWorkflows(t, &compliance.Config{
Address: listenAddress,
Port: randomPort,
OutputJSON: true,
Address: listenAddress,
Port: randomPort,
OutputJSON: true,
StorageInfo: storageInfo,
})
}
@ -59,8 +70,31 @@ func startServer() (*api.Controller, string) {
panic(err)
}
defaultDir = dir
firstSubDir, err := ioutil.TempDir("", "oci-repo-test")
if err != nil {
panic(err)
}
firstDir = firstSubDir
secondSubDir, err := ioutil.TempDir("", "oci-repo-test")
if err != nil {
panic(err)
}
secondDir = secondSubDir
subPaths := make(map[string]api.StorageConfig)
subPaths["/firsttest"] = api.StorageConfig{RootDirectory: firstSubDir}
subPaths["/secondtest"] = api.StorageConfig{RootDirectory: secondSubDir}
ctrl.Config.Storage.RootDirectory = dir
ctrl.Config.Storage.SubPaths = subPaths
go func() {
// this blocks
if err := ctrl.Run(); err != nil {

4
pkg/extensions/extensions.go
View File

@ -55,9 +55,9 @@ func EnableExtensions(extension *ExtensionConfig, log log.Logger, rootDir string
}
// SetupRoutes ...
func SetupRoutes(router *mux.Router, rootDir string, imgStore *storage.ImageStore, log log.Logger) {
func SetupRoutes(router *mux.Router, storeController storage.StoreController, log log.Logger) {
log.Info().Msg("setting up extensions routes")
resConfig := search.GetResolverConfig(rootDir, log, imgStore)
resConfig := search.GetResolverConfig(log, storeController)
router.PathPrefix("/query").Methods("GET", "POST").
Handler(gqlHandler.NewDefaultServer(search.NewExecutableSchema(resConfig)))
}

2
pkg/extensions/minimal.go
View File

@ -21,6 +21,6 @@ func EnableExtensions(extension *ExtensionConfig, log log.Logger, rootDir string
}
// SetupRoutes ...
func SetupRoutes(router *mux.Router, rootDir string, imgStore *storage.ImageStore, log log.Logger) {
func SetupRoutes(router *mux.Router, storeController storage.StoreController, log log.Logger) {
log.Warn().Msg("skipping setting up extensions routes because given zot binary doesn't support any extensions, please build zot full binary for this feature")
}

161
pkg/extensions/search/cve/cve.go
View File

@ -2,6 +2,7 @@ package cveinfo
import (
"encoding/json"
"fmt"
"io/ioutil"
"os"
"path"
@ -10,6 +11,7 @@ import (
"github.com/anuvu/zot/errors"
"github.com/anuvu/zot/pkg/log"
"github.com/anuvu/zot/pkg/storage"
integration "github.com/aquasecurity/trivy/integration"
config "github.com/aquasecurity/trivy/integration/config"
"github.com/aquasecurity/trivy/pkg/report"
@ -44,6 +46,84 @@ func ScanImage(config *config.Config) (report.Results, error) {
return integration.ScanTrivyImage(config.TrivyConfig)
}
func GetCVEInfo(storeController storage.StoreController, log log.Logger) (*CveInfo, error) {
cveController := CveTrivyController{}
subCveConfig := make(map[string]*config.Config)
if storeController.DefaultStore != nil {
imageStore := storeController.DefaultStore
rootDir := imageStore.RootDir()
config, err := NewTrivyConfig(rootDir)
if err != nil {
return nil, err
}
cveController.DefaultCveConfig = config
}
if storeController.SubStore != nil {
for route, storage := range storeController.SubStore {
rootDir := storage.RootDir()
config, err := NewTrivyConfig(rootDir)
if err != nil {
return nil, err
}
subCveConfig[route] = config
}
}
cveController.SubCveConfig = subCveConfig
return &CveInfo{Log: log, CveTrivyController: cveController, StoreController: storeController}, nil
}
func getRoutePrefix(name string) string {
names := strings.SplitN(name, "/", 2)
if len(names) != 2 { // nolint: gomnd
// it means route is of global storage e.g "centos:latest"
if len(names) == 1 {
return "/"
}
}
return fmt.Sprintf("/%s", names[0])
}
func (cveinfo CveInfo) GetTrivyConfig(image string) *config.Config {
// Split image to get route prefix
prefixName := getRoutePrefix(image)
var trivyConfig *config.Config
var ok bool
var rootDir string
// Get corresponding CVE trivy config, if no sub cve config present that means its default
trivyConfig, ok = cveinfo.CveTrivyController.SubCveConfig[prefixName]
if ok {
imgStore := cveinfo.StoreController.SubStore[prefixName]
rootDir = imgStore.RootDir()
} else {
trivyConfig = cveinfo.CveTrivyController.DefaultCveConfig
imgStore := cveinfo.StoreController.DefaultStore
rootDir = imgStore.RootDir()
}
trivyConfig.TrivyConfig.Input = path.Join(rootDir, image)
return trivyConfig
}
func (cveinfo CveInfo) IsValidImageFormat(imagePath string) (bool, error) {
imageDir, inputTag := getImageDirAndTag(imagePath)
@ -113,16 +193,85 @@ func getImageDirAndTag(imageName string) (string, string) {
return imageDir, imageTag
}
// Below method will return image path including root dir, root dir is determined by splitting.
func (cveinfo CveInfo) GetImageRepoPath(image string) string {
var rootDir string
prefixName := getRoutePrefix(image)
subStore := cveinfo.StoreController.SubStore
if subStore != nil {
imgStore, ok := cveinfo.StoreController.SubStore[prefixName]
if ok {
rootDir = imgStore.RootDir()
} else {
rootDir = cveinfo.StoreController.DefaultStore.RootDir()
}
} else {
rootDir = cveinfo.StoreController.DefaultStore.RootDir()
}
return path.Join(rootDir, image)
}
func (cveinfo CveInfo) GetImageListForCVE(repo string, id string, imgStore *storage.ImageStore,
trivyConfig *config.Config) ([]*string, error) {
tags := make([]*string, 0)
tagList, err := imgStore.GetImageTags(repo)
if err != nil {
cveinfo.Log.Error().Err(err).Msg("unable to get list of image tag")
return tags, err
}
rootDir := imgStore.RootDir()
for _, tag := range tagList {
trivyConfig.TrivyConfig.Input = fmt.Sprintf("%s:%s", path.Join(rootDir, repo), tag)
isValidImage, _ := cveinfo.IsValidImageFormat(trivyConfig.TrivyConfig.Input)
if !isValidImage {
cveinfo.Log.Debug().Str("image", repo+":"+tag).Msg("image media type not supported for scanning")
continue
}
cveinfo.Log.Info().Str("image", repo+":"+tag).Msg("scanning image")
results, err := ScanImage(trivyConfig)
if err != nil {
cveinfo.Log.Error().Err(err).Str("image", repo+":"+tag).Msg("unable to scan image")
continue
}
for _, result := range results {
for _, vulnerability := range result.Vulnerabilities {
if vulnerability.VulnerabilityID == id {
copyImgTag := tag
tags = append(tags, &copyImgTag)
break
}
}
}
}
return tags, nil
}
// GetImageTagsWithTimestamp returns a list of image tags with timestamp available in the specified repository.
func (cveinfo CveInfo) GetImageTagsWithTimestamp(rootDir string, repo string) ([]TagInfo, error) {
func (cveinfo CveInfo) GetImageTagsWithTimestamp(repo string) ([]TagInfo, error) {
tagsInfo := make([]TagInfo, 0)
dir := path.Join(rootDir, repo)
if !dirExists(dir) {
imagePath := cveinfo.GetImageRepoPath(repo)
if !dirExists(imagePath) {
return nil, errors.ErrRepoNotFound
}
manifests, err := cveinfo.getImageManifests(dir)
manifests, err := cveinfo.getImageManifests(imagePath)
if err != nil {
cveinfo.Log.Error().Err(err).Msg("unable to read image manifests")
@ -135,7 +284,7 @@ func (cveinfo CveInfo) GetImageTagsWithTimestamp(rootDir string, repo string) ([
v, ok := manifest.Annotations[ispec.AnnotationRefName]
if ok {
imageBlobManifest, err := cveinfo.getImageBlobManifest(dir, digest)
imageBlobManifest, err := cveinfo.getImageBlobManifest(imagePath, digest)
if err != nil {
cveinfo.Log.Error().Err(err).Msg("unable to read image blob manifest")
@ -143,7 +292,7 @@ func (cveinfo CveInfo) GetImageTagsWithTimestamp(rootDir string, repo string) ([
return tagsInfo, err
}
imageInfo, err := cveinfo.getImageInfo(dir, imageBlobManifest.Config.Digest)
imageInfo, err := cveinfo.getImageInfo(imagePath, imageBlobManifest.Config.Digest)
if err != nil {
cveinfo.Log.Error().Err(err).Msg("unable to read image info")

133
pkg/extensions/search/cve/cve_test.go
View File

@ -16,6 +16,7 @@ import (
ext "github.com/anuvu/zot/pkg/extensions"
cveinfo "github.com/anuvu/zot/pkg/extensions/search/cve"
"github.com/anuvu/zot/pkg/log"
"github.com/anuvu/zot/pkg/storage"
ispec "github.com/opencontainers/image-spec/specs-go/v1"
. "github.com/smartystreets/goconvey/convey"
"gopkg.in/resty.v1"
@ -23,8 +24,9 @@ import (
// nolint:gochecknoglobals
var (
cve *cveinfo.CveInfo
dbDir string
cve *cveinfo.CveInfo
dbDir string
updateDuration time.Duration
)
const (
@ -76,7 +78,11 @@ func testSetup() error {
return err
}
cve = &cveinfo.CveInfo{Log: log.NewLogger("debug", "")}
log := log.NewLogger("debug", "")
cve = &cveinfo.CveInfo{Log: log}
cve.StoreController = storage.StoreController{DefaultStore: storage.NewImageStore(dir, false, false, log)}
dbDir = dir
@ -365,6 +371,67 @@ func makeHtpasswdFile() string {
return f.Name()
}
func TestMultipleStoragePath(t *testing.T) {
Convey("Test multiple storage path", t, func() {
// Create temporary directory
firstRootDir, err := ioutil.TempDir("", "util_test")
if err != nil {
panic(err)
}
defer os.RemoveAll(firstRootDir)
secondRootDir, err := ioutil.TempDir("", "util_test")
if err != nil {
panic(err)
}
defer os.RemoveAll(secondRootDir)
thirdRootDir, err := ioutil.TempDir("", "util_test")
if err != nil {
panic(err)
}
defer os.RemoveAll(thirdRootDir)
log := log.NewLogger("debug", "")
// Create ImageStore
firstStore := storage.NewImageStore(firstRootDir, false, false, log)
secondStore := storage.NewImageStore(secondRootDir, false, false, log)
thirdStore := storage.NewImageStore(thirdRootDir, false, false, log)
storeController := storage.StoreController{}
storeController.DefaultStore = firstStore
subStore := make(map[string]*storage.ImageStore)
subStore["/a"] = secondStore
subStore["/b"] = thirdStore
storeController.SubStore = subStore
cveInfo, err := cveinfo.GetCVEInfo(storeController, log)
So(err, ShouldBeNil)
So(cveInfo.StoreController.DefaultStore, ShouldNotBeNil)
So(cveInfo.StoreController.SubStore, ShouldNotBeNil)
imagePath := cveInfo.GetImageRepoPath("zot-test")
So(imagePath, ShouldEqual, path.Join(firstRootDir, "zot-test"))
imagePath = cveInfo.GetImageRepoPath("a/zot-a-test")
So(imagePath, ShouldEqual, path.Join(secondRootDir, "a/zot-a-test"))
imagePath = cveInfo.GetImageRepoPath("b/zot-b-test")
So(imagePath, ShouldEqual, path.Join(thirdRootDir, "b/zot-b-test"))
imagePath = cveInfo.GetImageRepoPath("c/zot-c-test")
So(imagePath, ShouldEqual, path.Join(firstRootDir, "c/zot-c-test"))
})
}
func TestDownloadDB(t *testing.T) {
Convey("Download DB passing invalid dir", t, func() {
err := testSetup()
@ -425,35 +492,35 @@ func TestImageFormat(t *testing.T) {
func TestImageTag(t *testing.T) {
Convey("Test image tag", t, func() {
imageTags, err := cve.GetImageTagsWithTimestamp(dbDir, "zot-test")
imageTags, err := cve.GetImageTagsWithTimestamp("zot-test")
So(err, ShouldBeNil)
So(len(imageTags), ShouldNotEqual, 0)
imageTags, err = cve.GetImageTagsWithTimestamp(dbDir, "zot-tes")
imageTags, err = cve.GetImageTagsWithTimestamp("zot-tes")
So(err, ShouldNotBeNil)
So(imageTags, ShouldBeNil)
imageTags, err = cve.GetImageTagsWithTimestamp(dbDir, "zot-noindex-test")
imageTags, err = cve.GetImageTagsWithTimestamp("zot-noindex-test")
So(err, ShouldNotBeNil)
So(len(imageTags), ShouldEqual, 0)
imageTags, err = cve.GetImageTagsWithTimestamp(dbDir, "zot-squashfs-noblobs")
imageTags, err = cve.GetImageTagsWithTimestamp("zot-squashfs-noblobs")
So(err, ShouldNotBeNil)
So(len(imageTags), ShouldEqual, 0)
imageTags, err = cve.GetImageTagsWithTimestamp(dbDir, "zot-squashfs-invalid-index")
imageTags, err = cve.GetImageTagsWithTimestamp("zot-squashfs-invalid-index")
So(err, ShouldNotBeNil)
So(len(imageTags), ShouldEqual, 0)
imageTags, err = cve.GetImageTagsWithTimestamp(dbDir, "zot-squashfs-invalid-blob")
imageTags, err = cve.GetImageTagsWithTimestamp("zot-squashfs-invalid-blob")
So(err, ShouldNotBeNil)
So(len(imageTags), ShouldEqual, 0)
imageTags, err = cve.GetImageTagsWithTimestamp(dbDir, "zot-invalid-layer")
imageTags, err = cve.GetImageTagsWithTimestamp("zot-invalid-layer")
So(err, ShouldNotBeNil)
So(len(imageTags), ShouldEqual, 0)
imageTags, err = cve.GetImageTagsWithTimestamp(dbDir, "zot-no-layer")
imageTags, err = cve.GetImageTagsWithTimestamp("zot-no-layer")
So(err, ShouldNotBeNil)
So(len(imageTags), ShouldEqual, 0)
})
@ -461,7 +528,7 @@ func TestImageTag(t *testing.T) {
func TestCVESearch(t *testing.T) {
Convey("Test image vulenrability scanning", t, func() {
updateDuration, _ := time.ParseDuration("1h")
updateDuration, _ = time.ParseDuration("1h")
config := api.NewConfig()
config.HTTP.Port = SecurePort1
htpasswdPath := makeHtpasswdFile()
@ -473,7 +540,6 @@ func TestCVESearch(t *testing.T) {
},
}
c := api.NewController(config)
defer os.RemoveAll(dbDir)
c.Config.Storage.RootDirectory = dbDir
cveConfig := &ext.CVEConfig{
UpdateInterval: updateDuration,
@ -698,13 +764,30 @@ func TestCVEConfig(t *testing.T) {
},
}
c := api.NewController(config)
dir, err := ioutil.TempDir("", "oci-repo-test")
firstDir, err := ioutil.TempDir("", "oci-repo-test")
if err != nil {
panic(err)
}
defer os.RemoveAll(dir)
c.Config.Storage.RootDirectory = dir
c.Config.Extensions = &ext.ExtensionConfig{}
secondDir, err := ioutil.TempDir("", "oci-repo-test")
if err != nil {
panic(err)
}
defer os.RemoveAll(firstDir)
defer os.RemoveAll(secondDir)
err = copyFiles("../../../../test/data", path.Join(secondDir, "a"))
if err != nil {
panic(err)
}
c.Config.Storage.RootDirectory = firstDir
subPaths := make(map[string]api.StorageConfig)
subPaths["/a"] = api.StorageConfig{
RootDirectory: secondDir,
}
c.Config.Storage.SubPaths = subPaths
go func() {
// this blocks
if err := c.Run(); err != nil {
@ -721,6 +804,22 @@ func TestCVEConfig(t *testing.T) {
time.Sleep(100 * time.Millisecond)
}
resp, _ := resty.R().SetBasicAuth(username, passphrase).Get(BaseURL1 + "/v2/")
So(resp, ShouldNotBeNil)
So(resp.StatusCode(), ShouldEqual, 200)
resp, _ = resty.R().SetBasicAuth(username, passphrase).Get(BaseURL1 + "/v2/_catalog")
So(resp, ShouldNotBeNil)
So(resp.StatusCode(), ShouldEqual, 200)
resp, _ = resty.R().SetBasicAuth(username, passphrase).Get(BaseURL1 + "/v2/a/zot-test/tags/list")
So(resp, ShouldNotBeNil)
So(resp.StatusCode(), ShouldEqual, 200)
resp, _ = resty.R().SetBasicAuth(username, passphrase).Get(BaseURL1 + "/v2/zot-test/tags/list")
So(resp, ShouldNotBeNil)
So(resp.StatusCode(), ShouldEqual, 404)
defer func() {
ctx := context.Background()
_ = c.Server.Shutdown(ctx)

11
pkg/extensions/search/cve/models.go
View File

@ -5,13 +5,20 @@ import (
"time"
"github.com/anuvu/zot/pkg/log"
"github.com/anuvu/zot/pkg/storage"
config "github.com/aquasecurity/trivy/integration/config"
)
// CveInfo ...
type CveInfo struct {
Log log.Logger
CveTrivyConfig *config.Config
Log log.Logger
CveTrivyController CveTrivyController
StoreController storage.StoreController
}
type CveTrivyController struct {
DefaultCveConfig *config.Config
SubCveConfig map[string]*config.Config
}
type TagInfo struct {

130
pkg/extensions/search/resolver.go
View File

@ -4,10 +4,11 @@ package search
import (
"context"
"path"
"fmt"
"strings"
"github.com/anuvu/zot/pkg/log"
"github.com/aquasecurity/trivy/integration/config"
cveinfo "github.com/anuvu/zot/pkg/extensions/search/cve"
"github.com/anuvu/zot/pkg/storage"
@ -15,9 +16,8 @@ import (
// Resolver ...
type Resolver struct {
cveInfo *cveinfo.CveInfo
imgStore *storage.ImageStore
dir string
cveInfo *cveinfo.CveInfo
storeController storage.StoreController
}
// Query ...
@ -35,33 +35,31 @@ type cveDetail struct {
}
// GetResolverConfig ...
func GetResolverConfig(dir string, log log.Logger, imgstorage *storage.ImageStore) Config {
config, err := cveinfo.NewTrivyConfig(dir)
func GetResolverConfig(log log.Logger, storeController storage.StoreController) Config {
cveInfo, err := cveinfo.GetCVEInfo(storeController, log)
if err != nil {
panic(err)
}
cve := &cveinfo.CveInfo{Log: log, CveTrivyConfig: config}
resConfig := &Resolver{cveInfo: cve, imgStore: imgstorage, dir: dir}
resConfig := &Resolver{cveInfo: cveInfo, storeController: storeController}
return Config{Resolvers: resConfig, Directives: DirectiveRoot{},
Complexity: ComplexityRoot{}}
}
func (r *queryResolver) CVEListForImage(ctx context.Context, image string) (*CVEResultForImage, error) {
r.cveInfo.CveTrivyConfig.TrivyConfig.Input = path.Join(r.dir, image)
trivyConfig := r.cveInfo.GetTrivyConfig(image)
r.cveInfo.Log.Info().Str("image", image).Msg("scanning image")
isValidImage, err := r.cveInfo.IsValidImageFormat(r.cveInfo.CveTrivyConfig.TrivyConfig.Input)
isValidImage, err := r.cveInfo.IsValidImageFormat(trivyConfig.TrivyConfig.Input)
if !isValidImage {
r.cveInfo.Log.Debug().Str("image", image).Msg("image media type not supported for scanning")
return &CVEResultForImage{}, err
}
cveResults, err := cveinfo.ScanImage(r.cveInfo.CveTrivyConfig)
cveResults, err := cveinfo.ScanImage(trivyConfig)
if err != nil {
r.cveInfo.Log.Error().Err(err).Msg("unable to scan image repository")
@ -134,62 +132,70 @@ func (r *queryResolver) CVEListForImage(ctx context.Context, image string) (*CVE
}
func (r *queryResolver) ImageListForCve(ctx context.Context, id string) ([]*ImgResultForCve, error) {
cveResult := []*ImgResultForCve{}
finalCveResult := []*ImgResultForCve{}
r.cveInfo.Log.Info().Msg("extracting repositories")
repoList, err := r.imgStore.GetRepositories()
defaultStore := r.storeController.DefaultStore
defaultTrivyConfig := r.cveInfo.CveTrivyController.DefaultCveConfig
repoList, err := defaultStore.GetRepositories()
if err != nil {
r.cveInfo.Log.Error().Err(err).Msg("unable to search repositories")
return cveResult, err
return finalCveResult, err
}
r.cveInfo.Log.Info().Msg("scanning each repository")
r.cveInfo.Log.Info().Msg("scanning each global repository")
cveResult, err := r.getImageListForCVE(repoList, id, defaultStore, defaultTrivyConfig)
if err != nil {
r.cveInfo.Log.Error().Err(err).Msg("error getting cve list for global repositories")
return finalCveResult, err
}
finalCveResult = append(finalCveResult, cveResult...)
subStore := r.storeController.SubStore
for route, store := range subStore {
subRepoList, err := store.GetRepositories()
if err != nil {
r.cveInfo.Log.Error().Err(err).Msg("unable to search repositories")
return cveResult, err
}
subTrivyConfig := r.cveInfo.CveTrivyController.SubCveConfig[route]
subCveResult, err := r.getImageListForCVE(subRepoList, id, store, subTrivyConfig)
if err != nil {
r.cveInfo.Log.Error().Err(err).Msg("unable to get cve result for sub repositories")
return finalCveResult, err
}
finalCveResult = append(finalCveResult, subCveResult...)
}
return finalCveResult, nil
}
func (r *queryResolver) getImageListForCVE(repoList []string, id string, imgStore *storage.ImageStore,
trivyConfig *config.Config) ([]*ImgResultForCve, error) {
cveResult := []*ImgResultForCve{}
for _, repo := range repoList {
r.cveInfo.Log.Info().Str("repo", repo).Msg("extracting list of tags available in image repo")
tagList, err := r.imgStore.GetImageTags(repo)
name := repo
tags, err := r.cveInfo.GetImageListForCVE(repo, id, imgStore, trivyConfig)
if err != nil {
r.cveInfo.Log.Error().Err(err).Msg("unable to get list of image tag")
}
r.cveInfo.Log.Error().Err(err).Msg("error getting tag")
var name string
tags := make([]*string, 0)
for _, tag := range tagList {
r.cveInfo.CveTrivyConfig.TrivyConfig.Input = path.Join(r.dir, repo+":"+tag)
isValidImage, _ := r.cveInfo.IsValidImageFormat(r.cveInfo.CveTrivyConfig.TrivyConfig.Input)
if !isValidImage {
r.cveInfo.Log.Debug().Str("image", repo+":"+tag).Msg("image media type not supported for scanning")
continue
}
r.cveInfo.Log.Info().Str("image", repo+":"+tag).Msg("scanning image")
results, err := cveinfo.ScanImage(r.cveInfo.CveTrivyConfig)
if err != nil {
r.cveInfo.Log.Error().Err(err).Str("image", repo+":"+tag).Msg("unable to scan image")
continue
}
name = repo
for _, result := range results {
for _, vulnerability := range result.Vulnerabilities {
if vulnerability.VulnerabilityID == id {
copyImgTag := tag
tags = append(tags, &copyImgTag)
break
}
}
}
return cveResult, err
}
if len(tags) != 0 {
@ -203,9 +209,17 @@ func (r *queryResolver) ImageListForCve(ctx context.Context, id string) ([]*ImgR
func (r *queryResolver) ImageListWithCVEFixed(ctx context.Context, id string, image string) (*ImgResultForFixedCve, error) { // nolint: lll
imgResultForFixedCVE := &ImgResultForFixedCve{}
r.cveInfo.Log.Info().Str("image", image).Msg("retrieving image path")
imagePath := r.cveInfo.GetImageRepoPath(image)
r.cveInfo.Log.Info().Str("image", image).Msg("retrieving trivy config")
trivyConfig := r.cveInfo.GetTrivyConfig(image)
r.cveInfo.Log.Info().Str("image", image).Msg("extracting list of tags available in image")
tagsInfo, err := r.cveInfo.GetImageTagsWithTimestamp(r.dir, image)
tagsInfo, err := r.cveInfo.GetImageTagsWithTimestamp(image)
if err != nil {
r.cveInfo.Log.Error().Err(err).Msg("unable to read image tags")
@ -217,9 +231,9 @@ func (r *queryResolver) ImageListWithCVEFixed(ctx context.Context, id string, im
var hasCVE bool
for _, tag := range tagsInfo {
r.cveInfo.CveTrivyConfig.TrivyConfig.Input = path.Join(r.dir, image+":"+tag.Name)
trivyConfig.TrivyConfig.Input = fmt.Sprintf("%s:%s", imagePath, tag.Name)
isValidImage, _ := r.cveInfo.IsValidImageFormat(r.cveInfo.CveTrivyConfig.TrivyConfig.Input)
isValidImage, _ := r.cveInfo.IsValidImageFormat(trivyConfig.TrivyConfig.Input)
if !isValidImage {
r.cveInfo.Log.Debug().Str("image",
image+":"+tag.Name).Msg("image media type not supported for scanning, adding as an infected image")
@ -231,7 +245,7 @@ func (r *queryResolver) ImageListWithCVEFixed(ctx context.Context, id string, im
r.cveInfo.Log.Info().Str("image", image+":"+tag.Name).Msg("scanning image")
results, err := cveinfo.ScanImage(r.cveInfo.CveTrivyConfig)
results, err := cveinfo.ScanImage(trivyConfig)
if err != nil {
r.cveInfo.Log.Error().Err(err).Str("image", image+":"+tag.Name).Msg("unable to scan image")

69
pkg/storage/storage.go
View File

@ -10,6 +10,7 @@ import (
"os"
"path"
"path/filepath"
"strings"
"sync"
"time"
@ -37,6 +38,11 @@ type BlobUpload struct {
ID string
}
type StoreController struct {
DefaultStore *ImageStore
SubStore map[string]*ImageStore
}
// ImageStore provides the image storage operations.
type ImageStore struct {
rootDir string
@ -48,6 +54,39 @@ type ImageStore struct {
log zerolog.Logger
}
func (is *ImageStore) RootDir() string {
return is.rootDir
}
func getRoutePrefix(name string) string {
names := strings.SplitN(name, "/", 2)
if len(names) != 2 { // nolint: gomnd
// it means route is of global storage e.g "centos:latest"
if len(names) == 1 {
return "/"
}
}
return fmt.Sprintf("/%s", names[0])
}
func (sc StoreController) GetImageStore(name string) *ImageStore {
if sc.SubStore != nil {
// SubStore is being provided, now we need to find equivalent image store and this will be found by splitting name
prefixName := getRoutePrefix(name)
imgStore, ok := sc.SubStore[prefixName]
if !ok {
imgStore = sc.DefaultStore
}
return imgStore
}
return sc.DefaultStore
}
// NewImageStore returns a new image store backed by a file storage.
func NewImageStore(rootDir string, gc bool, dedupe bool, log zlog.Logger) *ImageStore {
if _, err := os.Stat(rootDir); os.IsNotExist(err) {
@ -1174,6 +1213,36 @@ func Scrub(dir string, fix bool) error {
// utility routines
func CheckHardLink(srcFileName string, destFileName string) error {
return os.Link(srcFileName, destFileName)
}
func ValidateHardLink(rootDir string) error {
err := ioutil.WriteFile(path.Join(rootDir, "hardlinkcheck.txt"), //nolint: gosec
[]byte("check whether hardlinks work on filesystem"), 0644)
if err != nil {
return err
}
err = CheckHardLink(path.Join(rootDir, "hardlinkcheck.txt"), path.Join(rootDir, "duphardlinkcheck.txt"))
if err != nil {
// Remove hardlinkcheck.txt if hardlink fails
zerr := os.RemoveAll(path.Join(rootDir, "hardlinkcheck.txt"))
if zerr != nil {
return zerr
}
return err
}
err = os.RemoveAll(path.Join(rootDir, "hardlinkcheck.txt"))
if err != nil {
return err
}
return os.RemoveAll(path.Join(rootDir, "duphardlinkcheck.txt"))
}
func dirExists(d string) bool {
fi, err := os.Stat(d)
if err != nil && os.IsNotExist(err) {

86
pkg/storage/storage_test.go
View File

@ -598,3 +598,89 @@ func TestNegativeCases(t *testing.T) {
So(err, ShouldNotBeNil)
})
}
func TestHardLink(t *testing.T) {
Convey("Test if filesystem supports hardlink", t, func() {
dir, err := ioutil.TempDir("", "storage-hard-test")
if err != nil {
panic(err)
}
defer os.RemoveAll(dir)
err = storage.ValidateHardLink(dir)
So(err, ShouldBeNil)
err = ioutil.WriteFile(path.Join(dir, "hardtest.txt"), []byte("testing hard link code"), 0644) //nolint: gosec
if err != nil {
panic(err)
}
err = os.Chmod(dir, 0400)
if err != nil {
panic(err)
}
err = storage.CheckHardLink(path.Join(dir, "hardtest.txt"), path.Join(dir, "duphardtest.txt"))
So(err, ShouldNotBeNil)
err = os.Chmod(dir, 0644)
if err != nil {
panic(err)
}
})
}
func TestStorageHandler(t *testing.T) {
Convey("Test storage handler", t, func() {
// Create temporary directory
firstRootDir, err := ioutil.TempDir("", "util_test")
if err != nil {
panic(err)
}
defer os.RemoveAll(firstRootDir)
secondRootDir, err := ioutil.TempDir("", "util_test")
if err != nil {
panic(err)
}
defer os.RemoveAll(secondRootDir)
thirdRootDir, err := ioutil.TempDir("", "util_test")
if err != nil {
panic(err)
}
defer os.RemoveAll(thirdRootDir)
log := log.NewLogger("debug", "")
// Create ImageStore
firstStore := storage.NewImageStore(firstRootDir, false, false, log)
secondStore := storage.NewImageStore(secondRootDir, false, false, log)
thirdStore := storage.NewImageStore(thirdRootDir, false, false, log)
storeController := storage.StoreController{}
storeController.DefaultStore = firstStore
subStore := make(map[string]*storage.ImageStore)
subStore["/a"] = secondStore
subStore["/b"] = thirdStore
storeController.SubStore = subStore
is := storeController.GetImageStore("zot-x-test")
So(is.RootDir(), ShouldEqual, firstRootDir)
is = storeController.GetImageStore("a/zot-a-test")
So(is.RootDir(), ShouldEqual, secondRootDir)
is = storeController.GetImageStore("b/zot-b-test")
So(is.RootDir(), ShouldEqual, thirdRootDir)
is = storeController.GetImageStore("c/zot-c-test")
So(is.RootDir(), ShouldEqual, firstRootDir)
})
}