storage: flush/sync contents to disk on file close

Behavior controlled by configuration (default=off)
It is a trade-off between performance and consistency.

References:
[1] https://github.com/golang/go/issues/20599

Signed-off-by: Ramkumar Chinchani <rchincha@cisco.com>

Makefile

@@ -69,8 +69,8 @@ test: check-skopeo $(NOTATION)
 	go test -tags extended,containers_image_openpgp -v -trimpath -race -timeout 15m -cover -coverpkg ./... -coverprofile=coverage-extended.txt -covermode=atomic ./...
 	go test -tags minimal,containers_image_openpgp -v -trimpath -race -cover -coverpkg ./... -coverprofile=coverage-minimal.txt -covermode=atomic ./...
 	# development-mode unit tests possibly using failure injection
-	go test -tags dev,extended,containers_image_openpgp -v -trimpath -race -timeout 15m -cover -coverpkg ./... -coverprofile=coverage-dev-extended.txt -covermode=atomic ./pkg/api/... ./pkg/test/...
+	go test -tags dev,extended,containers_image_openpgp -v -trimpath -race -timeout 15m -cover -coverpkg ./... -coverprofile=coverage-dev-extended.txt -covermode=atomic ./pkg/test/... ./pkg/storage/...
-	go test -tags dev,minimal,containers_image_openpgp -v -trimpath -race -cover -coverpkg ./... -coverprofile=coverage-dev-minimal.txt -covermode=atomic ./pkg/api/... ./pkg/test/...
+	go test -tags dev,minimal,containers_image_openpgp -v -trimpath -race -cover -coverpkg ./... -coverprofile=coverage-dev-minimal.txt -covermode=atomic ./pkg/test/... ./pkg/storage/...
 
 .PHONY: run-bench
 run-bench: binary bench

examples/config-commit.json

@@ -0,0 +1,15 @@
+{
+    "version": "0.1.0-dev",
+    "storage": {
+        "rootDirectory": "/tmp/zot",
+        "commit": true
+    },
+    "http": {
+        "address": "127.0.0.1",
+        "port": "8080",
+        "ReadOnly": false
+    },
+    "log": {
+        "level": "debug"
+    }
+}

pkg/api/config/config.go

@@ -21,6 +21,7 @@ type StorageConfig struct {
 	RootDirectory string
 	GC            bool
 	Dedupe        bool
+	Commit        bool
 	StorageDriver map[string]interface{} `mapstructure:",omitempty"`
 }
 
@@ -90,9 +91,10 @@ type LogConfig struct {
 }
 
 type GlobalStorageConfig struct {
-	RootDirectory string
 	Dedupe        bool
 	GC            bool
+	Commit        bool
+	RootDirectory string
 	StorageDriver map[string]interface{} `mapstructure:",omitempty"`
 	SubPaths      map[string]StorageConfig
 }

pkg/api/controller.go

@@ -214,7 +214,7 @@ func (c *Controller) InitImageStore() error {
 		var defaultStore storage.ImageStore
 		if len(c.Config.Storage.StorageDriver) == 0 {
 			defaultStore = storage.NewImageStore(c.Config.Storage.RootDirectory,
-				c.Config.Storage.GC, c.Config.Storage.Dedupe, c.Log, c.Metrics)
+				c.Config.Storage.GC, c.Config.Storage.Dedupe, c.Config.Storage.Commit, c.Log, c.Metrics)
 		} else {
 			storeName := fmt.Sprintf("%v", c.Config.Storage.StorageDriver["name"])
 			if storeName != storage.S3StorageDriverName {
@@ -230,7 +230,7 @@ func (c *Controller) InitImageStore() error {
 			}
 
 			defaultStore = s3.NewImageStore(c.Config.Storage.RootDirectory,
-				c.Config.Storage.GC, c.Config.Storage.Dedupe, c.Log, c.Metrics, store)
+				c.Config.Storage.GC, c.Config.Storage.Dedupe, c.Config.Storage.Commit, c.Log, c.Metrics, store)
 		}
 
 		c.StoreController.DefaultStore = defaultStore
@@ -266,7 +266,7 @@ func (c *Controller) InitImageStore() error {
 
 				if len(storageConfig.StorageDriver) == 0 {
 					subImageStore[route] = storage.NewImageStore(storageConfig.RootDirectory,
-						storageConfig.GC, storageConfig.Dedupe, c.Log, c.Metrics)
+						storageConfig.GC, storageConfig.Dedupe, storageConfig.Commit, c.Log, c.Metrics)
 				} else {
 					storeName := fmt.Sprintf("%v", storageConfig.StorageDriver["name"])
 					if storeName != storage.S3StorageDriverName {
@@ -282,7 +282,7 @@ func (c *Controller) InitImageStore() error {
 					}
 
 					subImageStore[route] = s3.NewImageStore(storageConfig.RootDirectory,
-						storageConfig.GC, storageConfig.Dedupe, c.Log, c.Metrics, store)
+						storageConfig.GC, storageConfig.Dedupe, storageConfig.Commit, c.Log, c.Metrics, store)
 				}
 
 				// Enable extensions if extension config is provided

pkg/api/controller_test.go

@@ -25,6 +25,7 @@ import (
 	"time"
 
 	"github.com/chartmuseum/auth"
+	"github.com/gorilla/mux"
 	"github.com/mitchellh/mapstructure"
 	vldap "github.com/nmcclain/ldap"
 	notreg "github.com/notaryproject/notation/pkg/registry"
@@ -2089,23 +2090,6 @@ func TestAuthorizationWithBasicAuth(t *testing.T) {
 		So(resp, ShouldNotBeNil)
 		So(resp.StatusCode(), ShouldEqual, http.StatusOK)
 
-		Convey("Hard to reach cases", func() {
-			injected := test.InjectFailure(0)
-
-			// get tags with read access should get 200
-			conf.AccessControl.Repositories[AuthorizationNamespace].Policies[0].Actions =
-				append(conf.AccessControl.Repositories[AuthorizationNamespace].Policies[0].Actions, "read")
-			resp, err = resty.R().SetBasicAuth(username, passphrase).
-				Get(baseURL + "/v2/" + AuthorizationNamespace + "/tags/list")
-			So(err, ShouldBeNil)
-			So(resp, ShouldNotBeNil)
-			if injected {
-				So(resp.StatusCode(), ShouldEqual, http.StatusNotFound)
-			} else {
-				So(resp.StatusCode(), ShouldEqual, http.StatusOK)
-			}
-		})
-
 		// head blob should get 200 now
 		resp, err = resty.R().SetBasicAuth(username, passphrase).
 			Head(baseURL + "/v2/" + AuthorizationNamespace + "/blobs/" + digest)
@@ -2818,16 +2802,28 @@ func TestParallelRequests(t *testing.T) {
 		panic(err)
 	}
 
+	t.Cleanup(func() {
+		os.RemoveAll(dir)
+	})
+
 	firstSubDir, err := ioutil.TempDir("", "oci-sub-dir")
 	if err != nil {
 		panic(err)
 	}
 
+	t.Cleanup(func() {
+		os.RemoveAll(firstSubDir)
+	})
+
 	secondSubDir, err := ioutil.TempDir("", "oci-sub-dir")
 	if err != nil {
 		panic(err)
 	}
 
+	t.Cleanup(func() {
+		os.RemoveAll(secondSubDir)
+	})
+
 	subPaths := make(map[string]config.StorageConfig)
 
 	subPaths["/a"] = config.StorageConfig{RootDirectory: firstSubDir}
@@ -2861,24 +2857,6 @@ func TestParallelRequests(t *testing.T) {
 				assert.Equal(t, err, nil, "Error should be nil")
 				assert.Equal(t, headResponse.StatusCode(), http.StatusNotFound, "response status code should return 404")
 
-				Convey("Hard to reach cases", t, func() {
-					_ = test.InjectFailure(0)
-
-					headResponse, err := client.R().SetBasicAuth(username, passphrase).
-						Head(baseURL + "/v2/" + testcase.destImageName + "/manifests/test:1.0")
-					assert.Equal(t, err, nil, "Error should be nil")
-					assert.Equal(t, headResponse.StatusCode(), http.StatusNotFound, "response status code should return 404")
-				})
-
-				Convey("Hard to reach cases", t, func() {
-					_ = test.InjectFailure(1)
-
-					headResponse, err := client.R().SetBasicAuth(username, passphrase).
-						Head(baseURL + "/v2/" + testcase.destImageName + "/manifests/test:1.0")
-					assert.Equal(t, err, nil, "Error should be nil")
-					assert.Equal(t, headResponse.StatusCode(), http.StatusNotFound, "response status code should return 404")
-				})
-
 				getResponse, err := client.R().SetBasicAuth(username, passphrase).
 					Get(baseURL + "/v2/" + testcase.destImageName + "/manifests/" + manifest)
 				assert.Equal(t, err, nil, "Error should be nil")
@@ -3158,6 +3136,7 @@ func TestImageSignatures(t *testing.T) {
 		content := []byte("this is a blob")
 		digest := godigest.FromBytes(content)
 		So(digest, ShouldNotBeNil)
+
 		// monolithic blob upload: success
 		resp, err = resty.R().SetQueryParam("digest", digest.String()).
 			SetHeader("Content-Type", "application/octet-stream").SetBody(content).Put(loc)
@@ -3447,6 +3426,413 @@ func TestImageSignatures(t *testing.T) {
 	})
 }
 
+func TestRouteFailures(t *testing.T) {
+	Convey("Make a new controller", t, func() {
+		port := test.GetFreePort()
+		baseURL := test.GetBaseURL(port)
+		conf := config.New()
+		conf.HTTP.Port = port
+
+		ctlr := api.NewController(conf)
+		dir, err := ioutil.TempDir("", "oci-repo-test")
+		if err != nil {
+			panic(err)
+		}
+		defer os.RemoveAll(dir)
+		ctlr.Config.Storage.RootDirectory = dir
+		ctlr.Config.Storage.Commit = true
+
+		go startServer(ctlr)
+		defer stopServer(ctlr)
+		test.WaitTillServerReady(baseURL)
+
+		rthdlr := api.NewRouteHandler(ctlr)
+
+		Convey("List tags", func() {
+			request, _ := http.NewRequestWithContext(context.TODO(), "GET", baseURL+"/v2/foo/tags/list", nil)
+			mux.SetURLVars(request, map[string]string{})
+			response := httptest.NewRecorder()
+
+			rthdlr.ListTags(response, request)
+
+			resp := response.Result()
+			defer resp.Body.Close()
+			So(resp, ShouldNotBeNil)
+			So(resp.StatusCode, ShouldEqual, http.StatusNotFound)
+
+			request, _ = http.NewRequestWithContext(context.TODO(), "GET", baseURL+"/v2/foo/tags/list", nil)
+			request = mux.SetURLVars(request, map[string]string{"name": "foo"})
+			response = httptest.NewRecorder()
+
+			rthdlr.ListTags(response, request)
+
+			resp = response.Result()
+			defer resp.Body.Close()
+			So(resp, ShouldNotBeNil)
+			So(resp.StatusCode, ShouldEqual, http.StatusNotFound)
+
+			request, _ = http.NewRequestWithContext(context.TODO(), "GET", baseURL+"/v2/foo/tags/list", nil)
+			request = mux.SetURLVars(request, map[string]string{"name": "foo"})
+			qparm := request.URL.Query()
+			qparm.Add("n", "a")
+			request.URL.RawQuery = qparm.Encode()
+			response = httptest.NewRecorder()
+
+			rthdlr.ListTags(response, request)
+
+			resp = response.Result()
+			defer resp.Body.Close()
+			So(resp, ShouldNotBeNil)
+			So(resp.StatusCode, ShouldEqual, http.StatusBadRequest)
+
+			request, _ = http.NewRequestWithContext(context.TODO(), "GET", baseURL+"/v2/foo/tags/list", nil)
+			request = mux.SetURLVars(request, map[string]string{"name": "foo"})
+			qparm = request.URL.Query()
+			qparm.Add("n", "abc")
+			request.URL.RawQuery = qparm.Encode()
+			response = httptest.NewRecorder()
+
+			rthdlr.ListTags(response, request)
+
+			resp = response.Result()
+			defer resp.Body.Close()
+			So(resp, ShouldNotBeNil)
+			So(resp.StatusCode, ShouldEqual, http.StatusBadRequest)
+
+			request, _ = http.NewRequestWithContext(context.TODO(), "GET", baseURL+"/v2/foo/tags/list", nil)
+			request = mux.SetURLVars(request, map[string]string{"name": "foo"})
+			qparm = request.URL.Query()
+			qparm.Add("n", "a")
+			qparm.Add("n", "abc")
+			request.URL.RawQuery = qparm.Encode()
+			response = httptest.NewRecorder()
+
+			rthdlr.ListTags(response, request)
+
+			resp = response.Result()
+			defer resp.Body.Close()
+			So(resp, ShouldNotBeNil)
+			So(resp.StatusCode, ShouldEqual, http.StatusBadRequest)
+
+			request, _ = http.NewRequestWithContext(context.TODO(), "GET", baseURL+"/v2/foo/tags/list", nil)
+			request = mux.SetURLVars(request, map[string]string{"name": "foo"})
+			qparm = request.URL.Query()
+			qparm.Add("n", "0")
+			request.URL.RawQuery = qparm.Encode()
+			response = httptest.NewRecorder()
+
+			rthdlr.ListTags(response, request)
+
+			resp = response.Result()
+			defer resp.Body.Close()
+			So(resp, ShouldNotBeNil)
+			So(resp.StatusCode, ShouldEqual, http.StatusNotFound)
+
+			request, _ = http.NewRequestWithContext(context.TODO(), "GET", baseURL+"/v2/foo/tags/list", nil)
+			request = mux.SetURLVars(request, map[string]string{"name": "foo"})
+			qparm = request.URL.Query()
+			qparm.Add("n", "1")
+			qparm.Add("last", "")
+			request.URL.RawQuery = qparm.Encode()
+			response = httptest.NewRecorder()
+
+			rthdlr.ListTags(response, request)
+
+			resp = response.Result()
+			defer resp.Body.Close()
+			So(resp, ShouldNotBeNil)
+			So(resp.StatusCode, ShouldEqual, http.StatusNotFound)
+
+			request, _ = http.NewRequestWithContext(context.TODO(), "GET", baseURL+"/v2/foo/tags/list", nil)
+			request = mux.SetURLVars(request, map[string]string{"name": "foo"})
+			qparm = request.URL.Query()
+			qparm.Add("n", "1")
+			qparm.Add("last", "a")
+			request.URL.RawQuery = qparm.Encode()
+			response = httptest.NewRecorder()
+
+			rthdlr.ListTags(response, request)
+
+			resp = response.Result()
+			defer resp.Body.Close()
+			So(resp, ShouldNotBeNil)
+			So(resp.StatusCode, ShouldEqual, http.StatusNotFound)
+
+			request, _ = http.NewRequestWithContext(context.TODO(), "GET", baseURL+"/v2/foo/tags/list", nil)
+			request = mux.SetURLVars(request, map[string]string{"name": "foo"})
+			qparm = request.URL.Query()
+			qparm.Add("n", "1")
+			qparm.Add("last", "a")
+			qparm.Add("last", "abc")
+			request.URL.RawQuery = qparm.Encode()
+			response = httptest.NewRecorder()
+
+			rthdlr.ListTags(response, request)
+
+			resp = response.Result()
+			defer resp.Body.Close()
+			So(resp, ShouldNotBeNil)
+			So(resp.StatusCode, ShouldEqual, http.StatusBadRequest)
+		})
+
+		Convey("Check manifest", func() {
+			request, _ := http.NewRequestWithContext(context.TODO(), "HEAD", baseURL+"/v2/foo/manifests/test:1.0", nil)
+			request = mux.SetURLVars(request, map[string]string{})
+			response := httptest.NewRecorder()
+
+			rthdlr.CheckManifest(response, request)
+
+			resp := response.Result()
+			defer resp.Body.Close()
+			So(resp, ShouldNotBeNil)
+			So(resp.StatusCode, ShouldEqual, http.StatusNotFound)
+
+			request, _ = http.NewRequestWithContext(context.TODO(), "HEAD", baseURL+"/v2/foo/manifests/test:1.0", nil)
+			request = mux.SetURLVars(request, map[string]string{"name": "foo"})
+			response = httptest.NewRecorder()
+
+			rthdlr.CheckManifest(response, request)
+
+			resp = response.Result()
+			defer resp.Body.Close()
+			So(resp, ShouldNotBeNil)
+			So(resp.StatusCode, ShouldEqual, http.StatusNotFound)
+
+			request, _ = http.NewRequestWithContext(context.TODO(), "HEAD", baseURL+"/v2/foo/manifests/test:1.0", nil)
+			request = mux.SetURLVars(request, map[string]string{"name": "foo", "reference": ""})
+			response = httptest.NewRecorder()
+
+			rthdlr.CheckManifest(response, request)
+
+			resp = response.Result()
+			defer resp.Body.Close()
+			So(resp, ShouldNotBeNil)
+			So(resp.StatusCode, ShouldEqual, http.StatusNotFound)
+		})
+	})
+}
+
+func TestStorageCommit(t *testing.T) {
+	Convey("Make a new controller", t, func() {
+		port := test.GetFreePort()
+		baseURL := test.GetBaseURL(port)
+		conf := config.New()
+		conf.HTTP.Port = port
+
+		ctlr := api.NewController(conf)
+		dir, err := ioutil.TempDir("", "oci-repo-test")
+		if err != nil {
+			panic(err)
+		}
+		defer os.RemoveAll(dir)
+		ctlr.Config.Storage.RootDirectory = dir
+		ctlr.Config.Storage.Commit = true
+
+		go startServer(ctlr)
+		defer stopServer(ctlr)
+		test.WaitTillServerReady(baseURL)
+
+		Convey("Manifests", func() {
+			_, _ = Print("\nManifests")
+			// create a blob/layer
+			resp, err := resty.R().Post(baseURL + "/v2/repo7/blobs/uploads/")
+			So(err, ShouldBeNil)
+			So(resp.StatusCode(), ShouldEqual, http.StatusAccepted)
+			loc := test.Location(baseURL, resp)
+			So(loc, ShouldNotBeEmpty)
+
+			// since we are not specifying any prefix i.e provided in config while starting server,
+			// so it should store repo7 to global root dir
+			_, err = os.Stat(path.Join(dir, "repo7"))
+			So(err, ShouldBeNil)
+
+			resp, err = resty.R().Get(loc)
+			So(err, ShouldBeNil)
+			So(resp.StatusCode(), ShouldEqual, http.StatusNoContent)
+			content := []byte("this is a blob5")
+			digest := godigest.FromBytes(content)
+			So(digest, ShouldNotBeNil)
+			// monolithic blob upload: success
+			resp, err = resty.R().SetQueryParam("digest", digest.String()).
+				SetHeader("Content-Type", "application/octet-stream").SetBody(content).Put(loc)
+			So(err, ShouldBeNil)
+			So(resp.StatusCode(), ShouldEqual, http.StatusCreated)
+			blobLoc := resp.Header().Get("Location")
+			So(blobLoc, ShouldNotBeEmpty)
+			So(resp.Header().Get("Content-Length"), ShouldEqual, "0")
+			So(resp.Header().Get(api.DistContentDigestKey), ShouldNotBeEmpty)
+
+			// check a non-existent manifest
+			resp, err = resty.R().SetHeader("Content-Type", "application/vnd.oci.image.manifest.v1+json").
+				SetBody(content).Head(baseURL + "/v2/unknown/manifests/test:1.0")
+			So(err, ShouldBeNil)
+			So(resp.StatusCode(), ShouldEqual, http.StatusNotFound)
+
+			// upload image config blob
+			resp, err = resty.R().Post(baseURL + "/v2/repo7/blobs/uploads/")
+			So(err, ShouldBeNil)
+			So(resp.StatusCode(), ShouldEqual, http.StatusAccepted)
+			loc = test.Location(baseURL, resp)
+			cblob, cdigest := test.GetRandomImageConfig()
+
+			resp, err = resty.R().
+				SetContentLength(true).
+				SetHeader("Content-Length", fmt.Sprintf("%d", len(cblob))).
+				SetHeader("Content-Type", "application/octet-stream").
+				SetQueryParam("digest", cdigest.String()).
+				SetBody(cblob).
+				Put(loc)
+			So(err, ShouldBeNil)
+			So(resp.StatusCode(), ShouldEqual, http.StatusCreated)
+
+			// create a manifest
+			manifest := ispec.Manifest{
+				Config: ispec.Descriptor{
+					MediaType: "application/vnd.oci.image.config.v1+json",
+					Digest:    cdigest,
+					Size:      int64(len(cblob)),
+				},
+				Layers: []ispec.Descriptor{
+					{
+						MediaType: "application/vnd.oci.image.layer.v1.tar",
+						Digest:    digest,
+						Size:      int64(len(content)),
+					},
+				},
+			}
+			manifest.SchemaVersion = 2
+			content, err = json.Marshal(manifest)
+			So(err, ShouldBeNil)
+			digest = godigest.FromBytes(content)
+			So(digest, ShouldNotBeNil)
+			resp, err = resty.R().SetHeader("Content-Type", "application/vnd.oci.image.manifest.v1+json").
+				SetBody(content).Put(baseURL + "/v2/repo7/manifests/test:1.0")
+			So(err, ShouldBeNil)
+			So(resp.StatusCode(), ShouldEqual, http.StatusCreated)
+			digestHdr := resp.Header().Get(api.DistContentDigestKey)
+			So(digestHdr, ShouldNotBeEmpty)
+			So(digestHdr, ShouldEqual, digest.String())
+
+			resp, err = resty.R().SetHeader("Content-Type", "application/vnd.oci.image.manifest.v1+json").
+				SetBody(content).Put(baseURL + "/v2/repo7/manifests/test:1.0.1")
+			So(err, ShouldBeNil)
+			So(resp.StatusCode(), ShouldEqual, http.StatusCreated)
+			digestHdr = resp.Header().Get(api.DistContentDigestKey)
+			So(digestHdr, ShouldNotBeEmpty)
+			So(digestHdr, ShouldEqual, digest.String())
+
+			content = []byte("this is a blob5")
+			digest = godigest.FromBytes(content)
+			So(digest, ShouldNotBeNil)
+
+			// upload image config blob
+			resp, err = resty.R().Post(baseURL + "/v2/repo7/blobs/uploads/")
+			So(err, ShouldBeNil)
+			So(resp.StatusCode(), ShouldEqual, http.StatusAccepted)
+			loc = test.Location(baseURL, resp)
+			cblob, cdigest = test.GetRandomImageConfig()
+
+			resp, err = resty.R().
+				SetContentLength(true).
+				SetHeader("Content-Length", fmt.Sprintf("%d", len(cblob))).
+				SetHeader("Content-Type", "application/octet-stream").
+				SetQueryParam("digest", cdigest.String()).
+				SetBody(cblob).
+				Put(loc)
+			So(err, ShouldBeNil)
+			So(resp.StatusCode(), ShouldEqual, http.StatusCreated)
+
+			// create a manifest with same blob but a different tag
+			manifest = ispec.Manifest{
+				Config: ispec.Descriptor{
+					MediaType: "application/vnd.oci.image.config.v1+json",
+					Digest:    cdigest,
+					Size:      int64(len(cblob)),
+				},
+				Layers: []ispec.Descriptor{
+					{
+						MediaType: "application/vnd.oci.image.layer.v1.tar",
+						Digest:    digest,
+						Size:      int64(len(content)),
+					},
+				},
+			}
+			manifest.SchemaVersion = 2
+			content, err = json.Marshal(manifest)
+			So(err, ShouldBeNil)
+			digest = godigest.FromBytes(content)
+			So(digest, ShouldNotBeNil)
+			resp, err = resty.R().SetHeader("Content-Type", "application/vnd.oci.image.manifest.v1+json").
+				SetBody(content).Put(baseURL + "/v2/repo7/manifests/test:2.0")
+			So(err, ShouldBeNil)
+			So(resp.StatusCode(), ShouldEqual, http.StatusCreated)
+			digestHdr = resp.Header().Get(api.DistContentDigestKey)
+			So(digestHdr, ShouldNotBeEmpty)
+			So(digestHdr, ShouldEqual, digest.String())
+
+			// check/get by tag
+			resp, err = resty.R().Head(baseURL + "/v2/repo7/manifests/test:1.0")
+			So(err, ShouldBeNil)
+			So(resp.StatusCode(), ShouldEqual, http.StatusOK)
+			So(resp.Header().Get("Content-Type"), ShouldNotBeEmpty)
+			resp, err = resty.R().Get(baseURL + "/v2/repo7/manifests/test:1.0")
+			So(err, ShouldBeNil)
+			So(resp.StatusCode(), ShouldEqual, http.StatusOK)
+			So(resp.Body(), ShouldNotBeEmpty)
+			// check/get by reference
+			resp, err = resty.R().Head(baseURL + "/v2/repo7/manifests/" + digest.String())
+			So(err, ShouldBeNil)
+			So(resp.StatusCode(), ShouldEqual, http.StatusOK)
+			So(resp.Header().Get("Content-Type"), ShouldNotBeEmpty)
+			resp, err = resty.R().Get(baseURL + "/v2/repo7/manifests/" + digest.String())
+			So(err, ShouldBeNil)
+			So(resp.StatusCode(), ShouldEqual, http.StatusOK)
+			So(resp.Body(), ShouldNotBeEmpty)
+
+			// delete manifest by tag should pass
+			resp, err = resty.R().Delete(baseURL + "/v2/repo7/manifests/test:1.0")
+			So(err, ShouldBeNil)
+			So(resp.StatusCode(), ShouldEqual, http.StatusAccepted)
+			// delete manifest by digest (1.0 deleted but 1.0.1 has same reference)
+			resp, err = resty.R().Delete(baseURL + "/v2/repo7/manifests/" + digest.String())
+			So(err, ShouldBeNil)
+			So(resp.StatusCode(), ShouldEqual, http.StatusAccepted)
+			// delete manifest by digest
+			resp, err = resty.R().Delete(baseURL + "/v2/repo7/manifests/" + digest.String())
+			So(err, ShouldBeNil)
+			So(resp.StatusCode(), ShouldEqual, http.StatusNotFound)
+			// delete again should fail
+			resp, err = resty.R().Delete(baseURL + "/v2/repo7/manifests/" + digest.String())
+			So(err, ShouldBeNil)
+			So(resp.StatusCode(), ShouldEqual, http.StatusNotFound)
+
+			// check/get by tag
+			resp, err = resty.R().Head(baseURL + "/v2/repo7/manifests/test:1.0")
+			So(err, ShouldBeNil)
+			So(resp.StatusCode(), ShouldEqual, http.StatusNotFound)
+			resp, err = resty.R().Get(baseURL + "/v2/repo7/manifests/test:1.0")
+			So(err, ShouldBeNil)
+			So(resp.StatusCode(), ShouldEqual, http.StatusNotFound)
+			So(resp.Body(), ShouldNotBeEmpty)
+			resp, err = resty.R().Head(baseURL + "/v2/repo7/manifests/test:2.0")
+			So(err, ShouldBeNil)
+			So(resp.StatusCode(), ShouldEqual, http.StatusNotFound)
+			resp, err = resty.R().Get(baseURL + "/v2/repo7/manifests/test:2.0")
+			So(err, ShouldBeNil)
+			So(resp.StatusCode(), ShouldEqual, http.StatusNotFound)
+			So(resp.Body(), ShouldNotBeEmpty)
+			// check/get by reference
+			resp, err = resty.R().Head(baseURL + "/v2/repo7/manifests/" + digest.String())
+			So(err, ShouldBeNil)
+			So(resp.StatusCode(), ShouldEqual, http.StatusNotFound)
+			resp, err = resty.R().Get(baseURL + "/v2/repo7/manifests/" + digest.String())
+			So(err, ShouldBeNil)
+			So(resp.StatusCode(), ShouldEqual, http.StatusNotFound)
+			So(resp.Body(), ShouldNotBeEmpty)
+		})
+	})
+}
+
 func getAllBlobs(imagePath string) []string {
 	blobList := make([]string, 0)
 

pkg/api/routes.go

@@ -32,7 +32,6 @@ import (
 	ext "zotregistry.io/zot/pkg/extensions"
 	"zotregistry.io/zot/pkg/log"
 	"zotregistry.io/zot/pkg/storage"
-	"zotregistry.io/zot/pkg/test"
 
 	// as required by swaggo.
 	_ "zotregistry.io/zot/swagger"
@@ -166,7 +165,7 @@ func (rh *RouteHandler) ListTags(response http.ResponseWriter, request *http.Req
 
 	name, ok := vars["name"]
 
-	if !test.Ok(ok) || name == "" {
+	if !ok || name == "" {
 		response.WriteHeader(http.StatusNotFound)
 
 		return
@@ -288,7 +287,7 @@ func (rh *RouteHandler) CheckManifest(response http.ResponseWriter, request *htt
 	vars := mux.Vars(request)
 	name, ok := vars["name"]
 
-	if !test.Ok(ok) || name == "" {
+	if !ok || name == "" {
 		response.WriteHeader(http.StatusNotFound)
 
 		return
@@ -297,7 +296,7 @@ func (rh *RouteHandler) CheckManifest(response http.ResponseWriter, request *htt
 	imgStore := rh.getImageStore(name)
 
 	reference, ok := vars["reference"]
-	if !test.Ok(ok) || reference == "" {
+	if !ok || reference == "" {
 		WriteJSON(response,
 			http.StatusNotFound,
 			NewErrorList(NewError(MANIFEST_INVALID, map[string]string{"reference": reference})))

pkg/extensions/search/common/common_test.go

@@ -123,7 +123,7 @@ func TestImageFormat(t *testing.T) {
 		dbDir := "../../../../test/data"
 
 		metrics := monitoring.NewMetricsServer(false, log)
-		defaultStore := storage.NewImageStore(dbDir, false, false, log, metrics)
+		defaultStore := storage.NewImageStore(dbDir, false, false, false, log, metrics)
 		storeController := storage.StoreController{DefaultStore: defaultStore}
 		olu := common.NewOciLayoutUtils(storeController, log)
 
@@ -411,9 +411,9 @@ func TestUtilsMethod(t *testing.T) {
 		defer os.RemoveAll(subRootDir)
 
 		metrics := monitoring.NewMetricsServer(false, log)
-		defaultStore := storage.NewImageStore(rootDir, false, false, log, metrics)
+		defaultStore := storage.NewImageStore(rootDir, false, false, false, log, metrics)
 
-		subStore := storage.NewImageStore(subRootDir, false, false, log, metrics)
+		subStore := storage.NewImageStore(subRootDir, false, false, false, log, metrics)
 
 		subStoreMap := make(map[string]storage.ImageStore)
 

pkg/extensions/search/cve/cve_test.go

@@ -90,7 +90,7 @@ func testSetup() error {
 	log := log.NewLogger("debug", "")
 	metrics := monitoring.NewMetricsServer(false, log)
 
-	storeController := storage.StoreController{DefaultStore: storage.NewImageStore(dir, false, false, log, metrics)}
+	storeController := storage.StoreController{DefaultStore: storage.NewImageStore(dir, false, false, false, log, metrics)}
 
 	layoutUtils := common.NewOciLayoutUtils(storeController, log)
 
@@ -347,11 +347,11 @@ func TestMultipleStoragePath(t *testing.T) {
 		metrics := monitoring.NewMetricsServer(false, log)
 
 		// Create ImageStore
-		firstStore := storage.NewImageStore(firstRootDir, false, false, log, metrics)
+		firstStore := storage.NewImageStore(firstRootDir, false, false, false, log, metrics)
 
-		secondStore := storage.NewImageStore(secondRootDir, false, false, log, metrics)
+		secondStore := storage.NewImageStore(secondRootDir, false, false, false, log, metrics)
 
-		thirdStore := storage.NewImageStore(thirdRootDir, false, false, log, metrics)
+		thirdStore := storage.NewImageStore(thirdRootDir, false, false, false, log, metrics)
 
 		storeController := storage.StoreController{}
 

pkg/extensions/search/digest/digest_test.go

@@ -97,7 +97,9 @@ func testSetup() error {
 
 	log := log.NewLogger("debug", "")
 	metrics := monitoring.NewMetricsServer(false, log)
-	storeController := storage.StoreController{DefaultStore: storage.NewImageStore(rootDir, false, false, log, metrics)}
+	storeController := storage.StoreController{
+		DefaultStore: storage.NewImageStore(rootDir, false, false, false, log, metrics),
+	}
 
 	digestInfo = digestinfo.NewDigestInfo(storeController, log)
 

pkg/extensions/sync/sync_internal_test.go

@@ -268,7 +268,7 @@ func TestSyncInternal(t *testing.T) {
 		log := log.Logger{Logger: zerolog.New(os.Stdout)}
 		metrics := monitoring.NewMetricsServer(false, log)
 
-		imageStore := storage.NewImageStore(storageDir, false, false, log, metrics)
+		imageStore := storage.NewImageStore(storageDir, false, false, false, log, metrics)
 
 		storeController := storage.StoreController{}
 		storeController.DefaultStore = imageStore
@@ -289,7 +289,7 @@ func TestSyncInternal(t *testing.T) {
 			panic(err)
 		}
 
-		testImageStore := storage.NewImageStore(testRootDir, false, false, log, metrics)
+		testImageStore := storage.NewImageStore(testRootDir, false, false, false, log, metrics)
 		manifestContent, _, _, err := testImageStore.GetImageManifest(testImage, testImageTag)
 		So(err, ShouldBeNil)
 

pkg/extensions/sync/utils.go

@@ -442,7 +442,7 @@ func pushSyncedLocalImage(repo, tag, localCachePath string,
 	imageStore := storeController.GetImageStore(repo)
 
 	metrics := monitoring.NewMetricsServer(false, log)
-	cacheImageStore := storage.NewImageStore(localCachePath, false, false, log, metrics)
+	cacheImageStore := storage.NewImageStore(localCachePath, false, false, false, log, metrics)
 
 	manifestContent, _, _, err := cacheImageStore.GetImageManifest(repo, tag)
 	if err != nil {

pkg/log/log.go

@@ -68,8 +68,8 @@ func NewAuditLogger(level string, audit string) *Logger {
 	return &Logger{Logger: auditLog.With().Timestamp().Logger()}
 }
 
-// goroutineID adds goroutine-id to logs to help debug concurrency issues.
+// GoroutineID adds goroutine-id to logs to help debug concurrency issues.
-func goroutineID() int {
+func GoroutineID() int {
 	var buf [64]byte
 	n := runtime.Stack(buf[:], false)
 	idField := strings.Fields(strings.TrimPrefix(string(buf[:n]), "goroutine "))[0]
@@ -86,6 +86,6 @@ type goroutineHook struct{}
 
 func (h goroutineHook) Run(e *zerolog.Event, level zerolog.Level, msg string) {
 	if level != zerolog.NoLevel {
-		e.Int("goroutine", goroutineID())
+		e.Int("goroutine", GoroutineID())
 	}
 }

pkg/storage/s3/s3_test.go

@@ -53,7 +53,7 @@ func skipIt(t *testing.T) {
 func createMockStorage(rootDir string, store driver.StorageDriver) storage.ImageStore {
 	log := log.Logger{Logger: zerolog.New(os.Stdout)}
 	metrics := monitoring.NewMetricsServer(false, log)
-	il := s3.NewImageStore(rootDir, false, false, log, metrics, store)
+	il := s3.NewImageStore(rootDir, false, false, false, log, metrics, store)
 
 	return il
 }
@@ -86,7 +86,7 @@ func createObjectsStore(rootDir string) (driver.StorageDriver, storage.ImageStor
 
 	log := log.Logger{Logger: zerolog.New(os.Stdout)}
 	metrics := monitoring.NewMetricsServer(false, log)
-	il := s3.NewImageStore(rootDir, false, false, log, metrics, store)
+	il := s3.NewImageStore(rootDir, false, false, false, log, metrics, store)
 
 	return store, il, err
 }

pkg/storage/s3/storage.go

@@ -63,7 +63,7 @@ func (is *ObjectStorage) DirExists(d string) bool {
 
 // NewObjectStorage returns a new image store backed by cloud storages.
 // see https://github.com/docker/docker.github.io/tree/master/registry/storage-drivers
-func NewImageStore(rootDir string, gc bool, dedupe bool, log zlog.Logger, metrics monitoring.MetricServer,
+func NewImageStore(rootDir string, gc, dedupe, commit bool, log zlog.Logger, metrics monitoring.MetricServer,
 	store driver.StorageDriver) storage.ImageStore {
 	imgStore := &ObjectStorage{
 		rootDir:           rootDir,

pkg/storage/scrub_test.go

@@ -36,7 +36,7 @@ func TestCheckAllBlobsIntegrity(t *testing.T) {
 
 	metrics := monitoring.NewMetricsServer(false, log)
 
-	imgStore := storage.NewImageStore(dir, true, true, log, metrics)
+	imgStore := storage.NewImageStore(dir, true, true, true, log, metrics)
 
 	Convey("Scrub only one repo", t, func(c C) {
 		// initialize repo

pkg/storage/storage_fs.go

@@ -29,6 +29,7 @@ import (
 	zerr "zotregistry.io/zot/errors"
 	"zotregistry.io/zot/pkg/extensions/monitoring"
 	zlog "zotregistry.io/zot/pkg/log"
+	"zotregistry.io/zot/pkg/test"
 )
 
 const (
@@ -61,6 +62,7 @@ type ImageStoreFS struct {
 	cache       *Cache
 	gc          bool
 	dedupe      bool
+	commit      bool
 	log         zerolog.Logger
 	metrics     monitoring.MetricServer
 }
@@ -103,7 +105,8 @@ func (sc StoreController) GetImageStore(name string) ImageStore {
 }
 
 // NewImageStore returns a new image store backed by a file storage.
-func NewImageStore(rootDir string, gc bool, dedupe bool, log zlog.Logger, metrics monitoring.MetricServer) ImageStore {
+func NewImageStore(rootDir string, gc, dedupe, commit bool,
+	log zlog.Logger, metrics monitoring.MetricServer) ImageStore {
 	if _, err := os.Stat(rootDir); os.IsNotExist(err) {
 		if err := os.MkdirAll(rootDir, DefaultDirPerms); err != nil {
 			log.Error().Err(err).Str("rootDir", rootDir).Msg("unable to create root dir")
@@ -118,6 +121,7 @@ func NewImageStore(rootDir string, gc bool, dedupe bool, log zlog.Logger, metric
 		blobUploads: make(map[string]BlobUpload),
 		gc:          gc,
 		dedupe:      dedupe,
+		commit:      commit,
 		log:         log.With().Caller().Logger(),
 		metrics:     metrics,
 	}
@@ -203,7 +207,7 @@ func (is *ImageStoreFS) initRepo(name string) error {
 			is.log.Panic().Err(err).Msg("unable to marshal JSON")
 		}
 
-		if err := ioutil.WriteFile(ilPath, buf, DefaultFilePerms); err != nil {
+		if err := is.writeFile(ilPath, buf); err != nil {
 			is.log.Error().Err(err).Str("file", ilPath).Msg("unable to write file")
 
 			return err
@@ -221,7 +225,7 @@ func (is *ImageStoreFS) initRepo(name string) error {
 			is.log.Panic().Err(err).Msg("unable to marshal JSON")
 		}
 
-		if err := ioutil.WriteFile(indexPath, buf, DefaultFilePerms); err != nil {
+		if err := is.writeFile(indexPath, buf); err != nil {
 			is.log.Error().Err(err).Str("file", indexPath).Msg("unable to write file")
 
 			return err
@@ -660,7 +664,7 @@ func (is *ImageStoreFS) PutImageManifest(repo string, reference string, mediaTyp
 	_ = ensureDir(dir, is.log)
 	file := path.Join(dir, mDigest.Encoded())
 
-	if err := ioutil.WriteFile(file, body, DefaultFilePerms); err != nil {
+	if err := is.writeFile(file, body); err != nil {
 		is.log.Error().Err(err).Str("file", file).Msg("unable to write")
 
 		return "", err
@@ -678,7 +682,7 @@ func (is *ImageStoreFS) PutImageManifest(repo string, reference string, mediaTyp
 		return "", err
 	}
 
-	if err := ioutil.WriteFile(file, buf, DefaultFilePerms); err != nil {
+	if err := is.writeFile(file, buf); err != nil {
 		is.log.Error().Err(err).Str("file", file).Msg("unable to write")
 
 		return "", err
@@ -781,7 +785,7 @@ func (is *ImageStoreFS) DeleteImageManifest(repo string, reference string) error
 		return err
 	}
 
-	if err := ioutil.WriteFile(file, buf, DefaultFilePerms); err != nil {
+	if err := is.writeFile(file, buf); err != nil {
 		return err
 	}
 
@@ -884,17 +888,20 @@ func (is *ImageStoreFS) PutBlobChunkStreamed(repo string, uuid string, body io.R
 		return -1, zerr.ErrUploadNotFound
 	}
 
-	file, err := os.OpenFile(
+	file, err := os.OpenFile(blobUploadPath, os.O_WRONLY|os.O_CREATE, DefaultFilePerms)
-		blobUploadPath,
-		os.O_WRONLY|os.O_CREATE,
-		DefaultFilePerms,
-	)
 	if err != nil {
 		is.log.Error().Err(err).Msg("failed to open file")
 
 		return -1, err
 	}
-	defer file.Close()
+
+	defer func() {
+		if is.commit {
+			_ = file.Sync()
+		}
+
+		_ = file.Close()
+	}()
 
 	if _, err := file.Seek(0, io.SeekEnd); err != nil {
 		is.log.Error().Err(err).Msg("failed to seek file")
@@ -929,17 +936,20 @@ func (is *ImageStoreFS) PutBlobChunk(repo string, uuid string, from int64, to in
 		return -1, zerr.ErrBadUploadRange
 	}
 
-	file, err := os.OpenFile(
+	file, err := os.OpenFile(blobUploadPath, os.O_WRONLY|os.O_CREATE, DefaultFilePerms)
-		blobUploadPath,
-		os.O_WRONLY|os.O_CREATE,
-		DefaultFilePerms,
-	)
 	if err != nil {
 		is.log.Error().Err(err).Msg("failed to open file")
 
 		return -1, err
 	}
-	defer file.Close()
+
+	defer func() {
+		if is.commit {
+			_ = file.Sync()
+		}
+
+		_ = file.Close()
+	}()
 
 	if _, err := file.Seek(from, io.SeekStart); err != nil {
 		is.log.Error().Err(err).Msg("failed to seek file")
@@ -1077,7 +1087,13 @@ func (is *ImageStoreFS) FullBlobUpload(repo string, body io.Reader, digest strin
 		return "", -1, zerr.ErrUploadNotFound
 	}
 
-	defer blobFile.Close()
+	defer func() {
+		if is.commit {
+			_ = blobFile.Sync()
+		}
+
+		_ = blobFile.Close()
+	}()
 
 	digester := sha256.New()
 	mw := io.MultiWriter(blobFile, digester)
@@ -1514,6 +1530,30 @@ func (is *ImageStoreFS) GetReferrers(repo, digest string, mediaType string) ([]a
 	return result, nil
 }
 
+func (is *ImageStoreFS) writeFile(filename string, data []byte) error {
+	if !is.commit {
+		return ioutil.WriteFile(filename, data, DefaultFilePerms)
+	}
+
+	fhandle, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, DefaultFilePerms)
+	if err != nil {
+		return err
+	}
+
+	_, err = fhandle.Write(data)
+
+	if err1 := test.Error(fhandle.Sync()); err1 != nil && err == nil {
+		err = err1
+		is.log.Error().Err(err).Str("filename", filename).Msg("unable to sync file")
+	}
+
+	if err1 := test.Error(fhandle.Close()); err1 != nil && err == nil {
+		err = err1
+	}
+
+	return err
+}
+
 func IsSupportedMediaType(mediaType string) bool {
 	return mediaType == ispec.MediaTypeImageManifest ||
 		mediaType == artifactspec.MediaTypeArtifactManifest

pkg/storage/storage_fs_test.go

@@ -34,7 +34,7 @@ func TestStorageFSAPIs(t *testing.T) {
 
 	log := log.Logger{Logger: zerolog.New(os.Stdout)}
 	metrics := monitoring.NewMetricsServer(false, log)
-	imgStore := storage.NewImageStore(dir, true, true, log, metrics)
+	imgStore := storage.NewImageStore(dir, true, true, true, log, metrics)
 
 	Convey("Repo layout", t, func(c C) {
 		repoName := "test"
@@ -171,7 +171,7 @@ func TestDedupeLinks(t *testing.T) {
 
 	log := log.Logger{Logger: zerolog.New(os.Stdout)}
 	metrics := monitoring.NewMetricsServer(false, log)
-	imgStore := storage.NewImageStore(dir, true, true, log, metrics)
+	imgStore := storage.NewImageStore(dir, true, true, true, log, metrics)
 
 	Convey("Dedupe", t, func(c C) {
 		// manifest1
@@ -311,7 +311,7 @@ func TestDedupe(t *testing.T) {
 
 			log := log.Logger{Logger: zerolog.New(os.Stdout)}
 			metrics := monitoring.NewMetricsServer(false, log)
-			il := storage.NewImageStore(dir, true, true, log, metrics)
+			il := storage.NewImageStore(dir, true, true, true, log, metrics)
 
 			So(il.DedupeBlob("", "", ""), ShouldNotBeNil)
 		})
@@ -330,9 +330,9 @@ func TestNegativeCases(t *testing.T) {
 		log := log.Logger{Logger: zerolog.New(os.Stdout)}
 		metrics := monitoring.NewMetricsServer(false, log)
 
-		So(storage.NewImageStore(dir, true, true, log, metrics), ShouldNotBeNil)
+		So(storage.NewImageStore(dir, true, true, true, log, metrics), ShouldNotBeNil)
 		if os.Geteuid() != 0 {
-			So(storage.NewImageStore("/deadBEEF", true, true, log, metrics), ShouldBeNil)
+			So(storage.NewImageStore("/deadBEEF", true, true, true, log, metrics), ShouldBeNil)
 		}
 	})
 
@@ -345,7 +345,7 @@ func TestNegativeCases(t *testing.T) {
 
 		log := log.Logger{Logger: zerolog.New(os.Stdout)}
 		metrics := monitoring.NewMetricsServer(false, log)
-		imgStore := storage.NewImageStore(dir, true, true, log, metrics)
+		imgStore := storage.NewImageStore(dir, true, true, true, log, metrics)
 
 		err = os.Chmod(dir, 0o000) // remove all perms
 		if err != nil {
@@ -384,7 +384,7 @@ func TestNegativeCases(t *testing.T) {
 
 		log := log.Logger{Logger: zerolog.New(os.Stdout)}
 		metrics := monitoring.NewMetricsServer(false, log)
-		imgStore := storage.NewImageStore(dir, true, true, log, metrics)
+		imgStore := storage.NewImageStore(dir, true, true, true, log, metrics)
 
 		So(imgStore, ShouldNotBeNil)
 		So(imgStore.InitRepo("test"), ShouldBeNil)
@@ -502,7 +502,7 @@ func TestNegativeCases(t *testing.T) {
 
 		log := log.Logger{Logger: zerolog.New(os.Stdout)}
 		metrics := monitoring.NewMetricsServer(false, log)
-		imgStore := storage.NewImageStore(dir, true, true, log, metrics)
+		imgStore := storage.NewImageStore(dir, true, true, true, log, metrics)
 
 		So(imgStore, ShouldNotBeNil)
 		So(imgStore.InitRepo("test"), ShouldBeNil)
@@ -529,7 +529,7 @@ func TestNegativeCases(t *testing.T) {
 
 		log := log.Logger{Logger: zerolog.New(os.Stdout)}
 		metrics := monitoring.NewMetricsServer(false, log)
-		imgStore := storage.NewImageStore(dir, true, true, log, metrics)
+		imgStore := storage.NewImageStore(dir, true, true, true, log, metrics)
 
 		So(imgStore, ShouldNotBeNil)
 		So(imgStore.InitRepo("test"), ShouldBeNil)
@@ -574,7 +574,7 @@ func TestNegativeCases(t *testing.T) {
 
 		log := log.Logger{Logger: zerolog.New(os.Stdout)}
 		metrics := monitoring.NewMetricsServer(false, log)
-		imgStore := storage.NewImageStore(dir, true, true, log, metrics)
+		imgStore := storage.NewImageStore(dir, true, true, true, log, metrics)
 
 		So(imgStore, ShouldNotBeNil)
 		So(imgStore.InitRepo("test"), ShouldBeNil)
@@ -636,7 +636,7 @@ func TestNegativeCases(t *testing.T) {
 
 		log := log.Logger{Logger: zerolog.New(os.Stdout)}
 		metrics := monitoring.NewMetricsServer(false, log)
-		imgStore := storage.NewImageStore(dir, true, true, log, metrics)
+		imgStore := storage.NewImageStore(dir, true, true, true, log, metrics)
 
 		upload, err := imgStore.NewBlobUpload("dedupe1")
 		So(err, ShouldBeNil)
@@ -788,6 +788,55 @@ func TestHardLink(t *testing.T) {
 	})
 }
 
+func TestWriteFile(t *testing.T) {
+	Convey("writeFile with commit", t, func() {
+		dir, err := ioutil.TempDir("", "oci-repo-test")
+		So(err, ShouldBeNil)
+		defer os.RemoveAll(dir)
+
+		log := log.Logger{Logger: zerolog.New(os.Stdout)}
+		metrics := monitoring.NewMetricsServer(false, log)
+		imgStore := storage.NewImageStore(dir, true, true, true, log, metrics)
+
+		Convey("Failure path1", func() {
+			injected := test.InjectFailure(0)
+
+			err := imgStore.InitRepo("repo1")
+			if injected {
+				So(err, ShouldNotBeNil)
+			} else {
+				So(err, ShouldBeNil)
+			}
+		})
+
+		Convey("Failure path2", func() {
+			injected := test.InjectFailure(1)
+
+			err := imgStore.InitRepo("repo2")
+			if injected {
+				So(err, ShouldNotBeNil)
+			} else {
+				So(err, ShouldBeNil)
+			}
+		})
+	})
+
+	Convey("writeFile without commit", t, func() {
+		dir, err := ioutil.TempDir("", "oci-repo-test")
+		So(err, ShouldBeNil)
+		defer os.RemoveAll(dir)
+
+		log := log.Logger{Logger: zerolog.New(os.Stdout)}
+		metrics := monitoring.NewMetricsServer(false, log)
+		imgStore := storage.NewImageStore(dir, true, true, false, log, metrics)
+
+		Convey("Failure path not reached", func() {
+			err := imgStore.InitRepo("repo1")
+			So(err, ShouldBeNil)
+		})
+	})
+}
+
 func randSeq(n int) string {
 	letters := []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
 

pkg/storage/storage_test.go

@@ -72,7 +72,7 @@ func createObjectsStore(rootDir string) (driver.StorageDriver, storage.ImageStor
 	log := log.Logger{Logger: zerolog.New(os.Stdout)}
 	metrics := monitoring.NewMetricsServer(false, log)
 
-	il := s3.NewImageStore(rootDir, false, false, log, metrics, store)
+	il := s3.NewImageStore(rootDir, false, false, false, log, metrics, store)
 
 	return store, il, err
 }
@@ -120,7 +120,7 @@ func TestStorageAPIs(t *testing.T) {
 
 				log := log.Logger{Logger: zerolog.New(os.Stdout)}
 				metrics := monitoring.NewMetricsServer(false, log)
-				imgStore = storage.NewImageStore(dir, true, true, log, metrics)
+				imgStore = storage.NewImageStore(dir, true, true, true, log, metrics)
 			}
 
 			Convey("Repo layout", t, func(c C) {
@@ -711,11 +711,11 @@ func TestStorageHandler(t *testing.T) {
 				metrics := monitoring.NewMetricsServer(false, log)
 
 				// Create ImageStore
-				firstStore = storage.NewImageStore(firstRootDir, false, false, log, metrics)
+				firstStore = storage.NewImageStore(firstRootDir, false, false, false, log, metrics)
 
-				secondStore = storage.NewImageStore(secondRootDir, false, false, log, metrics)
+				secondStore = storage.NewImageStore(secondRootDir, false, false, false, log, metrics)
 
-				thirdStore = storage.NewImageStore(thirdRootDir, false, false, log, metrics)
+				thirdStore = storage.NewImageStore(thirdRootDir, false, false, false, log, metrics)
 			}
 
 			Convey("Test storage handler", t, func() {

pkg/test/dev.go

@@ -9,6 +9,7 @@ import (
 	"sync"
 
 	zerr "zotregistry.io/zot/errors"
+	"zotregistry.io/zot/pkg/log"
 )
 
 func Ok(ok bool) bool {
@@ -42,40 +43,49 @@ func Error(err error) error {
  **/
 
 type inject struct {
-	skip    int
+	skip int
-	enabled bool
 }
 
 //nolint:gochecknoglobals // only used by test code
-var (
+var injMap sync.Map
-	injlock sync.Mutex
-	injst   = inject{}
-)
 
 func InjectFailure(skip int) bool {
-	injlock.Lock()
+	gid := log.GoroutineID()
-	injst = inject{enabled: true, skip: skip}
+	if gid < 0 {
-	injlock.Unlock()
+		panic("invalid goroutine id")
+	}
+
+	if _, ok := injMap.Load(gid); ok {
+		panic("prior incomplete fault injection")
+	}
+
+	injst := inject{skip: skip}
+	injMap.Store(gid, injst)
 
 	return true
 }
 
 func injectedFailure() bool {
-	injlock.Lock()
+	gid := log.GoroutineID()
-	defer injlock.Unlock()
 
-	if !injst.enabled {
+	val, ok := injMap.Load(gid)
+	if !ok {
 		return false
 	}
 
+	injst, ok := val.(inject)
+	if !ok {
+		panic("invalid type")
+	}
+
 	if injst.skip == 0 {
-		// disable the injection point
+		injMap.Delete(gid)
-		injst.enabled = false
 
 		return true
 	}
 
 	injst.skip--
+	injMap.Store(gid, injst)
 
 	return false
 }

pkg/test/inject_test.go

@@ -117,4 +117,9 @@ func TestInject(t *testing.T) {
 		ok := alwaysNotOk()
 		So(test.Ok(ok), ShouldBeFalse)
 	})
+
+	Convey("Incomplete injected failure", t, func(c C) {
+		test.InjectFailure(0) // inject a failure
+		So(func() { test.InjectFailure(0) }, ShouldPanic)
+	})
 }