feat(cache): dynamodb implementation (#953)
Signed-off-by: Catalin Hofnar <catalin.hofnar@gmail.com>
This commit is contained in:
Catalin-George Hofnar
GitHub
parent
49c3d05706
commit
31b9481713
5
.github/workflows/ci-cd.yml
vendored
5
.github/workflows/ci-cd.yml
vendored
@ -38,7 +38,7 @@ jobs:
|
||||
s3mock:
|
||||
image: ghcr.io/project-zot/localstack/localstack:1.2.0
|
||||
env:
|
||||
SERVICES: s3
|
||||
SERVICES: s3,dynamodb
|
||||
ports:
|
||||
- 4563-4599:4563-4599
|
||||
- 9090:8080
|
||||
@ -79,6 +79,7 @@ jobs:
|
||||
- name: Run build and test
|
||||
timeout-minutes: 60
|
||||
run: |
|
||||
aws dynamodb --endpoint-url http://localhost:4566 --region "us-east-2" create-table --table-name BlobTable --attribute-definitions AttributeName=Digest,AttributeType=S --key-schema AttributeName=Digest,KeyType=HASH --provisioned-throughput ReadCapacityUnits=10,WriteCapacityUnits=5
|
||||
echo "Building for $OS:$ARCH"
|
||||
cd $GITHUB_WORKSPACE
|
||||
if [[ $OS == "linux" && $ARCH == "amd64" ]]; then
|
||||
@ -87,8 +88,10 @@ jobs:
|
||||
else
|
||||
make OS=$OS ARCH=$ARCH binary binary-minimal binary-debug cli bench exporter-minimal
|
||||
fi
|
||||
aws dynamodb --endpoint-url "http://localhost:4566" --region "us-east-2" delete-table --table-name "BlobTable"
|
||||
env:
|
||||
S3MOCK_ENDPOINT: localhost:4566
|
||||
DYNAMODBMOCK_ENDPOINT: http://localhost:4566
|
||||
AWS_ACCESS_KEY_ID: fake
|
||||
AWS_SECRET_ACCESS_KEY: fake
|
||||
OS: ${{ matrix.os }}
|
||||
|
||||
17
.github/workflows/ecosystem-tools.yaml
vendored
17
.github/workflows/ecosystem-tools.yaml
vendored
@ -54,3 +54,20 @@ jobs:
|
||||
- name: Run annotations tests
|
||||
run: |
|
||||
make test-annotations
|
||||
- name: Install localstack
|
||||
run: |
|
||||
pip install --upgrade pyopenssl
|
||||
pip install localstack awscli-local[ver1] # install LocalStack cli and awslocal
|
||||
docker pull localstack/localstack # Make sure to pull the latest version of the image
|
||||
localstack start -d # Start LocalStack in the background
|
||||
|
||||
echo "Waiting for LocalStack startup..." # Wait 30 seconds for the LocalStack container
|
||||
localstack wait -t 30 # to become ready before timing out
|
||||
echo "Startup complete"
|
||||
- name: Run cloud-only tests
|
||||
run: |
|
||||
sudo mkdir /zot
|
||||
make test-cloud-only
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: fake
|
||||
AWS_SECRET_ACCESS_KEY: fake
|
||||
|
||||
8
Makefile
8
Makefile
@ -274,6 +274,14 @@ test-push-pull: binary check-skopeo $(BATS) $(REGCLIENT) $(ORAS) $(HELM)
|
||||
test-push-pull-verbose: binary check-skopeo $(BATS)
|
||||
$(BATS) --trace --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/pushpull.bats
|
||||
|
||||
.PHONY: test-cloud-only
|
||||
test-cloud-only: binary check-skopeo $(BATS)
|
||||
$(BATS) --trace --print-output-on-failure test/blackbox/cloud-only.bats
|
||||
|
||||
.PHONY: test-cloud-only-verbose
|
||||
test-cloud-only-verbose: binary check-skopeo $(BATS)
|
||||
$(BATS) --trace --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/cloud-only.bats
|
||||
|
||||
.PHONY: test-bats-sync
|
||||
test-bats-sync: EXTENSIONS=sync
|
||||
test-bats-sync: binary binary-minimal check-skopeo $(BATS) $(NOTATION) $(COSIGN)
|
||||
|
||||
30
examples/config-dynamodb.json
Normal file
30
examples/config-dynamodb.json
Normal file
@ -0,0 +1,30 @@
|
||||
{
|
||||
"distSpecVersion": "1.0.1-dev",
|
||||
"storage": {
|
||||
"rootDirectory": "/tmp/zot",
|
||||
"dedupe": true,
|
||||
"remoteCache": true,
|
||||
"storageDriver": {
|
||||
"name": "s3",
|
||||
"rootdirectory": "/zot",
|
||||
"region": "us-east-2",
|
||||
"regionendpoint": "localhost:4566",
|
||||
"bucket": "zot-storage",
|
||||
"secure": false,
|
||||
"skipverify": false
|
||||
},
|
||||
"cacheDriver": {
|
||||
"name": "dynamodb",
|
||||
"endpoint": "http://localhost:4566",
|
||||
"region": "us-east-2",
|
||||
"tableName": "BlobTable"
|
||||
}
|
||||
},
|
||||
"http": {
|
||||
"address": "127.0.0.1",
|
||||
"port": "8080"
|
||||
},
|
||||
"log": {
|
||||
"level": "debug"
|
||||
}
|
||||
}
|
||||
@ -11,6 +11,12 @@
|
||||
"secure": true,
|
||||
"skipverify": false
|
||||
},
|
||||
"cacheDriver": {
|
||||
"name": "dynamodb",
|
||||
"endpoint": "http://localhost:4566",
|
||||
"region": "us-east-2",
|
||||
"tableName": "MainTable"
|
||||
},
|
||||
"subPaths": {
|
||||
"/a": {
|
||||
"rootDirectory": "/tmp/zot1",
|
||||
@ -27,6 +33,7 @@
|
||||
"/b": {
|
||||
"rootDirectory": "/tmp/zot2",
|
||||
"dedupe": true,
|
||||
"remoteCache": false,
|
||||
"storageDriver": {
|
||||
"name": "s3",
|
||||
"rootdirectory": "/zot-b",
|
||||
@ -39,6 +46,7 @@
|
||||
"/c": {
|
||||
"rootDirectory": "/tmp/zot3",
|
||||
"dedupe": true,
|
||||
"remoteCache": true,
|
||||
"storageDriver": {
|
||||
"name": "s3",
|
||||
"rootdirectory": "/zot-c",
|
||||
@ -46,6 +54,12 @@
|
||||
"bucket": "zot-storage",
|
||||
"secure": false,
|
||||
"skipverify": false
|
||||
},
|
||||
"cacheDriver": {
|
||||
"name": "dynamodb",
|
||||
"endpoint": "http://localhost:4566",
|
||||
"region": "us-east-2",
|
||||
"tableName": "cTable"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
18
go.mod
18
go.mod
@ -52,6 +52,7 @@ require (
|
||||
|
||||
require (
|
||||
github.com/aquasecurity/trivy v0.0.0-00010101000000-000000000000
|
||||
github.com/aws/aws-sdk-go-v2/service/dynamodb v1.17.3
|
||||
github.com/containers/image/v5 v5.23.0
|
||||
github.com/notaryproject/notation-go v0.12.0-beta.1
|
||||
github.com/opencontainers/distribution-spec/specs-go v0.0.0-20220620172159-4ab4752c3b86
|
||||
@ -59,6 +60,12 @@ require (
|
||||
github.com/swaggo/http-swagger v1.3.3
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.13.22 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.10 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.7.19 // indirect
|
||||
)
|
||||
|
||||
require (
|
||||
bitbucket.org/creachadair/shell v0.0.7 // indirect
|
||||
cloud.google.com/go v0.104.0 // indirect
|
||||
@ -115,12 +122,13 @@ require (
|
||||
github.com/aquasecurity/tfsec v0.58.11 // indirect
|
||||
github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect
|
||||
github.com/aws/aws-sdk-go v1.44.114 // indirect
|
||||
github.com/aws/aws-sdk-go-v2 v1.16.16 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/config v1.17.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2 v1.17.1
|
||||
github.com/aws/aws-sdk-go-v2/config v1.17.8
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.12.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.10.2
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.17 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.23 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.17 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.25 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.19 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.24 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ecr v1.15.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ecrpublic v1.12.0 // indirect
|
||||
@ -128,7 +136,7 @@ require (
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.11.23 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.6 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.16.19 // indirect
|
||||
github.com/aws/smithy-go v1.13.3 // indirect
|
||||
github.com/aws/smithy-go v1.13.4 // indirect
|
||||
github.com/awslabs/amazon-ecr-credential-helper/ecr-login v0.0.0-20220228164355-396b2034c795 // indirect
|
||||
github.com/benbjohnson/clock v1.1.0 // indirect
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
|
||||
22
go.sum
22
go.sum
@ -428,32 +428,45 @@ github.com/aws/aws-sdk-go v1.44.114/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX
|
||||
github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g=
|
||||
github.com/aws/aws-sdk-go-v2 v1.7.1/go.mod h1:L5LuPC1ZgDr2xQS7AmIec/Jlc7O/Y1u2KxJyNVab250=
|
||||
github.com/aws/aws-sdk-go-v2 v1.14.0/go.mod h1:ZA3Y8V0LrlWj63MQAnRHgKf/5QB//LSZCPNWlWrNGLU=
|
||||
github.com/aws/aws-sdk-go-v2 v1.16.16 h1:M1fj4FE2lB4NzRb9Y0xdWsn2P0+2UHVxwKyOa4YJNjk=
|
||||
github.com/aws/aws-sdk-go-v2 v1.16.16/go.mod h1:SwiyXi/1zTUZ6KIAmLK5V5ll8SiURNUYOqTerZPaF9k=
|
||||
github.com/aws/aws-sdk-go-v2 v1.17.1 h1:02c72fDJr87N8RAC2s3Qu0YuvMRZKNZJ9F+lAehCazk=
|
||||
github.com/aws/aws-sdk-go-v2 v1.17.1/go.mod h1:JLnGeGONAyi2lWXI1p0PCIOIy333JMVK1U7Hf0aRFLw=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.5.0/go.mod h1:RWlPOAW3E3tbtNAqTwvSW54Of/yP3oiZXMI0xfUdjyA=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.17.8 h1:b9LGqNnOdg9vR4Q43tBTVWk4J6F+W774MSchvKJsqnE=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.17.8/go.mod h1:UkCI3kb0sCdvtjiXYiU4Zx5h07BOpgBTtkPu/49r+kA=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.3.1/go.mod h1:r0n73xwsIVagq8RsxmZbGSRQFj9As3je72C2WzUIToc=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.12.21 h1:4tjlyCD0hRGNQivh5dN8hbP30qQhMLBE/FgQR1vHHWM=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.12.21/go.mod h1:O+4XyAt4e+oBAoIwNUYkRg3CVMscaIJdmZBOcPgJ8D8=
|
||||
github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.10.2 h1:UBAIkLzejHf9CDlzKKe28k7xYTYleA2LIC5DUbNx+50=
|
||||
github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.10.2/go.mod h1:4CTiMSedeR1/yn5WoD1q9tQAN6aZadfY5rsXad/LiVQ=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.3.0/go.mod h1:2LAuqPx1I6jNfaGDucWfA2zqQCYCOMCDHiCOciALyNw=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.17 h1:r08j4sbZu/RVi+BNxkBJwPMUYY3P8mgSDuKkZ/ZN1lE=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.17/go.mod h1:yIkQcCDYNsZfXpd5UX2Cy+sWA1jPgIhGTw9cOBzfVnQ=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.5/go.mod h1:2hXc8ooJqF2nAznsbJQIn+7h851/bu8GVC80OVTTqf8=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.23 h1:s4g/wnzMf+qepSNgTvaQQHNxyMLKSawNhKCPNy++2xY=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.23/go.mod h1:2DFxAQ9pfIRy0imBCJv+vZ2X6RKxves6fbnEuSry6b4=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.25 h1:nBO/RFxeq/IS5G9Of+ZrgucRciie2qpLy++3UGZ+q2E=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.25/go.mod h1:Zb29PYkf42vVYQY6pvSyJCJcFHlPIiY+YKdPtwnvMkY=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.3.0/go.mod h1:miRSv9l093jX/t/j+mBCaLqFHo9xKYzJ7DGm1BsGoJM=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.17 h1:/K482T5A3623WJgWT8w1yRAFK4RzGzEl7y39yhtn9eA=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.17/go.mod h1:pRwaTYCJemADaqCbUAxltMoHKata7hmB5PjEXeu0kfg=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.19 h1:oRHDrwCTVT8ZXi4sr9Ld+EXk7N/KGssOr2ygNeojEhw=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.19/go.mod h1:6Q0546uHDp421okhmmGfbxzq2hBqbXFNpi4k+Q1JnQA=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.1.1/go.mod h1:Zy8smImhTdOETZqfyn01iNOe0CNggVbPjCajyaz6Gvg=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.24 h1:wj5Rwc05hvUSvKuOF29IYb9QrCLjU+rHAy/x/o0DK2c=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.24/go.mod h1:jULHjqqjDlbyTa7pfM7WICATnOv+iOhjletM3N0Xbu8=
|
||||
github.com/aws/aws-sdk-go-v2/service/dynamodb v1.17.3 h1:2oB4ikNEMLaPtu6lbNFJyTSayBILvrOfa2VfOffcuvU=
|
||||
github.com/aws/aws-sdk-go-v2/service/dynamodb v1.17.3/go.mod h1:BiglbKCG56L8tmMnUEyEQo422BO9xnNR8vVHnOsByf8=
|
||||
github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.13.22 h1:vSUuWw6gsDfLEqZr1qHKV2uKW3rc6tND2DoGUk34iHs=
|
||||
github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.13.22/go.mod h1:5lIdkQbMmEblCTEAyFAsLduBtMPD9Bqt9fwPjBK1KWU=
|
||||
github.com/aws/aws-sdk-go-v2/service/ecr v1.4.1/go.mod h1:FglZcyeiBqcbvyinl+n14aT/EWC7S1MIH+Gan2iizt0=
|
||||
github.com/aws/aws-sdk-go-v2/service/ecr v1.15.0 h1:lY2Z2sBP+zSbJ6CvvmnFgPcgknoQ0OJV88AwVetRRFk=
|
||||
github.com/aws/aws-sdk-go-v2/service/ecr v1.15.0/go.mod h1:4zYI85WiYDhFaU1jPFVfkD7HlBcdnITDE3QxDwy4Kus=
|
||||
github.com/aws/aws-sdk-go-v2/service/ecrpublic v1.4.1/go.mod h1:eD5Eo4drVP2FLTw0G+SMIPWNWvQRGGTtIZR2XeAagoA=
|
||||
github.com/aws/aws-sdk-go-v2/service/ecrpublic v1.12.0 h1:LsqBpyRofMG6eDs6YGud6FhdGyIyXelAasPOZ6wWLro=
|
||||
github.com/aws/aws-sdk-go-v2/service/ecrpublic v1.12.0/go.mod h1:IArQ3IBR00FkuraKwudKZZU32OxJfdTdwV+W5iZh3Y4=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.10 h1:dpiPHgmFstgkLG07KaYAewvuptq5kvo52xn7tVSrtrQ=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.10/go.mod h1:9cBNUHI2aW4ho0A5T87O294iPDuuUOSIEDjnd1Lq/z0=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.7.19 h1:V03dAtcAN4Qtly7H3/0B6m3t/cyl4FgyKFqK738fyJw=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.7.19/go.mod h1:2WpVWFC5n4DYhjNXzObtge8xfgId9UP6GWca46KJFLo=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.2.1/go.mod h1:zceowr5Z1Nh2WVP8bf/3ikB41IZW59E4yIYbg+pC6mw=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.17 h1:Jrd/oMh0PKQc6+BowB+pLEwLIgaQF29eYbe7E1Av9Ug=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.17/go.mod h1:4nYOrY41Lrbk2170/BGkcJKBhws9Pfn8MG3aGqjjeFI=
|
||||
@ -468,8 +481,9 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.16.19 h1:9pPi0PsFNAGILFfPCk8Y0iyEBGc
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.16.19/go.mod h1:h4J3oPZQbxLhzGnk+j9dfYHi5qIOVJ5kczZd658/ydM=
|
||||
github.com/aws/smithy-go v1.6.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E=
|
||||
github.com/aws/smithy-go v1.11.0/go.mod h1:3xHYmszWVx2c0kIwQeEVf9uSm4fYZt67FBJnwub1bgM=
|
||||
github.com/aws/smithy-go v1.13.3 h1:l7LYxGuzK6/K+NzJ2mC+VvLUbae0sL3bXU//04MkmnA=
|
||||
github.com/aws/smithy-go v1.13.3/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
|
||||
github.com/aws/smithy-go v1.13.4 h1:/RN2z1txIJWeXeOkzX+Hk/4Uuvv7dWtCjbmVJcrskyk=
|
||||
github.com/aws/smithy-go v1.13.4/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
|
||||
github.com/awslabs/amazon-ecr-credential-helper/ecr-login v0.0.0-20220228164355-396b2034c795 h1:IWeCJzU+IYaO2rVEBlGPTBfe90cmGXFTLdhUFlzDGsY=
|
||||
github.com/awslabs/amazon-ecr-credential-helper/ecr-login v0.0.0-20220228164355-396b2034c795/go.mod h1:8vJsEZ4iRqG+Vx6pKhWK6U00qcj0KC37IsfszMkY6UE=
|
||||
github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I=
|
||||
|
||||
@ -29,6 +29,7 @@ type StorageConfig struct {
|
||||
GCDelay time.Duration
|
||||
GCInterval time.Duration
|
||||
StorageDriver map[string]interface{} `mapstructure:",omitempty"`
|
||||
CacheDriver map[string]interface{} `mapstructure:",omitempty"`
|
||||
}
|
||||
|
||||
type TLSConfig struct {
|
||||
|
||||
@ -268,7 +268,7 @@ func (c *Controller) InitImageStore(reloadCtx context.Context) error {
|
||||
var defaultStore storage.ImageStore
|
||||
if c.Config.Storage.StorageDriver == nil {
|
||||
// false positive lint - linter does not implement Lint method
|
||||
//nolint:typecheck
|
||||
//nolint:typecheck,contextcheck
|
||||
defaultStore = local.NewImageStore(c.Config.Storage.RootDirectory,
|
||||
c.Config.Storage.GC, c.Config.Storage.GCDelay,
|
||||
c.Config.Storage.Dedupe, c.Config.Storage.Commit, c.Log, c.Metrics, linter,
|
||||
@ -296,7 +296,7 @@ func (c *Controller) InitImageStore(reloadCtx context.Context) error {
|
||||
}
|
||||
|
||||
// false positive lint - linter does not implement Lint method
|
||||
//nolint: typecheck
|
||||
//nolint: typecheck,contextcheck
|
||||
defaultStore = s3.NewImageStore(rootDir, c.Config.Storage.RootDirectory,
|
||||
c.Config.Storage.GC, c.Config.Storage.GCDelay, c.Config.Storage.Dedupe,
|
||||
c.Config.Storage.Commit, c.Log, c.Metrics, linter, store,
|
||||
@ -315,6 +315,7 @@ func (c *Controller) InitImageStore(reloadCtx context.Context) error {
|
||||
if len(c.Config.Storage.SubPaths) > 0 {
|
||||
subPaths := c.Config.Storage.SubPaths
|
||||
|
||||
//nolint: contextcheck
|
||||
subImageStore, err := c.getSubStore(subPaths, linter)
|
||||
if err != nil {
|
||||
c.Log.Error().Err(err).Msg("controller: error getting sub image store")
|
||||
@ -443,7 +444,18 @@ func CreateCacheDatabaseDriver(storageConfig config.StorageConfig, log log.Logge
|
||||
|
||||
return driver
|
||||
}
|
||||
// used for tests, dynamodb when it comes
|
||||
// dynamodb
|
||||
if storageConfig.CacheDriver != nil {
|
||||
dynamoParams := cache.DynamoDBDriverParameters{}
|
||||
dynamoParams.Endpoint, _ = storageConfig.CacheDriver["endpoint"].(string)
|
||||
dynamoParams.Region, _ = storageConfig.CacheDriver["region"].(string)
|
||||
dynamoParams.TableName, _ = storageConfig.CacheDriver["tablename"].(string)
|
||||
|
||||
driver, _ := storage.Create("dynamodb", dynamoParams, log)
|
||||
|
||||
return driver
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@ -99,6 +99,14 @@ func skipIt(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func skipDynamo(t *testing.T) {
|
||||
t.Helper()
|
||||
|
||||
if os.Getenv("DYNAMODBMOCK_ENDPOINT") == "" {
|
||||
t.Skip("Skipping testing without AWS DynamoDB mock server")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNew(t *testing.T) {
|
||||
Convey("Make a new controller", t, func() {
|
||||
conf := config.New()
|
||||
@ -108,7 +116,7 @@ func TestNew(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestCreateCacheDatabaseDriver(t *testing.T) {
|
||||
Convey("Test CreateCacheDatabaseDriver", t, func() {
|
||||
Convey("Test CreateCacheDatabaseDriver boltdb", t, func() {
|
||||
log := log.NewLogger("debug", "")
|
||||
|
||||
// fail create db, no perm
|
||||
@ -132,6 +140,35 @@ func TestCreateCacheDatabaseDriver(t *testing.T) {
|
||||
driver = api.CreateCacheDatabaseDriver(conf.Storage.StorageConfig, log)
|
||||
So(driver, ShouldBeNil)
|
||||
})
|
||||
skipDynamo(t)
|
||||
skipIt(t)
|
||||
Convey("Test CreateCacheDatabaseDriver dynamodb", t, func() {
|
||||
log := log.NewLogger("debug", "")
|
||||
dir := t.TempDir()
|
||||
// good config
|
||||
conf := config.New()
|
||||
conf.Storage.RootDirectory = dir
|
||||
conf.Storage.Dedupe = true
|
||||
conf.Storage.RemoteCache = true
|
||||
conf.Storage.StorageDriver = map[string]interface{}{
|
||||
"name": "s3",
|
||||
"rootdirectory": "/zot",
|
||||
"region": "us-east-2",
|
||||
"bucket": "zot-storage",
|
||||
"secure": true,
|
||||
"skipverify": false,
|
||||
}
|
||||
|
||||
conf.Storage.CacheDriver = map[string]interface{}{
|
||||
"name": "dynamodb",
|
||||
"endpoint": "http://localhost:4566",
|
||||
"region": "us-east-2",
|
||||
"tableName": "BlobTable",
|
||||
}
|
||||
|
||||
driver := api.CreateCacheDatabaseDriver(conf.Storage.StorageConfig, log)
|
||||
So(driver, ShouldNotBeNil)
|
||||
})
|
||||
}
|
||||
|
||||
func TestRunAlreadyRunningServer(t *testing.T) {
|
||||
|
||||
@ -237,6 +237,75 @@ func validateStorageConfig(cfg *config.Config) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateCacheConfig(cfg *config.Config) error {
|
||||
// global
|
||||
// dedupe true, remote storage, remoteCache true, but no cacheDriver (remote)
|
||||
//nolint: lll
|
||||
if cfg.Storage.Dedupe && cfg.Storage.StorageDriver != nil && cfg.Storage.RemoteCache && cfg.Storage.CacheDriver == nil {
|
||||
log.Error().Err(errors.ErrBadConfig).Msg(
|
||||
"dedupe set to true with remote storage and caching, but no remote cache configured!")
|
||||
|
||||
return errors.ErrBadConfig
|
||||
}
|
||||
|
||||
if cfg.Storage.CacheDriver != nil && cfg.Storage.RemoteCache {
|
||||
// local storage with remote caching
|
||||
if cfg.Storage.StorageDriver == nil {
|
||||
log.Error().Err(errors.ErrBadConfig).Msg("cannot have local storage driver with remote caching!")
|
||||
|
||||
return errors.ErrBadConfig
|
||||
}
|
||||
|
||||
// unsupported cache driver
|
||||
if cfg.Storage.CacheDriver["name"] != storageConstants.DynamoDBDriverName {
|
||||
log.Error().Err(errors.ErrBadConfig).Msgf("unsupported cache driver: %s", cfg.Storage.CacheDriver["name"])
|
||||
|
||||
return errors.ErrBadConfig
|
||||
}
|
||||
}
|
||||
|
||||
if !cfg.Storage.RemoteCache && cfg.Storage.CacheDriver != nil {
|
||||
log.Warn().Err(errors.ErrBadConfig).Msgf(
|
||||
"remoteCache set to false but cacheDriver config (remote caching) provided for %s,"+
|
||||
"will ignore and use local caching", cfg.Storage.RootDirectory)
|
||||
}
|
||||
|
||||
// subpaths
|
||||
for _, subPath := range cfg.Storage.SubPaths {
|
||||
// dedupe true, remote storage, remoteCache true, but no cacheDriver (remote)
|
||||
//nolint: lll
|
||||
if subPath.Dedupe && subPath.StorageDriver != nil && subPath.RemoteCache && subPath.CacheDriver == nil {
|
||||
log.Error().Err(errors.ErrBadConfig).Msg("dedupe set to true with remote storage and caching, but no remote cache configured!")
|
||||
|
||||
return errors.ErrBadConfig
|
||||
}
|
||||
|
||||
if subPath.CacheDriver != nil && subPath.RemoteCache {
|
||||
// local storage with remote caching
|
||||
if subPath.StorageDriver == nil {
|
||||
log.Error().Err(errors.ErrBadConfig).Msg("cannot have local storage driver with remote caching!")
|
||||
|
||||
return errors.ErrBadConfig
|
||||
}
|
||||
|
||||
// unsupported cache driver
|
||||
if subPath.CacheDriver["name"] != storageConstants.DynamoDBDriverName {
|
||||
log.Error().Err(errors.ErrBadConfig).Msgf("unsupported cache driver: %s", subPath.CacheDriver["name"])
|
||||
|
||||
return errors.ErrBadConfig
|
||||
}
|
||||
}
|
||||
|
||||
if !subPath.RemoteCache && subPath.CacheDriver != nil {
|
||||
log.Warn().Err(errors.ErrBadConfig).Msgf(
|
||||
"remoteCache set to false but cacheDriver config (remote caching) provided for %s,"+
|
||||
"will ignore and use local caching", subPath.RootDirectory)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateConfiguration(config *config.Config) error {
|
||||
if err := validateHTTP(config); err != nil {
|
||||
return err
|
||||
@ -258,6 +327,10 @@ func validateConfiguration(config *config.Config) error {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := validateCacheConfig(config); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// check authorization config, it should have basic auth enabled or ldap
|
||||
if config.HTTP.RawAccessControl != nil {
|
||||
// checking for anonymous policy only authorization config: no users, no policies but anonymous policy
|
||||
|
||||
@ -117,52 +117,301 @@ func TestVerify(t *testing.T) {
|
||||
So(err, ShouldBeNil)
|
||||
defer os.Remove(tmpfile.Name()) // clean up
|
||||
|
||||
// dedup true, can't parse database type
|
||||
content := []byte(`{"storage":{"rootDirectory":"/tmp/zot", "dedupe": true,
|
||||
"cache": {"type": 123}},
|
||||
"http":{"address":"127.0.0.1","port":"8080","realm":"zot",
|
||||
"auth":{"htpasswd":{"path":"test/data/htpasswd"},"failDelay":1}}}`)
|
||||
// dedupe true, remote storage, remoteCache true, but no cacheDriver (remote)
|
||||
content := []byte(`{
|
||||
"storage":{
|
||||
"rootDirectory":"/tmp/zot",
|
||||
"dedupe":true,
|
||||
"remoteCache":true,
|
||||
"storageDriver":{
|
||||
"name":"s3",
|
||||
"rootdirectory":"/zot",
|
||||
"region":"us-east-2",
|
||||
"bucket":"zot-storage",
|
||||
"secure":true,
|
||||
"skipverify":false
|
||||
}
|
||||
},
|
||||
"http":{
|
||||
"address":"127.0.0.1",
|
||||
"port":"8080",
|
||||
"realm":"zot",
|
||||
"auth":{
|
||||
"htpasswd":{
|
||||
"path":"test/data/htpasswd"
|
||||
},
|
||||
"failDelay":1
|
||||
}
|
||||
}
|
||||
}`)
|
||||
err = os.WriteFile(tmpfile.Name(), content, 0o0600)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
os.Args = []string{"cli_test", "verify", tmpfile.Name()}
|
||||
So(func() { _ = cli.NewServerRootCmd().Execute() }, ShouldPanic)
|
||||
|
||||
// dedup true, wrong database type
|
||||
content = []byte(`{"storage":{"rootDirectory":"/tmp/zot", "dedupe": true,
|
||||
"cache": {"type": "wrong"}},
|
||||
"http":{"address":"127.0.0.1","port":"8080","realm":"zot",
|
||||
"auth":{"htpasswd":{"path":"test/data/htpasswd"},"failDelay":1}}}`)
|
||||
// local storage with remote caching
|
||||
content = []byte(`{
|
||||
"storage":{
|
||||
"rootDirectory":"/tmp/zot",
|
||||
"dedupe":true,
|
||||
"remoteCache":true,
|
||||
"cacheDriver":{
|
||||
"name":"dynamodb",
|
||||
"endpoint":"http://localhost:4566",
|
||||
"region":"us-east-2",
|
||||
"tableName":"BlobTable"
|
||||
}
|
||||
},
|
||||
"http":{
|
||||
"address":"127.0.0.1",
|
||||
"port":"8080",
|
||||
"realm":"zot",
|
||||
"auth":{
|
||||
"htpasswd":{
|
||||
"path":"test/data/htpasswd"
|
||||
},
|
||||
"failDelay":1
|
||||
}
|
||||
}
|
||||
}`)
|
||||
err = os.WriteFile(tmpfile.Name(), content, 0o0600)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
os.Args = []string{"cli_test", "verify", tmpfile.Name()}
|
||||
So(func() { _ = cli.NewServerRootCmd().Execute() }, ShouldPanic)
|
||||
|
||||
// unsupported cache driver
|
||||
content = []byte(`{
|
||||
"storage":{
|
||||
"rootDirectory":"/tmp/zot",
|
||||
"dedupe":true,
|
||||
"remoteCache":true,
|
||||
"cacheDriver":{
|
||||
"name":"unsupportedDriver"
|
||||
},
|
||||
"storageDriver":{
|
||||
"name":"s3",
|
||||
"rootdirectory":"/zot",
|
||||
"region":"us-east-2",
|
||||
"bucket":"zot-storage",
|
||||
"secure":true,
|
||||
"skipverify":false
|
||||
}
|
||||
},
|
||||
"http":{
|
||||
"address":"127.0.0.1",
|
||||
"port":"8080",
|
||||
"realm":"zot",
|
||||
"auth":{
|
||||
"htpasswd":{
|
||||
"path":"test/data/htpasswd"
|
||||
},
|
||||
"failDelay":1
|
||||
}
|
||||
}
|
||||
}`)
|
||||
err = os.WriteFile(tmpfile.Name(), content, 0o0600)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
os.Args = []string{"cli_test", "verify", tmpfile.Name()}
|
||||
So(func() { _ = cli.NewServerRootCmd().Execute() }, ShouldPanic)
|
||||
|
||||
// remoteCache false but provided cacheDriver config, ignored
|
||||
content = []byte(`{
|
||||
"storage":{
|
||||
"rootDirectory":"/tmp/zot",
|
||||
"dedupe":true,
|
||||
"remoteCache":false,
|
||||
"cacheDriver":{
|
||||
"name":"dynamodb",
|
||||
"endpoint":"http://localhost:4566",
|
||||
"region":"us-east-2",
|
||||
"tableName":"BlobTable"
|
||||
},
|
||||
"storageDriver":{
|
||||
"name":"s3",
|
||||
"rootdirectory":"/zot",
|
||||
"region":"us-east-2",
|
||||
"bucket":"zot-storage",
|
||||
"secure":true,
|
||||
"skipverify":false
|
||||
}
|
||||
},
|
||||
"http":{
|
||||
"address":"127.0.0.1",
|
||||
"port":"8080",
|
||||
"realm":"zot",
|
||||
"auth":{
|
||||
"htpasswd":{
|
||||
"path":"test/data/htpasswd"
|
||||
},
|
||||
"failDelay":1
|
||||
}
|
||||
}
|
||||
}`)
|
||||
|
||||
err = os.WriteFile(tmpfile.Name(), content, 0o0600)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
os.Args = []string{"cli_test", "verify", tmpfile.Name()}
|
||||
So(func() { _ = cli.NewServerRootCmd().Execute() }, ShouldNotPanic)
|
||||
|
||||
// SubPaths
|
||||
// dedup true, wrong database type
|
||||
content = []byte(`{"storage":{"rootDirectory":"/tmp/zot", "dedupe": false,
|
||||
"subPaths": {"/a": {"rootDirectory": "/zot-a", "dedupe": true,
|
||||
"cache": {"type": "wrong"}}}},
|
||||
"http":{"address":"127.0.0.1","port":"8080","realm":"zot",
|
||||
"auth":{"htpasswd":{"path":"test/data/htpasswd"},"failDelay":1}}}`)
|
||||
// dedupe true, remote storage, remoteCache true, but no cacheDriver (remote)
|
||||
content = []byte(`{
|
||||
"storage":{
|
||||
"rootDirectory":"/tmp/zot",
|
||||
"dedupe":false,
|
||||
"subPaths":{
|
||||
"/a":{
|
||||
"rootDirectory":"/zot-a",
|
||||
"dedupe":true,
|
||||
"remoteCache":true,
|
||||
"storageDriver":{
|
||||
"name":"s3",
|
||||
"rootdirectory":"/zot",
|
||||
"region":"us-east-2",
|
||||
"bucket":"zot-storage",
|
||||
"secure":true,
|
||||
"skipverify":false
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"http":{
|
||||
"address":"127.0.0.1",
|
||||
"port":"8080",
|
||||
"realm":"zot",
|
||||
"auth":{
|
||||
"htpasswd":{
|
||||
"path":"test/data/htpasswd"
|
||||
},
|
||||
"failDelay":1
|
||||
}
|
||||
}
|
||||
}`)
|
||||
err = os.WriteFile(tmpfile.Name(), content, 0o0600)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
os.Args = []string{"cli_test", "verify", tmpfile.Name()}
|
||||
So(func() { _ = cli.NewServerRootCmd().Execute() }, ShouldPanic)
|
||||
|
||||
// dedup true, can't parse database type
|
||||
content = []byte(`{"storage":{"rootDirectory":"/tmp/zot", "dedupe": false,
|
||||
"subPaths": {"/a": {"rootDirectory": "/zot-a", "dedupe": true,
|
||||
"cache": {"type": 123}}}},
|
||||
"http":{"address":"127.0.0.1","port":"8080","realm":"zot",
|
||||
"auth":{"htpasswd":{"path":"test/data/htpasswd"},"failDelay":1}}}`)
|
||||
// local storage with remote caching
|
||||
content = []byte(`{
|
||||
"storage":{
|
||||
"rootDirectory":"/tmp/zot",
|
||||
"dedupe":false,
|
||||
"subPaths":{
|
||||
"/a":{
|
||||
"rootDirectory":"/zot-a",
|
||||
"dedupe":true,
|
||||
"remoteCache":true,
|
||||
"cacheDriver":{
|
||||
"name":"dynamodb",
|
||||
"endpoint":"http://localhost:4566",
|
||||
"region":"us-east-2",
|
||||
"tableName":"BlobTable"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"http":{
|
||||
"address":"127.0.0.1",
|
||||
"port":"8080",
|
||||
"realm":"zot",
|
||||
"auth":{
|
||||
"htpasswd":{
|
||||
"path":"test/data/htpasswd"
|
||||
},
|
||||
"failDelay":1
|
||||
}
|
||||
}
|
||||
}`)
|
||||
err = os.WriteFile(tmpfile.Name(), content, 0o0600)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
os.Args = []string{"cli_test", "verify", tmpfile.Name()}
|
||||
So(func() { _ = cli.NewServerRootCmd().Execute() }, ShouldPanic)
|
||||
|
||||
// unsupported cache driver
|
||||
content = []byte(`{
|
||||
"storage":{
|
||||
"rootDirectory":"/tmp/zot",
|
||||
"dedupe":false,
|
||||
"subPaths":{
|
||||
"/a":{
|
||||
"rootDirectory":"/zot-a",
|
||||
"dedupe":true,
|
||||
"remoteCache":true,
|
||||
"cacheDriver":{
|
||||
"name":"badDriverName"
|
||||
},
|
||||
"storageDriver":{
|
||||
"name":"s3",
|
||||
"rootdirectory":"/zot",
|
||||
"region":"us-east-2",
|
||||
"bucket":"zot-storage",
|
||||
"secure":true,
|
||||
"skipverify":false
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"http":{
|
||||
"address":"127.0.0.1",
|
||||
"port":"8080",
|
||||
"realm":"zot",
|
||||
"auth":{
|
||||
"htpasswd":{
|
||||
"path":"test/data/htpasswd"
|
||||
},
|
||||
"failDelay":1
|
||||
}
|
||||
}
|
||||
}`)
|
||||
err = os.WriteFile(tmpfile.Name(), content, 0o0600)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
os.Args = []string{"cli_test", "verify", tmpfile.Name()}
|
||||
So(func() { _ = cli.NewServerRootCmd().Execute() }, ShouldPanic)
|
||||
|
||||
// remoteCache false but provided cacheDriver config, ignored
|
||||
content = []byte(`{
|
||||
"storage":{
|
||||
"rootDirectory":"/tmp/zot",
|
||||
"dedupe":false,
|
||||
"subPaths":{
|
||||
"/a":{
|
||||
"rootDirectory":"/zot-a",
|
||||
"dedupe":true,
|
||||
"remoteCache":false,
|
||||
"cacheDriver":{
|
||||
"name":"dynamodb",
|
||||
"endpoint":"http://localhost:4566",
|
||||
"region":"us-east-2",
|
||||
"tableName":"BlobTable"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"http":{
|
||||
"address":"127.0.0.1",
|
||||
"port":"8080",
|
||||
"realm":"zot",
|
||||
"auth":{
|
||||
"htpasswd":{
|
||||
"path":"test/data/htpasswd"
|
||||
},
|
||||
"failDelay":1
|
||||
}
|
||||
}
|
||||
}`)
|
||||
err = os.WriteFile(tmpfile.Name(), content, 0o0600)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
os.Args = []string{"cli_test", "verify", tmpfile.Name()}
|
||||
So(func() { _ = cli.NewServerRootCmd().Execute() }, ShouldNotPanic)
|
||||
})
|
||||
|
||||
Convey("Test apply defaults cache db", t, func(c C) {
|
||||
|
||||
@ -185,7 +185,7 @@ func syncOneImage(ctx context.Context, imageChannel chan error, cfg Config, stor
|
||||
upstreamAddr: upstreamAddr,
|
||||
copyOptions: options,
|
||||
}
|
||||
|
||||
//nolint:contextcheck
|
||||
skipped, copyErr := syncRun(regCfg, localRepo, upstreamRepo, reference, syncContextUtils, sig, log)
|
||||
if skipped {
|
||||
continue
|
||||
|
||||
@ -12,6 +12,10 @@ func Create(dbtype string, parameters interface{}, log zlog.Logger) (cache.Cache
|
||||
{
|
||||
return cache.NewBoltDBCache(parameters, log), nil
|
||||
}
|
||||
case "dynamodb":
|
||||
{
|
||||
return cache.NewDynamoDBCache(parameters, log), nil
|
||||
}
|
||||
default:
|
||||
{
|
||||
return nil, errors.ErrBadConfig
|
||||
|
||||
17
pkg/storage/cache/boltdb.go
vendored
17
pkg/storage/cache/boltdb.go
vendored
@ -32,18 +32,14 @@ func NewBoltDBCache(parameters interface{}, log zlog.Logger) Cache {
|
||||
panic("Failed type assertion")
|
||||
}
|
||||
|
||||
return NewCache(properParameters, log)
|
||||
}
|
||||
|
||||
func NewCache(parameters BoltDBDriverParameters, log zlog.Logger) *BoltDBDriver {
|
||||
err := os.MkdirAll(parameters.RootDir, constants.DefaultDirPerms)
|
||||
err := os.MkdirAll(properParameters.RootDir, constants.DefaultDirPerms)
|
||||
if err != nil {
|
||||
log.Error().Err(err).Msgf("unable to create directory for cache db: %v", parameters.RootDir)
|
||||
log.Error().Err(err).Msgf("unable to create directory for cache db: %v", properParameters.RootDir)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
dbPath := path.Join(parameters.RootDir, parameters.Name+constants.DBExtensionName)
|
||||
dbPath := path.Join(properParameters.RootDir, properParameters.Name+constants.DBExtensionName)
|
||||
dbOpts := &bbolt.Options{
|
||||
Timeout: constants.DBCacheLockCheckTimeout,
|
||||
FreelistType: bbolt.FreelistArrayType,
|
||||
@ -72,7 +68,12 @@ func NewCache(parameters BoltDBDriverParameters, log zlog.Logger) *BoltDBDriver
|
||||
return nil
|
||||
}
|
||||
|
||||
return &BoltDBDriver{rootDir: parameters.RootDir, db: cacheDB, useRelPaths: parameters.UseRelPaths, log: log}
|
||||
return &BoltDBDriver{
|
||||
rootDir: properParameters.RootDir,
|
||||
db: cacheDB,
|
||||
useRelPaths: properParameters.UseRelPaths,
|
||||
log: log,
|
||||
}
|
||||
}
|
||||
|
||||
func (d *BoltDBDriver) Name() string {
|
||||
|
||||
216
pkg/storage/cache/dynamodb.go
vendored
Normal file
216
pkg/storage/cache/dynamodb.go
vendored
Normal file
@ -0,0 +1,216 @@
|
||||
package cache
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/config"
|
||||
"github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue"
|
||||
"github.com/aws/aws-sdk-go-v2/service/dynamodb"
|
||||
"github.com/aws/aws-sdk-go-v2/service/dynamodb/types"
|
||||
godigest "github.com/opencontainers/go-digest"
|
||||
|
||||
zerr "zotregistry.io/zot/errors"
|
||||
zlog "zotregistry.io/zot/pkg/log"
|
||||
)
|
||||
|
||||
type DynamoDBDriver struct {
|
||||
client *dynamodb.Client
|
||||
log zlog.Logger
|
||||
tableName string
|
||||
}
|
||||
|
||||
type DynamoDBDriverParameters struct {
|
||||
Endpoint, Region, TableName string
|
||||
}
|
||||
|
||||
type Blob struct {
|
||||
Digest string `dynamodbav:"Digest,string"`
|
||||
BlobPath []string `dynamodbav:"BlobPath,stringset"`
|
||||
}
|
||||
|
||||
// Use ONLY for tests.
|
||||
func (d *DynamoDBDriver) NewTable(tableName string) error {
|
||||
//nolint:gomnd
|
||||
_, err := d.client.CreateTable(context.TODO(), &dynamodb.CreateTableInput{
|
||||
TableName: &tableName,
|
||||
AttributeDefinitions: []types.AttributeDefinition{
|
||||
{
|
||||
AttributeName: aws.String("Digest"),
|
||||
AttributeType: types.ScalarAttributeTypeS,
|
||||
},
|
||||
},
|
||||
KeySchema: []types.KeySchemaElement{
|
||||
{
|
||||
AttributeName: aws.String("Digest"),
|
||||
KeyType: types.KeyTypeHash,
|
||||
},
|
||||
},
|
||||
ProvisionedThroughput: &types.ProvisionedThroughput{
|
||||
ReadCapacityUnits: aws.Int64(10),
|
||||
WriteCapacityUnits: aws.Int64(5),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
d.tableName = tableName
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func NewDynamoDBCache(parameters interface{}, log zlog.Logger) Cache {
|
||||
properParameters, ok := parameters.(DynamoDBDriverParameters)
|
||||
if !ok {
|
||||
panic("Failed type assertion!")
|
||||
}
|
||||
|
||||
// custom endpoint resolver to point to localhost
|
||||
customResolver := aws.EndpointResolverWithOptionsFunc(
|
||||
func(service, region string, options ...interface{}) (aws.Endpoint, error) {
|
||||
return aws.Endpoint{
|
||||
PartitionID: "aws",
|
||||
URL: properParameters.Endpoint,
|
||||
SigningRegion: region,
|
||||
}, nil
|
||||
})
|
||||
|
||||
// Using the SDK's default configuration, loading additional config
|
||||
// and credentials values from the environment variables, shared
|
||||
// credentials, and shared configuration files
|
||||
cfg, err := config.LoadDefaultConfig(context.TODO(), config.WithRegion(properParameters.Region),
|
||||
config.WithEndpointResolverWithOptions(customResolver))
|
||||
if err != nil {
|
||||
log.Error().Msgf("unable to load AWS SDK config for dynamodb, %v", err)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Using the Config value, create the DynamoDB client
|
||||
return &DynamoDBDriver{client: dynamodb.NewFromConfig(cfg), tableName: properParameters.TableName, log: log}
|
||||
}
|
||||
|
||||
func (d *DynamoDBDriver) Name() string {
|
||||
return "dynamodb"
|
||||
}
|
||||
|
||||
// Returns the first path of the blob if it exists.
|
||||
func (d *DynamoDBDriver) GetBlob(digest godigest.Digest) (string, error) {
|
||||
resp, err := d.client.GetItem(context.TODO(), &dynamodb.GetItemInput{
|
||||
TableName: aws.String(d.tableName),
|
||||
Key: map[string]types.AttributeValue{
|
||||
"Digest": &types.AttributeValueMemberS{Value: digest.String()},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
d.log.Error().Msgf("failed to get blob %v, %v", d.tableName, err)
|
||||
|
||||
return "", err
|
||||
}
|
||||
|
||||
out := Blob{}
|
||||
|
||||
if resp.Item == nil {
|
||||
return "", zerr.ErrCacheMiss
|
||||
}
|
||||
|
||||
_ = attributevalue.UnmarshalMap(resp.Item, &out)
|
||||
|
||||
if len(out.BlobPath) == 0 {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
return out.BlobPath[0], nil
|
||||
}
|
||||
|
||||
func (d *DynamoDBDriver) PutBlob(digest godigest.Digest, path string) error {
|
||||
if path == "" {
|
||||
d.log.Error().Err(zerr.ErrEmptyValue).Str("digest", digest.String()).Msg("empty path provided")
|
||||
|
||||
return zerr.ErrEmptyValue
|
||||
}
|
||||
|
||||
marshaledKey, _ := attributevalue.MarshalMap(map[string]interface{}{"Digest": digest.String()})
|
||||
expression := "ADD BlobPath :i"
|
||||
attrPath := types.AttributeValueMemberSS{Value: []string{path}}
|
||||
|
||||
if _, err := d.client.UpdateItem(context.TODO(), &dynamodb.UpdateItemInput{
|
||||
Key: marshaledKey,
|
||||
TableName: &d.tableName,
|
||||
UpdateExpression: &expression,
|
||||
ExpressionAttributeValues: map[string]types.AttributeValue{":i": &attrPath},
|
||||
}); err != nil {
|
||||
d.log.Error().Err(err)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *DynamoDBDriver) HasBlob(digest godigest.Digest, path string) bool {
|
||||
resp, err := d.client.GetItem(context.TODO(), &dynamodb.GetItemInput{
|
||||
TableName: aws.String(d.tableName),
|
||||
Key: map[string]types.AttributeValue{
|
||||
"Digest": &types.AttributeValueMemberS{Value: digest.String()},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
d.log.Error().Msgf("failed to get blob %v, %v", d.tableName, err)
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
out := Blob{}
|
||||
|
||||
if resp.Item == nil {
|
||||
d.log.Error().Err(zerr.ErrCacheMiss)
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
_ = attributevalue.UnmarshalMap(resp.Item, &out)
|
||||
|
||||
for _, item := range out.BlobPath {
|
||||
if item == path {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
d.log.Error().Err(zerr.ErrCacheMiss)
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func (d *DynamoDBDriver) DeleteBlob(digest godigest.Digest, path string) error {
|
||||
marshaledKey, _ := attributevalue.MarshalMap(map[string]interface{}{"Digest": digest.String()})
|
||||
|
||||
expression := "DELETE BlobPath :i"
|
||||
attrPath := types.AttributeValueMemberSS{Value: []string{path}}
|
||||
|
||||
_, err := d.client.UpdateItem(context.TODO(), &dynamodb.UpdateItemInput{
|
||||
Key: marshaledKey,
|
||||
TableName: &d.tableName,
|
||||
UpdateExpression: &expression,
|
||||
ExpressionAttributeValues: map[string]types.AttributeValue{":i": &attrPath},
|
||||
})
|
||||
if err != nil {
|
||||
d.log.Error().Err(err).Str("digest", digest.String()).Str("path", path).Msg("unable to delete")
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
result, _ := d.GetBlob(digest)
|
||||
|
||||
if result == "" {
|
||||
d.log.Debug().Str("digest", digest.String()).Str("path", path).Msg("deleting empty bucket")
|
||||
|
||||
_, _ = d.client.DeleteItem(context.TODO(), &dynamodb.DeleteItemInput{
|
||||
Key: marshaledKey,
|
||||
TableName: &d.tableName,
|
||||
})
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
111
pkg/storage/cache/dynamodb_test.go
vendored
Normal file
111
pkg/storage/cache/dynamodb_test.go
vendored
Normal file
@ -0,0 +1,111 @@
|
||||
package cache_test
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path"
|
||||
"testing"
|
||||
|
||||
godigest "github.com/opencontainers/go-digest"
|
||||
. "github.com/smartystreets/goconvey/convey"
|
||||
|
||||
"zotregistry.io/zot/pkg/log"
|
||||
"zotregistry.io/zot/pkg/storage"
|
||||
"zotregistry.io/zot/pkg/storage/cache"
|
||||
)
|
||||
|
||||
func skipIt(t *testing.T) {
|
||||
t.Helper()
|
||||
|
||||
if os.Getenv("DYNAMODBMOCK_ENDPOINT") == "" {
|
||||
t.Skip("Skipping testing without AWS DynamoDB mock server")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDynamoDB(t *testing.T) {
|
||||
skipIt(t)
|
||||
Convey("Test dynamoDB", t, func(c C) {
|
||||
log := log.NewLogger("debug", "")
|
||||
dir := t.TempDir()
|
||||
|
||||
// bad params
|
||||
|
||||
So(func() {
|
||||
_ = cache.NewDynamoDBCache("bad params", log)
|
||||
}, ShouldPanic)
|
||||
|
||||
keyDigest := godigest.FromString("key")
|
||||
|
||||
cacheDriver, err := storage.Create("dynamodb", cache.DynamoDBDriverParameters{
|
||||
Endpoint: "http://brokenlink",
|
||||
TableName: "BlobTable",
|
||||
Region: "us-east-2",
|
||||
}, log)
|
||||
So(cacheDriver, ShouldNotBeNil)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
val, err := cacheDriver.GetBlob(keyDigest)
|
||||
So(err, ShouldNotBeNil)
|
||||
So(val, ShouldBeEmpty)
|
||||
|
||||
err = cacheDriver.PutBlob(keyDigest, path.Join(dir, "value"))
|
||||
So(err, ShouldNotBeNil)
|
||||
|
||||
exists := cacheDriver.HasBlob(keyDigest, path.Join(dir, "value"))
|
||||
So(exists, ShouldBeFalse)
|
||||
|
||||
err = cacheDriver.DeleteBlob(keyDigest, path.Join(dir, "value"))
|
||||
So(err, ShouldNotBeNil)
|
||||
|
||||
cacheDriver, err = storage.Create("dynamodb", cache.DynamoDBDriverParameters{
|
||||
Endpoint: os.Getenv("DYNAMODBMOCK_ENDPOINT"),
|
||||
TableName: "BlobTable",
|
||||
Region: "us-east-2",
|
||||
}, log)
|
||||
So(cacheDriver, ShouldNotBeNil)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
returnedName := cacheDriver.Name()
|
||||
So(returnedName, ShouldEqual, "dynamodb")
|
||||
|
||||
val, err = cacheDriver.GetBlob(keyDigest)
|
||||
So(err, ShouldNotBeNil)
|
||||
So(val, ShouldBeEmpty)
|
||||
|
||||
err = cacheDriver.PutBlob(keyDigest, "")
|
||||
So(err, ShouldNotBeNil)
|
||||
|
||||
err = cacheDriver.PutBlob(keyDigest, path.Join(dir, "value"))
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
val, err = cacheDriver.GetBlob(keyDigest)
|
||||
So(err, ShouldBeNil)
|
||||
So(val, ShouldNotBeEmpty)
|
||||
|
||||
exists = cacheDriver.HasBlob(keyDigest, path.Join(dir, "value"))
|
||||
So(exists, ShouldBeTrue)
|
||||
|
||||
err = cacheDriver.DeleteBlob(keyDigest, path.Join(dir, "value"))
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
exists = cacheDriver.HasBlob(keyDigest, path.Join(dir, "value"))
|
||||
So(exists, ShouldBeFalse)
|
||||
|
||||
err = cacheDriver.PutBlob(keyDigest, path.Join(dir, "value1"))
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
err = cacheDriver.PutBlob(keyDigest, path.Join(dir, "value2"))
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
err = cacheDriver.DeleteBlob(keyDigest, path.Join(dir, "value1"))
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
exists = cacheDriver.HasBlob(keyDigest, path.Join(dir, "value2"))
|
||||
So(exists, ShouldBeTrue)
|
||||
|
||||
exists = cacheDriver.HasBlob(keyDigest, path.Join(dir, "value1"))
|
||||
So(exists, ShouldBeFalse)
|
||||
|
||||
err = cacheDriver.DeleteBlob(keyDigest, path.Join(dir, "value2"))
|
||||
So(err, ShouldBeNil)
|
||||
})
|
||||
}
|
||||
520
pkg/storage/cache_benchmark_test.go
Normal file
520
pkg/storage/cache_benchmark_test.go
Normal file
@ -0,0 +1,520 @@
|
||||
package storage_test
|
||||
|
||||
import (
|
||||
"math/rand"
|
||||
"os/exec"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
godigest "github.com/opencontainers/go-digest"
|
||||
|
||||
"zotregistry.io/zot/pkg/log"
|
||||
"zotregistry.io/zot/pkg/storage"
|
||||
"zotregistry.io/zot/pkg/storage/cache"
|
||||
)
|
||||
|
||||
const (
|
||||
region string = "us-east-2"
|
||||
localEndpoint string = "http://localhost:4566"
|
||||
awsEndpoint string = "https://dynamodb.us-east-2.amazonaws.com"
|
||||
datasetSize int = 5000
|
||||
)
|
||||
|
||||
func generateRandomString() string {
|
||||
//nolint: gosec
|
||||
seededRand := rand.New(rand.NewSource(time.Now().UnixNano()))
|
||||
charset := "abcdefghijklmnopqrstuvwxyz" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
|
||||
|
||||
randomBytes := make([]byte, 10)
|
||||
for i := range randomBytes {
|
||||
randomBytes[i] = charset[seededRand.Intn(len(charset))]
|
||||
}
|
||||
|
||||
return string(randomBytes)
|
||||
}
|
||||
|
||||
func generateData() map[godigest.Digest]string {
|
||||
dataMap := make(map[godigest.Digest]string, datasetSize)
|
||||
//nolint: gosec
|
||||
seededRand := rand.New(rand.NewSource(time.Now().UnixNano()))
|
||||
|
||||
for i := 0; i < datasetSize; i++ {
|
||||
randomString := generateRandomString()
|
||||
counter := 0
|
||||
|
||||
for seededRand.Float32() < 0.5 && counter < 5 {
|
||||
counter++
|
||||
randomString += "/"
|
||||
randomString += generateRandomString()
|
||||
}
|
||||
digest := godigest.FromString(randomString)
|
||||
dataMap[digest] = randomString
|
||||
}
|
||||
|
||||
return dataMap
|
||||
}
|
||||
|
||||
func helperPutAll(cache cache.Cache, testData map[godigest.Digest]string) {
|
||||
for digest, path := range testData {
|
||||
_ = cache.PutBlob(digest, path)
|
||||
}
|
||||
}
|
||||
|
||||
func helperDeleteAll(cache cache.Cache, testData map[godigest.Digest]string) {
|
||||
for digest, path := range testData {
|
||||
_ = cache.DeleteBlob(digest, path)
|
||||
}
|
||||
}
|
||||
|
||||
func helperHasAll(cache cache.Cache, testData map[godigest.Digest]string) {
|
||||
for digest, path := range testData {
|
||||
_ = cache.HasBlob(digest, path)
|
||||
}
|
||||
}
|
||||
|
||||
func helperGetAll(cache cache.Cache, testData map[godigest.Digest]string) {
|
||||
for digest := range testData {
|
||||
_, _ = cache.GetBlob(digest)
|
||||
}
|
||||
}
|
||||
|
||||
func helperMix(cache cache.Cache, testData map[godigest.Digest]string, digestSlice []godigest.Digest) {
|
||||
// The test data contains datasetSize entries by default, and each set of operations uses 5 entries
|
||||
for i := 0; i < 1000; i++ {
|
||||
_ = cache.PutBlob(digestSlice[i*5], testData[digestSlice[i*5]])
|
||||
_ = cache.PutBlob(digestSlice[i*5+1], testData[digestSlice[i*5+1]])
|
||||
_ = cache.PutBlob(digestSlice[i*5+2], testData[digestSlice[i*5+2]])
|
||||
_ = cache.PutBlob(digestSlice[i*5+2], testData[digestSlice[i*5+3]])
|
||||
_ = cache.DeleteBlob(digestSlice[i*5+1], testData[digestSlice[i*5+1]])
|
||||
_ = cache.DeleteBlob(digestSlice[i*5+2], testData[digestSlice[i*5+3]])
|
||||
_ = cache.DeleteBlob(digestSlice[i*5+2], testData[digestSlice[i*5+2]])
|
||||
_ = cache.HasBlob(digestSlice[i*5], testData[digestSlice[i*5]])
|
||||
_ = cache.HasBlob(digestSlice[i*5+1], testData[digestSlice[i*5+1]])
|
||||
_, _ = cache.GetBlob(digestSlice[i*5])
|
||||
_, _ = cache.GetBlob(digestSlice[i*5+1])
|
||||
_ = cache.PutBlob(digestSlice[i*5], testData[digestSlice[i*5+4]])
|
||||
_, _ = cache.GetBlob(digestSlice[i*5+4])
|
||||
_ = cache.DeleteBlob(digestSlice[i*5], testData[digestSlice[i*5+4]])
|
||||
_ = cache.DeleteBlob(digestSlice[i*5], testData[digestSlice[i*5]])
|
||||
}
|
||||
}
|
||||
|
||||
// BoltDB tests
|
||||
|
||||
func BenchmarkPutLocal(b *testing.B) {
|
||||
dir := b.TempDir()
|
||||
log := log.NewLogger("error", "")
|
||||
cache, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{
|
||||
RootDir: dir,
|
||||
Name: "cache_test",
|
||||
UseRelPaths: false,
|
||||
}, log)
|
||||
testData := generateData()
|
||||
|
||||
b.ResetTimer()
|
||||
|
||||
helperPutAll(cache, testData)
|
||||
}
|
||||
|
||||
func BenchmarkDeleteLocal(b *testing.B) {
|
||||
dir := b.TempDir()
|
||||
log := log.NewLogger("error", "")
|
||||
cache, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{
|
||||
RootDir: dir,
|
||||
Name: "cache_test",
|
||||
UseRelPaths: false,
|
||||
}, log)
|
||||
testData := generateData()
|
||||
|
||||
for digest, path := range testData {
|
||||
_ = cache.PutBlob(digest, path)
|
||||
}
|
||||
|
||||
b.ResetTimer()
|
||||
|
||||
helperDeleteAll(cache, testData)
|
||||
}
|
||||
|
||||
func BenchmarkHasLocal(b *testing.B) {
|
||||
dir := b.TempDir()
|
||||
log := log.NewLogger("error", "")
|
||||
cache, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{
|
||||
RootDir: dir,
|
||||
Name: "cache_test",
|
||||
UseRelPaths: false,
|
||||
}, log)
|
||||
testData := generateData()
|
||||
|
||||
for digest, path := range testData {
|
||||
_ = cache.PutBlob(digest, path)
|
||||
}
|
||||
|
||||
b.ResetTimer()
|
||||
|
||||
helperHasAll(cache, testData)
|
||||
}
|
||||
|
||||
func BenchmarkGetLocal(b *testing.B) {
|
||||
dir := b.TempDir()
|
||||
log := log.NewLogger("error", "")
|
||||
cache, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{
|
||||
RootDir: dir,
|
||||
Name: "cache_test",
|
||||
UseRelPaths: false,
|
||||
}, log)
|
||||
testData := generateData()
|
||||
counter := 1
|
||||
|
||||
var previousDigest godigest.Digest
|
||||
|
||||
for digest, path := range testData {
|
||||
if counter != 10 {
|
||||
_ = cache.PutBlob(digest, path)
|
||||
previousDigest = digest
|
||||
counter++
|
||||
} else {
|
||||
_ = cache.PutBlob(previousDigest, path)
|
||||
counter = 1
|
||||
}
|
||||
}
|
||||
|
||||
b.ResetTimer()
|
||||
|
||||
helperGetAll(cache, testData)
|
||||
}
|
||||
|
||||
func BenchmarkMixLocal(b *testing.B) {
|
||||
dir := b.TempDir()
|
||||
log := log.NewLogger("error", "")
|
||||
cache, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{
|
||||
RootDir: dir,
|
||||
Name: "cache_test",
|
||||
UseRelPaths: false,
|
||||
}, log)
|
||||
testData := generateData()
|
||||
digestSlice := make([]godigest.Digest, datasetSize)
|
||||
counter := 0
|
||||
|
||||
for key := range testData {
|
||||
digestSlice[counter] = key
|
||||
counter++
|
||||
}
|
||||
|
||||
b.ResetTimer()
|
||||
|
||||
helperMix(cache, testData, digestSlice)
|
||||
}
|
||||
|
||||
// DynamoDB Local tests
|
||||
|
||||
func BenchmarkPutLocalstack(b *testing.B) {
|
||||
log := log.NewLogger("error", "")
|
||||
tableName := generateRandomString()
|
||||
|
||||
// Create Table
|
||||
_, err := exec.Command("aws", "dynamodb", "--region", region, "--endpoint-url", localEndpoint, "create-table",
|
||||
"--table-name", tableName, "--attribute-definitions", "AttributeName=Digest,AttributeType=S",
|
||||
"--key-schema", "AttributeName=Digest,KeyType=HASH",
|
||||
"--billing-mode", "PAY_PER_REQUEST").Output()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
cache, _ := storage.Create("dynamodb", cache.DynamoDBDriverParameters{
|
||||
Endpoint: localEndpoint,
|
||||
Region: region,
|
||||
TableName: tableName,
|
||||
}, log)
|
||||
testData := generateData()
|
||||
|
||||
b.ResetTimer()
|
||||
|
||||
helperPutAll(cache, testData)
|
||||
}
|
||||
|
||||
func BenchmarkDeleteLocalstack(b *testing.B) {
|
||||
log := log.NewLogger("error", "")
|
||||
tableName := generateRandomString()
|
||||
|
||||
// Create Table
|
||||
_, err := exec.Command("aws", "dynamodb", "--region", region, "--endpoint-url", localEndpoint, "create-table",
|
||||
"--table-name", tableName, "--attribute-definitions", "AttributeName=Digest,AttributeType=S",
|
||||
"--key-schema", "AttributeName=Digest,KeyType=HASH",
|
||||
"--billing-mode", "PAY_PER_REQUEST").Output()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
cache, _ := storage.Create("dynamodb", cache.DynamoDBDriverParameters{
|
||||
Endpoint: localEndpoint,
|
||||
Region: region,
|
||||
TableName: tableName,
|
||||
}, log)
|
||||
testData := generateData()
|
||||
|
||||
for digest, path := range testData {
|
||||
_ = cache.PutBlob(digest, path)
|
||||
}
|
||||
|
||||
b.ResetTimer()
|
||||
|
||||
helperDeleteAll(cache, testData)
|
||||
}
|
||||
|
||||
func BenchmarkHasLocalstack(b *testing.B) {
|
||||
log := log.NewLogger("error", "")
|
||||
tableName := generateRandomString()
|
||||
|
||||
// Create Table
|
||||
_, err := exec.Command("aws", "dynamodb", "--region", region, "--endpoint-url", localEndpoint, "create-table",
|
||||
"--table-name", tableName, "--attribute-definitions", "AttributeName=Digest,AttributeType=S",
|
||||
"--key-schema", "AttributeName=Digest,KeyType=HASH",
|
||||
"--billing-mode", "PAY_PER_REQUEST").Output()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
cache, _ := storage.Create("dynamodb", cache.DynamoDBDriverParameters{
|
||||
Endpoint: localEndpoint,
|
||||
Region: region,
|
||||
TableName: tableName,
|
||||
}, log)
|
||||
testData := generateData()
|
||||
|
||||
for digest, path := range testData {
|
||||
_ = cache.PutBlob(digest, path)
|
||||
}
|
||||
|
||||
b.ResetTimer()
|
||||
|
||||
helperHasAll(cache, testData)
|
||||
}
|
||||
|
||||
func BenchmarkGetLocalstack(b *testing.B) {
|
||||
log := log.NewLogger("error", "")
|
||||
tableName := generateRandomString()
|
||||
|
||||
// Create Table
|
||||
_, err := exec.Command("aws", "dynamodb", "--region", region, "--endpoint-url", localEndpoint, "create-table",
|
||||
"--table-name", tableName, "--attribute-definitions", "AttributeName=Digest,AttributeType=S",
|
||||
"--key-schema", "AttributeName=Digest,KeyType=HASH",
|
||||
"--billing-mode", "PAY_PER_REQUEST").Output()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
cache, _ := storage.Create("dynamodb", cache.DynamoDBDriverParameters{
|
||||
Endpoint: localEndpoint,
|
||||
Region: region,
|
||||
TableName: tableName,
|
||||
}, log)
|
||||
testData := generateData()
|
||||
counter := 1
|
||||
|
||||
var previousDigest godigest.Digest
|
||||
|
||||
for digest, path := range testData {
|
||||
if counter != 10 {
|
||||
_ = cache.PutBlob(digest, path)
|
||||
previousDigest = digest
|
||||
counter++
|
||||
} else {
|
||||
_ = cache.PutBlob(previousDigest, path)
|
||||
counter = 1
|
||||
}
|
||||
}
|
||||
|
||||
b.ResetTimer()
|
||||
|
||||
helperGetAll(cache, testData)
|
||||
}
|
||||
|
||||
func BenchmarkMixLocalstack(b *testing.B) {
|
||||
log := log.NewLogger("error", "")
|
||||
tableName := generateRandomString()
|
||||
|
||||
// Create Table
|
||||
_, err := exec.Command("aws", "dynamodb", "--region", region, "--endpoint-url", localEndpoint, "create-table",
|
||||
"--table-name", tableName, "--attribute-definitions", "AttributeName=Digest,AttributeType=S",
|
||||
"--key-schema", "AttributeName=Digest,KeyType=HASH",
|
||||
"--billing-mode", "PAY_PER_REQUEST").Output()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
cache, _ := storage.Create("dynamodb", cache.DynamoDBDriverParameters{
|
||||
Endpoint: localEndpoint,
|
||||
Region: region,
|
||||
TableName: tableName,
|
||||
}, log)
|
||||
testData := generateData()
|
||||
digestSlice := make([]godigest.Digest, datasetSize)
|
||||
counter := 0
|
||||
|
||||
for key := range testData {
|
||||
digestSlice[counter] = key
|
||||
counter++
|
||||
}
|
||||
|
||||
b.ResetTimer()
|
||||
|
||||
helperMix(cache, testData, digestSlice)
|
||||
}
|
||||
|
||||
// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
|
||||
// DANGER ZONE: tests with true AWS endpoint
|
||||
// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
|
||||
|
||||
func BenchmarkPutAWS(b *testing.B) {
|
||||
log := log.NewLogger("error", "")
|
||||
tableName := generateRandomString()
|
||||
|
||||
// Create Table
|
||||
_, err := exec.Command("aws", "dynamodb", "--region", region, "--endpoint-url", awsEndpoint, "create-table",
|
||||
"--table-name", tableName, "--attribute-definitions", "AttributeName=Digest,AttributeType=S",
|
||||
"--key-schema", "AttributeName=Digest,KeyType=HASH",
|
||||
"--billing-mode", "PAY_PER_REQUEST").Output()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
cache, _ := storage.Create("dynamodb", cache.DynamoDBDriverParameters{
|
||||
Endpoint: awsEndpoint,
|
||||
Region: region,
|
||||
TableName: tableName,
|
||||
}, log)
|
||||
testData := generateData()
|
||||
|
||||
b.ResetTimer()
|
||||
|
||||
helperPutAll(cache, testData)
|
||||
}
|
||||
|
||||
func BenchmarkDeleteAWS(b *testing.B) {
|
||||
log := log.NewLogger("error", "")
|
||||
tableName := generateRandomString()
|
||||
|
||||
// Create Table
|
||||
_, err := exec.Command("aws", "dynamodb", "--region", region, "--endpoint-url", awsEndpoint, "create-table",
|
||||
"--table-name", tableName, "--attribute-definitions", "AttributeName=Digest,AttributeType=S",
|
||||
"--key-schema", "AttributeName=Digest,KeyType=HASH",
|
||||
"--billing-mode", "PAY_PER_REQUEST").Output()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
cache, _ := storage.Create("dynamodb", cache.DynamoDBDriverParameters{
|
||||
Endpoint: awsEndpoint,
|
||||
Region: region,
|
||||
TableName: tableName,
|
||||
}, log)
|
||||
testData := generateData()
|
||||
|
||||
for digest, path := range testData {
|
||||
_ = cache.PutBlob(digest, path)
|
||||
}
|
||||
|
||||
b.ResetTimer()
|
||||
|
||||
helperDeleteAll(cache, testData)
|
||||
}
|
||||
|
||||
func BenchmarkHasAWS(b *testing.B) {
|
||||
log := log.NewLogger("error", "")
|
||||
tableName := generateRandomString()
|
||||
|
||||
// Create Table
|
||||
_, err := exec.Command("aws", "dynamodb", "--region", region, "--endpoint-url", awsEndpoint, "create-table",
|
||||
"--table-name", tableName, "--attribute-definitions", "AttributeName=Digest,AttributeType=S",
|
||||
"--key-schema", "AttributeName=Digest,KeyType=HASH",
|
||||
"--billing-mode", "PAY_PER_REQUEST").Output()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
cache, _ := storage.Create("dynamodb", cache.DynamoDBDriverParameters{
|
||||
Endpoint: awsEndpoint,
|
||||
Region: region,
|
||||
TableName: tableName,
|
||||
}, log)
|
||||
testData := generateData()
|
||||
|
||||
for digest, path := range testData {
|
||||
_ = cache.PutBlob(digest, path)
|
||||
}
|
||||
|
||||
b.ResetTimer()
|
||||
|
||||
helperHasAll(cache, testData)
|
||||
}
|
||||
|
||||
func BenchmarkGetAWS(b *testing.B) {
|
||||
log := log.NewLogger("error", "")
|
||||
tableName := generateRandomString()
|
||||
|
||||
// Create Table
|
||||
_, err := exec.Command("aws", "dynamodb", "--region", region, "--endpoint-url", awsEndpoint, "create-table",
|
||||
"--table-name", tableName, "--attribute-definitions", "AttributeName=Digest,AttributeType=S",
|
||||
"--key-schema", "AttributeName=Digest,KeyType=HASH",
|
||||
"--billing-mode", "PAY_PER_REQUEST").Output()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
cache, _ := storage.Create("dynamodb", cache.DynamoDBDriverParameters{
|
||||
Endpoint: awsEndpoint,
|
||||
Region: region,
|
||||
TableName: tableName,
|
||||
}, log)
|
||||
testData := generateData()
|
||||
counter := 1
|
||||
|
||||
var previousDigest godigest.Digest
|
||||
|
||||
for digest, path := range testData {
|
||||
if counter != 10 {
|
||||
_ = cache.PutBlob(digest, path)
|
||||
previousDigest = digest
|
||||
counter++
|
||||
} else {
|
||||
_ = cache.PutBlob(previousDigest, path)
|
||||
counter = 1
|
||||
}
|
||||
}
|
||||
|
||||
b.ResetTimer()
|
||||
|
||||
helperGetAll(cache, testData)
|
||||
}
|
||||
|
||||
func BenchmarkMixAWS(b *testing.B) {
|
||||
log := log.NewLogger("error", "")
|
||||
tableName := generateRandomString()
|
||||
|
||||
// Create Table
|
||||
_, err := exec.Command("aws", "dynamodb", "--region", region, "--endpoint-url", awsEndpoint, "create-table",
|
||||
"--table-name", tableName, "--attribute-definitions", "AttributeName=Digest,AttributeType=S",
|
||||
"--key-schema", "AttributeName=Digest,KeyType=HASH",
|
||||
"--billing-mode", "PAY_PER_REQUEST").Output()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
cache, _ := storage.Create("dynamodb", cache.DynamoDBDriverParameters{
|
||||
Endpoint: awsEndpoint,
|
||||
Region: region,
|
||||
TableName: tableName,
|
||||
}, log)
|
||||
testData := generateData()
|
||||
digestSlice := make([]godigest.Digest, datasetSize)
|
||||
counter := 0
|
||||
|
||||
for key := range testData {
|
||||
digestSlice[counter] = key
|
||||
counter++
|
||||
}
|
||||
|
||||
b.ResetTimer()
|
||||
|
||||
helperMix(cache, testData, digestSlice)
|
||||
}
|
||||
@ -19,5 +19,5 @@ const (
|
||||
DBCacheLockCheckTimeout = 10 * time.Second
|
||||
BoltdbName = "cache"
|
||||
ReferrerFilterAnnotation = "org.opencontainers.references.filtersApplied"
|
||||
//
|
||||
DynamoDBDriverName = "dynamodb"
|
||||
)
|
||||
|
||||
@ -44,6 +44,7 @@ var (
|
||||
fileInfoSize = 10
|
||||
errorText = "new s3 error"
|
||||
errS3 = errors.New(errorText)
|
||||
zotStorageTest = "zot-storage-test"
|
||||
)
|
||||
|
||||
func cleanupStorage(store driver.StorageDriver, name string) {
|
||||
@ -56,6 +57,10 @@ func skipIt(t *testing.T) {
|
||||
if os.Getenv("S3MOCK_ENDPOINT") == "" {
|
||||
t.Skip("Skipping testing without AWS S3 mock server")
|
||||
}
|
||||
|
||||
if os.Getenv("DYNAMODBMOCK_ENDPOINT") == "" {
|
||||
t.Skip("Skipping testing without AWS DynamoDB mock server")
|
||||
}
|
||||
}
|
||||
|
||||
func createMockStorage(rootDir string, cacheDir string, dedupe bool, store driver.StorageDriver) storage.ImageStore {
|
||||
@ -84,7 +89,7 @@ func createObjectsStore(rootDir string, cacheDir string, dedupe bool) (
|
||||
storage.ImageStore,
|
||||
error,
|
||||
) {
|
||||
bucket := "zot-storage-test"
|
||||
bucket := zotStorageTest
|
||||
endpoint := os.Getenv("S3MOCK_ENDPOINT")
|
||||
storageDriverParams := map[string]interface{}{
|
||||
"rootDir": rootDir,
|
||||
@ -130,6 +135,66 @@ func createObjectsStore(rootDir string, cacheDir string, dedupe bool) (
|
||||
return store, il, err
|
||||
}
|
||||
|
||||
func createObjectsStoreDynamo(rootDir string, cacheDir string, dedupe bool, tableName string) (
|
||||
driver.StorageDriver,
|
||||
storage.ImageStore,
|
||||
error,
|
||||
) {
|
||||
bucket := zotStorageTest
|
||||
endpoint := os.Getenv("S3MOCK_ENDPOINT")
|
||||
storageDriverParams := map[string]interface{}{
|
||||
"rootDir": rootDir,
|
||||
"name": "s3",
|
||||
"region": "us-east-2",
|
||||
"bucket": bucket,
|
||||
"regionendpoint": endpoint,
|
||||
"accesskey": "minioadmin",
|
||||
"secretkey": "minioadmin",
|
||||
"secure": false,
|
||||
"skipverify": false,
|
||||
}
|
||||
|
||||
storeName := fmt.Sprintf("%v", storageDriverParams["name"])
|
||||
|
||||
store, err := factory.Create(storeName, storageDriverParams)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
// create bucket if it doesn't exists
|
||||
_, err = resty.R().Put("http://" + endpoint + "/" + bucket)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
log := log.Logger{Logger: zerolog.New(os.Stdout)}
|
||||
metrics := monitoring.NewMetricsServer(false, log)
|
||||
|
||||
var cacheDriver cache.Cache
|
||||
|
||||
// from pkg/cli/root.go/applyDefaultValues, s3 magic
|
||||
|
||||
cacheDriver, _ = storage.Create("dynamodb", cache.DynamoDBDriverParameters{
|
||||
Endpoint: os.Getenv("DYNAMODBMOCK_ENDPOINT"),
|
||||
Region: os.Getenv("us-east-2"),
|
||||
TableName: tableName,
|
||||
}, log)
|
||||
|
||||
tableName = strings.ReplaceAll(tableName, "/", "")
|
||||
//nolint:errcheck
|
||||
cacheDriverDynamo, _ := cacheDriver.(*cache.DynamoDBDriver)
|
||||
|
||||
err = cacheDriverDynamo.NewTable(tableName)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
il := s3.NewImageStore(rootDir, cacheDir, false, storage.DefaultGCDelay,
|
||||
dedupe, false, log, metrics, nil, store, cacheDriver)
|
||||
|
||||
return store, il, err
|
||||
}
|
||||
|
||||
type FileInfoMock struct {
|
||||
IsDirFn func() bool
|
||||
SizeFn func() int64
|
||||
@ -607,7 +672,7 @@ func TestNegativeCasesObjectsStorage(t *testing.T) {
|
||||
})
|
||||
|
||||
Convey("Unable to create subpath cache db", func(c C) {
|
||||
bucket := "zot-storage-test"
|
||||
bucket := zotStorageTest
|
||||
endpoint := os.Getenv("S3MOCK_ENDPOINT")
|
||||
|
||||
storageDriverParams := config.GlobalStorageConfig{
|
||||
@ -1448,6 +1513,189 @@ func TestS3Dedupe(t *testing.T) {
|
||||
So(fi1.Size(), ShouldEqual, fi3.Size())
|
||||
})
|
||||
})
|
||||
|
||||
Convey("Dedupe with dynamodb", t, func(c C) {
|
||||
uuid, err := guuid.NewV4()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
testDir := path.Join("/oci-repo-test", uuid.String())
|
||||
|
||||
tdir := t.TempDir()
|
||||
|
||||
storeDriver, imgStore, _ := createObjectsStoreDynamo(testDir, tdir, true, tdir)
|
||||
defer cleanupStorage(storeDriver, testDir)
|
||||
|
||||
// manifest1
|
||||
upload, err := imgStore.NewBlobUpload("dedupe1")
|
||||
So(err, ShouldBeNil)
|
||||
So(upload, ShouldNotBeEmpty)
|
||||
|
||||
content := []byte("test-data3")
|
||||
buf := bytes.NewBuffer(content)
|
||||
buflen := buf.Len()
|
||||
digest := godigest.FromBytes(content)
|
||||
blob, err := imgStore.PutBlobChunkStreamed("dedupe1", upload, buf)
|
||||
So(err, ShouldBeNil)
|
||||
So(blob, ShouldEqual, buflen)
|
||||
blobDigest1 := digest
|
||||
So(blobDigest1, ShouldNotBeEmpty)
|
||||
|
||||
err = imgStore.FinishBlobUpload("dedupe1", upload, buf, digest)
|
||||
So(err, ShouldBeNil)
|
||||
So(blob, ShouldEqual, buflen)
|
||||
|
||||
_, checkBlobSize1, err := imgStore.CheckBlob("dedupe1", digest)
|
||||
So(checkBlobSize1, ShouldBeGreaterThan, 0)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
blobReadCloser, getBlobSize1, err := imgStore.GetBlob("dedupe1", digest,
|
||||
"application/vnd.oci.image.layer.v1.tar+gzip")
|
||||
So(getBlobSize1, ShouldBeGreaterThan, 0)
|
||||
So(err, ShouldBeNil)
|
||||
err = blobReadCloser.Close()
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
cblob, cdigest := test.GetRandomImageConfig()
|
||||
_, clen, err := imgStore.FullBlobUpload("dedupe1", bytes.NewReader(cblob), cdigest)
|
||||
So(err, ShouldBeNil)
|
||||
So(clen, ShouldEqual, len(cblob))
|
||||
hasBlob, _, err := imgStore.CheckBlob("dedupe1", cdigest)
|
||||
So(err, ShouldBeNil)
|
||||
So(hasBlob, ShouldEqual, true)
|
||||
|
||||
manifest := ispec.Manifest{
|
||||
Config: ispec.Descriptor{
|
||||
MediaType: "application/vnd.oci.image.config.v1+json",
|
||||
Digest: cdigest,
|
||||
Size: int64(len(cblob)),
|
||||
},
|
||||
Layers: []ispec.Descriptor{
|
||||
{
|
||||
MediaType: "application/vnd.oci.image.layer.v1.tar",
|
||||
Digest: digest,
|
||||
Size: int64(buflen),
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
manifest.SchemaVersion = 2
|
||||
manifestBuf, err := json.Marshal(manifest)
|
||||
So(err, ShouldBeNil)
|
||||
digest = godigest.FromBytes(manifestBuf)
|
||||
_, err = imgStore.PutImageManifest("dedupe1", digest.String(),
|
||||
ispec.MediaTypeImageManifest, manifestBuf)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
_, _, _, err = imgStore.GetImageManifest("dedupe1", digest.String())
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
// manifest2
|
||||
upload, err = imgStore.NewBlobUpload("dedupe2")
|
||||
So(err, ShouldBeNil)
|
||||
So(upload, ShouldNotBeEmpty)
|
||||
|
||||
content = []byte("test-data3")
|
||||
buf = bytes.NewBuffer(content)
|
||||
buflen = buf.Len()
|
||||
digest = godigest.FromBytes(content)
|
||||
|
||||
blob, err = imgStore.PutBlobChunkStreamed("dedupe2", upload, buf)
|
||||
So(err, ShouldBeNil)
|
||||
So(blob, ShouldEqual, buflen)
|
||||
blobDigest2 := digest
|
||||
So(blobDigest2, ShouldNotBeEmpty)
|
||||
|
||||
err = imgStore.FinishBlobUpload("dedupe2", upload, buf, digest)
|
||||
So(err, ShouldBeNil)
|
||||
So(blob, ShouldEqual, buflen)
|
||||
|
||||
_, checkBlobSize2, err := imgStore.CheckBlob("dedupe2", digest)
|
||||
So(err, ShouldBeNil)
|
||||
So(checkBlobSize2, ShouldBeGreaterThan, 0)
|
||||
|
||||
blobReadCloser, getBlobSize2, err := imgStore.GetBlob("dedupe2", digest,
|
||||
"application/vnd.oci.image.layer.v1.tar+gzip")
|
||||
So(err, ShouldBeNil)
|
||||
So(getBlobSize2, ShouldBeGreaterThan, 0)
|
||||
So(checkBlobSize1, ShouldEqual, checkBlobSize2)
|
||||
So(getBlobSize1, ShouldEqual, getBlobSize2)
|
||||
err = blobReadCloser.Close()
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
cblob, cdigest = test.GetRandomImageConfig()
|
||||
_, clen, err = imgStore.FullBlobUpload("dedupe2", bytes.NewReader(cblob), cdigest)
|
||||
So(err, ShouldBeNil)
|
||||
So(clen, ShouldEqual, len(cblob))
|
||||
hasBlob, _, err = imgStore.CheckBlob("dedupe2", cdigest)
|
||||
So(err, ShouldBeNil)
|
||||
So(hasBlob, ShouldEqual, true)
|
||||
|
||||
manifest = ispec.Manifest{
|
||||
Config: ispec.Descriptor{
|
||||
MediaType: "application/vnd.oci.image.config.v1+json",
|
||||
Digest: cdigest,
|
||||
Size: int64(len(cblob)),
|
||||
},
|
||||
Layers: []ispec.Descriptor{
|
||||
{
|
||||
MediaType: "application/vnd.oci.image.layer.v1.tar",
|
||||
Digest: digest,
|
||||
Size: int64(buflen),
|
||||
},
|
||||
},
|
||||
}
|
||||
manifest.SchemaVersion = 2
|
||||
manifestBuf, err = json.Marshal(manifest)
|
||||
So(err, ShouldBeNil)
|
||||
digest = godigest.FromBytes(manifestBuf)
|
||||
_, err = imgStore.PutImageManifest("dedupe2", "1.0", ispec.MediaTypeImageManifest,
|
||||
manifestBuf)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
_, _, _, err = imgStore.GetImageManifest("dedupe2", digest.String())
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
fi1, err := storeDriver.Stat(context.Background(), path.Join(testDir, "dedupe1", "blobs", "sha256",
|
||||
blobDigest1.Encoded()))
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
fi2, err := storeDriver.Stat(context.Background(), path.Join(testDir, "dedupe2", "blobs", "sha256",
|
||||
blobDigest2.Encoded()))
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
// original blob should have the real content of blob
|
||||
So(fi1.Size(), ShouldNotEqual, fi2.Size())
|
||||
So(fi1.Size(), ShouldBeGreaterThan, 0)
|
||||
// deduped blob should be of size 0
|
||||
So(fi2.Size(), ShouldEqual, 0)
|
||||
|
||||
Convey("Check that delete blobs moves the real content to the next contenders", func() {
|
||||
// if we delete blob1, the content should be moved to blob2
|
||||
err = imgStore.DeleteBlob("dedupe1", blobDigest1)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
_, err = storeDriver.Stat(context.Background(), path.Join(testDir, "dedupe1", "blobs", "sha256",
|
||||
blobDigest1.Encoded()))
|
||||
So(err, ShouldNotBeNil)
|
||||
|
||||
fi2, err = storeDriver.Stat(context.Background(), path.Join(testDir, "dedupe2", "blobs", "sha256",
|
||||
blobDigest2.Encoded()))
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
So(fi2.Size(), ShouldBeGreaterThan, 0)
|
||||
// the second blob should now be equal to the deleted blob.
|
||||
So(fi2.Size(), ShouldEqual, fi1.Size())
|
||||
|
||||
err = imgStore.DeleteBlob("dedupe2", blobDigest2)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
_, err = storeDriver.Stat(context.Background(), path.Join(testDir, "dedupe2", "blobs", "sha256",
|
||||
blobDigest2.Encoded()))
|
||||
So(err, ShouldNotBeNil)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
func TestS3PullRange(t *testing.T) {
|
||||
|
||||
88
test/blackbox/cloud-only.bats
Normal file
88
test/blackbox/cloud-only.bats
Normal file
@ -0,0 +1,88 @@
|
||||
load helpers_cloud
|
||||
|
||||
function setup() {
|
||||
# Verify prerequisites are available
|
||||
if ! verify_prerequisites; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Setup zot server
|
||||
local zot_root_dir=${BATS_FILE_TMPDIR}/zot
|
||||
local zot_config_file=${BATS_FILE_TMPDIR}/zot_config.json
|
||||
|
||||
echo ${zot_root_dir} >&3
|
||||
|
||||
mkdir -p ${zot_root_dir}
|
||||
|
||||
cat > ${zot_config_file}<<EOF
|
||||
{
|
||||
"distSpecVersion": "1.0.1-dev",
|
||||
"storage": {
|
||||
"rootDirectory": "${zot_root_dir}",
|
||||
"dedupe": true,
|
||||
"remoteCache": true,
|
||||
"storageDriver": {
|
||||
"name": "s3",
|
||||
"rootdirectory": "/zot",
|
||||
"region": "us-east-2",
|
||||
"regionendpoint": "localhost:4566",
|
||||
"bucket": "zot-storage",
|
||||
"secure": false,
|
||||
"skipverify": false
|
||||
},
|
||||
"cacheDriver": {
|
||||
"name": "dynamodb",
|
||||
"endpoint": "http://localhost:4566",
|
||||
"region": "us-east-2",
|
||||
"tableName": "BlobTable"
|
||||
}
|
||||
},
|
||||
"http": {
|
||||
"address": "127.0.0.1",
|
||||
"port": "8080"
|
||||
},
|
||||
"log": {
|
||||
"level": "debug"
|
||||
},
|
||||
"extensions": {
|
||||
"metrics": {
|
||||
"enable": true,
|
||||
"prometheus": {
|
||||
"path": "/metrics"
|
||||
}
|
||||
},
|
||||
"search": {
|
||||
"cve": {
|
||||
"updateInterval": "2h"
|
||||
}
|
||||
},
|
||||
"scrub": {
|
||||
"enable": true,
|
||||
"interval": "24h"
|
||||
}
|
||||
}
|
||||
}
|
||||
EOF
|
||||
awslocal s3 --region "us-east-2" mb s3://zot-storage
|
||||
awslocal dynamodb --endpoint-url "http://localhost:4566" --region "us-east-2" create-table --table-name "BlobTable" --attribute-definitions AttributeName=Digest,AttributeType=S --key-schema AttributeName=Digest,KeyType=HASH --provisioned-throughput ReadCapacityUnits=10,WriteCapacityUnits=5
|
||||
zot_serve_strace ${zot_config_file}
|
||||
wait_zot_reachable "http://127.0.0.1:8080/v2/_catalog"
|
||||
}
|
||||
|
||||
function teardown() {
|
||||
local zot_root_dir=${BATS_FILE_TMPDIR}/zot
|
||||
zot_stop
|
||||
rm -rf ${zot_root_dir}
|
||||
aws s3 --endpoint-url "http://localhost:4566" rb s3://"zot-storage" --force
|
||||
aws dynamodb --endpoint-url "http://localhost:4566" --region "us-east-2" delete-table --table-name "BlobTable"
|
||||
}
|
||||
|
||||
@test "check for local disk writes" {
|
||||
run skopeo --insecure-policy copy --dest-tls-verify=false \
|
||||
docker://centos:centos8 docker://localhost:8080/centos:8
|
||||
[ "$status" -eq 0 ]
|
||||
cat strace.txt | grep openat | grep -v O_RDONLY | grep -Eo '\".*\"' | while read -r line ; do
|
||||
echo ${line} >&3
|
||||
[[ "$line" =~ .*metadata.* || "$line" =~ .*trivy.* ]]
|
||||
done
|
||||
}
|
||||
44
test/blackbox/helpers_cloud.bash
Normal file
44
test/blackbox/helpers_cloud.bash
Normal file
@ -0,0 +1,44 @@
|
||||
ROOT_DIR=$(git rev-parse --show-toplevel)
|
||||
OS="${OS:-linux}"
|
||||
ARCH="${ARCH:-amd64}"
|
||||
ZOT_PATH=${ROOT_DIR}/bin/zot-${OS}-${ARCH}
|
||||
|
||||
|
||||
function verify_prerequisites {
|
||||
if [ ! -f ${ZOT_PATH} ]; then
|
||||
echo "you need to build ${ZOT_PATH} before running the tests" >&3
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [ ! command -v skopeo &> /dev/null ]; then
|
||||
echo "you need to install skopeo as a prerequisite to running the tests" >&3
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [ ! command -v awslocal ] &>/dev/null; then
|
||||
echo "you need to install aws cli as a prerequisite to running the tests" >&3
|
||||
return 1
|
||||
fi
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
function zot_serve_strace() {
|
||||
local config_file=${1}
|
||||
strace -o "strace.txt" -f -e trace=openat ${ZOT_PATH} serve ${config_file} &
|
||||
}
|
||||
|
||||
function zot_stop() {
|
||||
pkill zot
|
||||
}
|
||||
|
||||
function wait_zot_reachable() {
|
||||
zot_url=${1}
|
||||
curl --connect-timeout 3 \
|
||||
--max-time 3 \
|
||||
--retry 10 \
|
||||
--retry-delay 0 \
|
||||
--retry-max-time 60 \
|
||||
--retry-connrefused \
|
||||
${zot_url}
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user