refactor(makefile): consolidate the make targets used for bats tests (#1746)

New examples of running tests:

1. To run a specific bats file (with and without verbose output):
make run-blackbox-tests BATS_TEST_FILE_PATH=test/blackbox/delete_images.bats
make run-blackbox-tests BATS_TEST_FILE_PATH=test/blackbox/delete_images.bats BATS_VERBOSITY=2

2. To run the CI tests (with and without verbose output)
make run-blackbox-ci
make run-blackbox-ci BATS_VERBOSITY=2

BATS_TEST_FILE_PATH is used to pass on the test file to run using `run-blackbox-tests`
BATS_VERBOSITY controls the verbosity of the bats framework output, if unspecified the output only
contains test results and failure message in case of failures.
If BATS_VERBOSITY is 1, then also show commands as they are executed.
If BATS_VERBOSITY is 2, on top of the above it also shows output of passed tests.

Other changes in this PR:
- Update some of the tests to show logs after the run ends.
- Run the linters before the tests, as it saves time on failures when running in GH

Signed-off-by: Andrei Aaron <aaaron@luxoft.com>
This commit is contained in:
Andrei Aaron 2023-09-07 21:06:21 +03:00 committed by GitHub
parent f5b63963be
commit 6115eed4ec
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
22 changed files with 199 additions and 211 deletions

View File

@ -69,42 +69,9 @@ jobs:
sudo du -sh /var/lib/docker/
du -sh /home/runner/work/
set +x
- name: Run referrers tests
- name: Run CI tests
run: |
make test-bats-referrers
- name: Run metadata tests
run: |
make test-bats-metadata
- name: Run push-pull tests
run: |
make test-push-pull
- name: Run push-pull-authn tests
run: |
make test-push-pull-authn
- name: Run metrics tests
run: |
make test-bats-metrics
- name: Run cve tests
run: |
make test-bats-cve
- name: Run sync test
run: |
make test-bats-sync
- name: Run scrub tests
run: |
make test-bats-scrub
- name: Run anonymous-push-pull tests
run: |
make test-anonymous-push-pull
- name: Run detect-manifest-collision tests
run: |
make test-detect-manifest-collision
- name: Run annotations tests
run: |
make test-annotations
- name: Run garbage collect tests
run: |
make test-garbage-collect
make run-blackbox-ci
- name: Install localstack
run: |
pip install --upgrade pyopenssl
@ -117,7 +84,7 @@ jobs:
echo "Startup complete"
- name: Run cloud-only tests
run: |
make test-cloud-only
make run-blackbox-cloud-ci
env:
AWS_ACCESS_KEY_ID: fake
AWS_SECRET_ACCESS_KEY: fake

View File

@ -44,15 +44,11 @@ jobs:
echo "Waiting for LocalStack startup..." # Wait 30 seconds for the LocalStack container
localstack wait -t 30 # to become ready before timing out
echo "Startup complete"
- name: Run restore s3 blobs after cache is deleted
- name: Run blackbox nightly dedupe tests
run: |
make test-restore-s3-blobs
env:
AWS_ACCESS_KEY_ID: fake
AWS_SECRET_ACCESS_KEY: fake
- name: Run dedupe filesystem blobs after switching dedupe to enable.
run: |
make test-push-pull-running-dedupe
# test restoring s3 blobs after cache is deleted
# test deduping filesystem blobs after switching dedupe to enable
make run-blackbox-dedupe-nightly
env:
AWS_ACCESS_KEY_ID: fake
AWS_SECRET_ACCESS_KEY: fake
@ -73,7 +69,8 @@ jobs:
go mod download
- name: Run sync harness
run: |
make test-sync-harness
make run-blackbox-sync-nightly
gc-referrers-stress-s3:
name: GC(with referrers) on S3 with short interval
runs-on: ubuntu-latest

177
Makefile
View File

@ -54,8 +54,17 @@ filter-valid = $(foreach ext, $(merged-extensions), $(if $(findstring $(ext),$(A
add-extensions = $(subst $(1),$(2),$(sort $(filter-valid)))
BUILD_LABELS = $(call add-extensions,$(space),$(comma))
BATS_TEST_FILE_PATH ?= replace_me
ifeq ($(BATS_VERBOSITY),2)
BATS_FLAGS = --trace --verbose-run --show-output-of-passing-tests --print-output-on-failure
else ifeq ($(BATS_VERBOSITY),1)
BATS_FLAGS = --trace --verbose-run --print-output-on-failure
else
BATS_FLAGS = --print-output-on-failure
endif
.PHONY: all
all: modcheck swaggercheck binary binary-minimal binary-debug cli bench exporter-minimal verify-config test covhtml check check-gh-actions
all: modcheck swaggercheck binary binary-minimal binary-debug cli bench exporter-minimal verify-config check check-gh-actions test covhtml
.PHONY: modtidy
modtidy:
@ -162,6 +171,10 @@ run-bench: binary bench
check-skopeo:
skopeo -v || (echo "You need skopeo to be installed in order to run tests"; exit 1)
.PHONY: check-awslocal
check-awslocal:
awslocal --version || (echo "You need awslocal to be installed in order to run tests"; exit 1)
$(NOTATION):
mkdir -p $(TOOLSDIR)/bin
curl -Lo notation.tar.gz https://github.com/notaryproject/notation/releases/download/v$(NOTATION_VERSION)/notation_$(NOTATION_VERSION)_linux_amd64.tar.gz
@ -349,127 +362,57 @@ $(BATS):
cd bats-core; ./install.sh $(TOOLSDIR); cd ..; \
rm -rf bats-core
.PHONY: test-push-pull
test-push-pull: check-linux binary check-skopeo $(BATS) $(REGCLIENT) $(ORAS) $(HELM) $(CRICTL)
$(BATS) --trace --print-output-on-failure test/blackbox/pushpull.bats
.PHONY: check-blackbox-prerequisites
check-blackbox-prerequisites: check-linux check-skopeo $(BATS) $(REGCLIENT) $(ORAS) $(HELM) $(CRICTL) $(NOTATION) $(COSIGN) $(STACKER)
which skopeo && skopeo --version; \
which stacker && stacker --version; \
which regctl && regctl version; \
which oras && oras version; \
which helm && helm version; \
which crictl && crictl version; \
which notation && notation version; \
which cosign && cosign version;
.PHONY: test-push-pull-verbose
test-push-pull-verbose: check-linux binary check-skopeo $(BATS) $(REGCLIENT) $(ORAS) $(HELM) $(CRICTL)
$(BATS) --trace --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/pushpull.bats
.PHONY: run-blackbox-tests
run-blackbox-tests: $(BATS_TEST_FILE_PATH) check-blackbox-prerequisites binary binary-minimal cli bench
echo running bats test "$(BATS_TEST_FILE_PATH)"; \
$(BATS) $(BATS_FLAGS) $(BATS_TEST_FILE_PATH)
.PHONY: test-garbage-collect
test-garbage-collect: binary check-skopeo $(BATS) $(REGCLIENT) $(ORAS)
$(BATS) --trace --print-output-on-failure test/blackbox/garbage_collect.bats
.PHONY: run-blackbox-ci
run-blackbox-ci: check-blackbox-prerequisites binary binary-minimal cli
# ideally we would run a single bats command but too much disk space would be used at once
echo running CI bats tests; \
$(BATS) $(BATS_FLAGS) test/blackbox/pushpull.bats && \
$(BATS) $(BATS_FLAGS) test/blackbox/pushpull_authn.bats && \
$(BATS) $(BATS_FLAGS) test/blackbox/delete_images.bats && \
$(BATS) $(BATS_FLAGS) test/blackbox/referrers.bats && \
$(BATS) $(BATS_FLAGS) test/blackbox/metadata.bats && \
$(BATS) $(BATS_FLAGS) test/blackbox/anonymous_policy.bats && \
$(BATS) $(BATS_FLAGS) test/blackbox/annotations.bats && \
$(BATS) $(BATS_FLAGS) test/blackbox/detect_manifest_collision.bats && \
$(BATS) $(BATS_FLAGS) test/blackbox/cve.bats && \
$(BATS) $(BATS_FLAGS) test/blackbox/sync.bats && \
$(BATS) $(BATS_FLAGS) test/blackbox/sync_docker.bats && \
$(BATS) $(BATS_FLAGS) test/blackbox/sync_replica_cluster.bats && \
$(BATS) $(BATS_FLAGS) test/blackbox/scrub.bats && \
$(BATS) $(BATS_FLAGS) test/blackbox/garbage_collect.bats && \
$(BATS) $(BATS_FLAGS) test/blackbox/metrics.bats
.PHONY: test-garbage-collect-verbose
test-garbage-collect-verbose: binary check-skopeo $(BATS) $(REGCLIENT) $(ORAS)
$(BATS) --trace --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/garbage_collect.bats
.PHONY: run-blackbox-cloud-ci
run-blackbox-cloud-ci: check-blackbox-prerequisites check-awslocal binary $(BATS)
echo running cloud CI bats tests; \
$(BATS) $(BATS_FLAGS) test/blackbox/cloud_only.bats
.PHONY: test-push-pull-running-dedupe
test-push-pull-running-dedupe: check-linux binary check-skopeo $(BATS) $(REGCLIENT) $(ORAS) $(HELM)
$(BATS) --trace --print-output-on-failure test/blackbox/pushpull_running_dedupe.bats
.PHONY: run-blackbox-dedupe-nightly
run-blackbox-dedupe-nightly: check-blackbox-prerequisites check-awslocal binary binary-minimal
echo running nightly dedupe tests; \
$(BATS) $(BATS_FLAGS) test/blackbox/restore_s3_blobs.bats && \
$(BATS) $(BATS_FLAGS) test/blackbox/pushpull_running_dedupe.bats
.PHONY: test-push-pull-running-dedupe-verbose
test-push-pull-running-dedupe-verbose: check-linux binary check-skopeo $(BATS) $(REGCLIENT) $(ORAS) $(HELM)
$(BATS) --trace --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/pushpull_running_dedupe.bats
.PHONY: test-push-pull-authn
test-push-pull-authn: check-linux binary check-skopeo $(BATS) $(REGCLIENT)
$(BATS) --trace --print-output-on-failure test/blackbox/pushpull_authn.bats
.PHONY: test-sync-harness
test-sync-harness: check-linux binary binary-minimal bench check-skopeo $(BATS)
$(BATS) --trace --print-output-on-failure test/blackbox/sync_harness.bats
.PHONY: test-sync-harness-verbose
test-sync-harness-verbose: check-linux binary binary-minimal bench check-skopeo $(BATS)
$(BATS) --trace --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/sync_harness.bats
.PHONY: test-restore-s3-blobs
test-restore-s3-blobs: check-linux binary check-skopeo $(BATS) $(REGCLIENT) $(ORAS) $(HELM)
$(BATS) --trace --print-output-on-failure test/blackbox/restore_s3_blobs.bats
.PHONY: test-restore-s3-blobs-verbose
test-restore-s3-blobs-verbose: check-linux binary check-skopeo $(BATS) $(REGCLIENT) $(ORAS) $(HELM)
$(BATS) --trace --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/restore_s3_blobs.bats
.PHONY: test-bats-referrers
test-bats-referrers: BUILD_LABELS=search
test-bats-referrers: check-linux binary check-skopeo $(BATS) $(ORAS)
$(BATS) --trace --print-output-on-failure test/blackbox/referrers.bats
.PHONY: test-bats-metadata
test-bats-metadata: BUILD_LABELS=search,userprefs
test-bats-metadata: check-linux binary check-skopeo $(BATS)
$(BATS) --trace --print-output-on-failure test/blackbox/metadata.bats
.PHONY: test-cloud-only
test-cloud-only: check-linux binary check-skopeo $(BATS)
$(BATS) --trace --print-output-on-failure test/blackbox/cloud_only.bats
.PHONY: test-cloud-only-verbose
test-cloud-only-verbose: check-linux binary check-skopeo $(BATS)
$(BATS) --trace --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/cloud_only.bats
.PHONY: test-bats-sync
test-bats-sync: BUILD_LABELS=sync
test-bats-sync: check-linux binary binary-minimal bench check-skopeo $(BATS) $(NOTATION) $(COSIGN) $(HELM)
$(BATS) --trace --print-output-on-failure test/blackbox/sync.bats
$(BATS) --trace --print-output-on-failure test/blackbox/sync_docker.bats
$(BATS) --trace --print-output-on-failure test/blackbox/sync_replica_cluster.bats
.PHONY: test-bats-delete-image
test-bats-delete-image: check-linux binary binary-minimal bench check-skopeo $(BATS)
$(BATS) --trace --print-output-on-failure test/blackbox/delete_images.bats
.PHONY: test-bats-sync-verbose
test-bats-sync-verbose: BUILD_LABELS=sync
test-bats-sync-verbose: check-linux binary binary-minimal bench check-skopeo $(BATS) $(NOTATION) $(COSIGN) $(HELM)
$(BATS) --trace -t -x -p --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/sync.bats
$(BATS) --trace -t -x -p --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/sync_docker.bats
$(BATS) --trace -t -x -p --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/sync_replica_cluster.bats
.PHONY: test-bats-cve
test-bats-cve: BUILD_LABELS=search
test-bats-cve: check-linux binary cli check-skopeo $(BATS)
$(BATS) --trace --print-output-on-failure test/blackbox/cve.bats
.PHONY: test-bats-cve-verbose
test-bats-cve-verbose: BUILD_LABELS=search
test-bats-cve-verbose: check-linux binary cli check-skopeo $(BATS)
$(BATS) --trace -t -x -p --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/cve.bats
.PHONY: test-bats-scrub
test-bats-scrub: BUILD_LABELS=scrub
test-bats-scrub: check-linux binary check-skopeo $(BATS)
$(BATS) --trace --print-output-on-failure test/blackbox/scrub.bats
.PHONY: test-bats-scrub-verbose
test-bats-scrub-verbose: BUILD_LABELS=scrub
test-bats-scrub-verbose: check-linux binary check-skopeo $(BATS)
$(BATS) --trace -p --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/scrub.bats
.PHONY: test-bats-metrics
test-bats-metrics: BUILD_LABELS=metrics
test-bats-metrics: check-linux binary check-skopeo $(BATS)
$(BATS) --trace --print-output-on-failure test/blackbox/metrics.bats
.PHONY: test-bats-metrics-verbose
test-bats-metrics-verbose: BUILD_LABELS=metrics
test-bats-metrics-verbose: check-linux binary check-skopeo $(BATS)
$(BATS) --trace -p --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/metrics.bats
.PHONY: test-anonymous-push-pull
test-anonymous-push-pull: check-linux binary check-skopeo $(BATS)
$(BATS) --trace --print-output-on-failure test/blackbox/anonymous_policy.bats
.PHONY: test-annotations
test-annotations: check-linux binary check-skopeo $(BATS) $(STACKER) $(NOTATION) $(COSIGN)
$(BATS) --trace --print-output-on-failure test/blackbox/annotations.bats
.PHONY: test-detect-manifest-collision
test-detect-manifest-collision: check-linux binary check-skopeo $(BATS) $(REGCLIENT)
$(BATS) --trace --print-output-on-failure test/blackbox/detect_manifest_collision.bats
.PHONY: run-blackbox-sync-nightly
run-blackbox-sync-nightly: check-blackbox-prerequisites binary binary-minimal bench
echo running nightly sync tests; \
$(BATS) $(BATS_FLAGS) test/blackbox/sync_harness.bats
.PHONY: fuzz-all
fuzz-all: fuzztime=${1}

View File

@ -1,4 +1,4 @@
# Note: Intended to be run as "make test-annotations"
# Note: Intended to be run as "make run-blackbox-tests" or "make run-blackbox-ci"
# Makefile target installs & checks all necessary tooling
# Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()
@ -79,8 +79,12 @@ EOF
wait_zot_reachable 8080
}
function teardown() {
# conditionally printing on failure is possible from teardown but not from from teardown_file
cat ${BATS_FILE_TMPDIR}/zot.log
}
function teardown_file() {
cat ${BATS_FILE_TMPDIR}/zot.log >&3
zot_stop_all
run rm -rf ${HOME}/.config/notation
}

View File

@ -1,4 +1,4 @@
# Note: Intended to be run as "make test-anonymous-push-pull"
# Note: Intended to be run as "make run-blackbox-tests" or "make run-blackbox-ci"
# Makefile target installs & checks all necessary tooling
# Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()
@ -59,7 +59,8 @@ function setup_file() {
}
},
"log": {
"level": "debug"
"level": "debug",
"output": "${BATS_FILE_TMPDIR}/zot.log"
}
}
EOF
@ -67,6 +68,11 @@ EOF
wait_zot_reachable 8080
}
function teardown() {
# conditionally printing on failure is possible from teardown but not from from teardown_file
cat ${BATS_FILE_TMPDIR}/zot.log
}
function teardown_file() {
zot_stop_all
}

View File

@ -1,3 +1,7 @@
# Note: Intended to be run as "make run-blackbox-tests" or "make run-blackbox-cloud-ci"
# Makefile target installs & checks all necessary tooling
# Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()
load helpers_cloud
load helpers_wait
@ -17,8 +21,8 @@ function setup() {
cat > ${zot_config_file}<<EOF
{
"distSpecVersion": "1.1.0-dev",
"storage": {
"distSpecVersion": "1.1.0-dev",
"storage": {
"rootDirectory": "${zot_root_dir}",
"dedupe": true,
"remoteCache": true,
@ -43,10 +47,10 @@ function setup() {
"apiKeyTablename":"ApiKeyTable",
"versionTablename": "Version"
}
},
"http": {
"address": "127.0.0.1",
"port": "8080",
},
"http": {
"address": "127.0.0.1",
"port": "8080",
"realm": "zot",
"auth": {
"openid": {
@ -68,25 +72,25 @@ function setup() {
}
}
}
},
"log": {
"level": "debug"
},
"extensions": {
"metrics": {
},
"log": {
"level": "debug"
},
"extensions": {
"metrics": {
"enable": true,
"prometheus": {
"path": "/metrics"
}
},
"search": {
"search": {
"enable": true
},
"scrub": {
"enable": true,
"interval": "24h"
}
}
},
"scrub": {
"enable": true,
"interval": "24h"
}
}
}
EOF
awslocal s3 --region "us-east-2" mb s3://zot-storage

View File

@ -1,4 +1,4 @@
# Note: Intended to be run as "make test-bats-cve" or "make test-bats-cve-verbose"
# Note: Intended to be run as "make run-blackbox-tests" or "make run-blackbox-ci"
# Makefile target installs & checks all necessary tooling
# Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()
@ -42,7 +42,8 @@ function setup_file() {
"port": "8080"
},
"log": {
"level": "debug"
"level": "debug",
"output": "${BATS_FILE_TMPDIR}/zot.log"
},
"extensions": {
"search": {
@ -63,6 +64,11 @@ EOF
zli_add_config ${registry_name} ${registry_url}
}
function teardown() {
# conditionally printing on failure is possible from teardown but not from from teardown_file
cat ${BATS_FILE_TMPDIR}/zot.log
}
function teardown_file() {
zot_stop_all
}

View File

@ -1,3 +1,7 @@
# Note: Intended to be run as "make run-blackbox-tests" or "make run-blackbox-ci"
# Makefile target installs & checks all necessary tooling
# Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()
load helpers_zot
function verify_prerequisites {
@ -14,7 +18,7 @@ function verify_prerequisites {
function setup_file() {
# Verify prerequisites are available
if ! (verify_prerequisites); then
if ! verify_prerequisites; then
exit 1
fi
@ -22,10 +26,10 @@ function setup_file() {
skopeo --insecure-policy copy --format=oci docker://ghcr.io/project-zot/test-images/alpine:3.17.3 oci:${TEST_DATA_DIR}/alpine:3.17.3
# Setup zot server
ZOT_ROOT_DIR=${BATS_RUN_TMPDIR}/zot
ZOT_ROOT_DIR=${BATS_FILE_TMPDIR}/zot
echo ${ZOT_ROOT_DIR}
local zot_log_file=${BATS_RUN_TMPDIR}/zot-log.json
local zot_config_file=${BATS_RUN_TMPDIR}/zot_config.json
local zot_log_file=${BATS_FILE_TMPDIR}/zot-log.json
local zot_config_file=${BATS_FILE_TMPDIR}/zot_config.json
mkdir -p ${ZOT_ROOT_DIR}
touch ${zot_log_file}
cat >${zot_config_file} <<EOF
@ -65,6 +69,11 @@ EOF
echo ${MANIFEST_DIGEST}
}
function teardown() {
# conditionally printing on failure is possible from teardown but not from from teardown_file
cat ${BATS_FILE_TMPDIR}/zot-log.json
}
function teardown_file() {
zot_stop_all
}

View File

@ -1,4 +1,4 @@
# Note: Intended to be run as "make test-detect-manifest-collision"
# Note: Intended to be run as "make run-blackbox-tests" or "make run-blackbox-ci"
# Makefile target installs & checks all necessary tooling
# Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()
@ -64,7 +64,8 @@ function setup_file() {
}
},
"log": {
"level": "debug"
"level": "debug",
"output": "${BATS_FILE_TMPDIR}/zot.log"
}
}
EOF
@ -72,6 +73,11 @@ EOF
wait_zot_reachable 8080
}
function teardown() {
# conditionally printing on failure is possible from teardown but not from from teardown_file
cat ${BATS_FILE_TMPDIR}/zot.log
}
function teardown_file() {
zot_stop_all
}

View File

@ -45,7 +45,7 @@ function setup_file() {
},
"log": {
"level": "debug",
"output": "/tmp/gc.log"
"output": "${BATS_FILE_TMPDIR}/gc.log"
}
}
EOF
@ -53,6 +53,11 @@ EOF
wait_zot_reachable 8080
}
function teardown() {
# conditionally printing on failure is possible from teardown but not from from teardown_file
cat ${BATS_FILE_TMPDIR}/gc.log
}
function teardown_file() {
zot_stop_all
}

View File

@ -1,4 +1,4 @@
# Note: Intended to be run as "make test-bats-metadata"
# Note: Intended to be run as "make run-blackbox-tests" or "make run-blackbox-ci"
# Makefile target installs & checks all necessary tooling
# Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()
@ -77,7 +77,8 @@ function setup_file() {
}
},
"log": {
"level": "debug"
"level": "debug",
"output": "${BATS_FILE_TMPDIR}/zot.log"
}
}
EOF
@ -86,11 +87,15 @@ EOF
wait_zot_reachable 8080
}
function teardown() {
# conditionally printing on failure is possible from teardown but not from from teardown_file
cat ${BATS_FILE_TMPDIR}/zot.log
}
function teardown_file() {
zot_stop_all
}
@test "push image user policy" {
run skopeo --insecure-policy copy --dest-creds test:test --dest-tls-verify=false \
oci:${TEST_DATA_DIR}/golang:1.18 \

View File

@ -1,4 +1,4 @@
# Note: Intended to be run as "make test-bats-metrics" or "make test-bats-metrics-verbose"
# Note: Intended to be run as "make run-blackbox-tests" or "make run-blackbox-ci"
# Makefile target installs & checks all necessary tooling
# Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()
@ -59,6 +59,11 @@ EOF
}
function teardown() {
# conditionally printing on failure is possible from teardown but not from from teardown_file
cat ${BATS_FILE_TMPDIR}/zot/zot-log.json
}
function teardown_file() {
zot_stop_all
}

View File

@ -1,4 +1,4 @@
# Note: Intended to be run as "make test-push-pull" or "test-push-pull-verbose"
# Note: Intended to be run as "make run-blackbox-tests" or "make run-blackbox-ci"
# Makefile target installs & checks all necessary tooling
# Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()
@ -42,7 +42,8 @@ function setup_file() {
"port": "8080"
},
"log": {
"level": "debug"
"level": "debug",
"output": "${BATS_FILE_TMPDIR}/zot.log"
}
}
EOF
@ -51,6 +52,11 @@ EOF
wait_zot_reachable 8080
}
function teardown() {
# conditionally printing on failure is possible from teardown but not from from teardown_file
cat ${BATS_FILE_TMPDIR}/zot.log
}
function teardown_file() {
zot_stop_all
}

View File

@ -69,7 +69,8 @@ function setup_file() {
}
},
"log":{
"level":"debug"
"level":"debug",
"output": "${BATS_FILE_TMPDIR}/zot.log"
}
}
EOF
@ -77,6 +78,11 @@ EOF
wait_zot_reachable 8080
}
function teardown() {
# conditionally printing on failure is possible from teardown but not from from teardown_file
cat ${BATS_FILE_TMPDIR}/zot.log
}
function teardown_file() {
zot_stop_all
}

View File

@ -1,4 +1,4 @@
# Note: Intended to be run as "make test-push-pull-running-dedupe" or "test-push-pull-running-dedupe-verbose"
# Note: Intended to be run as "make run-blackbox-tests" or "make run-blackbox-dedupe-nightly"
# Makefile target installs & checks all necessary tooling
# Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()
@ -45,7 +45,8 @@ function setup_file() {
"port": "8080"
},
"log": {
"level": "debug"
"level": "debug",
"output": "${BATS_FILE_TMPDIR}/zot.log"
}
}
EOF
@ -54,6 +55,11 @@ EOF
wait_zot_reachable 8080
}
function teardown() {
# conditionally printing on failure is possible from teardown but not from from teardown_file
cat ${BATS_FILE_TMPDIR}/zot.log
}
function teardown_file() {
zot_stop_all
}

View File

@ -1,4 +1,4 @@
# Note: Intended to be run as "make test-bats-referrers"
# Note: Intended to be run as "make run-blackbox-ci"
# Makefile target installs & checks all necessary tooling
# Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()
@ -87,6 +87,11 @@ EOF
curl -X GET http://127.0.0.1:8080/v2/golang/referrers/${MANIFEST_DIGEST}?artifactType=image.artifact/type
}
function teardown() {
# conditionally printing on failure is possible from teardown but not from from teardown_file
cat ${BATS_FILE_TMPDIR}/zot/zot-log.json
}
function teardown_file() {
zot_stop_all
}

View File

@ -1,3 +1,7 @@
# Note: Intended to be run as "make run-blackbox-tests" or "make run-blackbox-dedupe-nightly"
# Makefile target installs & checks all necessary tooling
# Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()
load helpers_cloud
load helpers_wait
@ -81,6 +85,11 @@ EOF
wait_zot_reachable 8080
}
function teardown() {
# conditionally printing on failure is possible from teardown but not from from teardown_file
cat ${BATS_FILE_TMPDIR}/zot/zot-log.json
}
function teardown_file() {
local zot_root_dir=${BATS_FILE_TMPDIR}/zot
zot_stop

View File

@ -1,4 +1,4 @@
# Note: Intended to be run as "make test-bats-scrub" or "make test-bats-scrub-verbose"
# Note: Intended to be run as "make run-blackbox-tests" or "make run-blackbox-ci"
# Makefile target installs & checks all necessary tooling
# Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()
@ -53,11 +53,10 @@ EOF
}
function teardown() {
cat ${BATS_FILE_TMPDIR}/zot/zot-log.json
zot_stop_all
}
@test "blobs/manifest integrity not affected" {
add_test_files
echo ${ZOT_CONFIG_FILE}

View File

@ -1,4 +1,4 @@
# Note: Intended to be run as "make test-bats-sync" or "make test-bats-sync-verbose"
# Note: Intended to be run as "make run-blackbox-tests" or "make run-blackbox-ci"
# Makefile target installs & checks all necessary tooling
# Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()

View File

@ -1,4 +1,4 @@
# Note: Intended to be run as "make test-bats-sync" or "make test-bats-sync-verbose"
# Note: Intended to be run as "make run-blackbox-tests" or "make run-blackbox-ci"
# Makefile target installs & checks all necessary tooling
# Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()

View File

@ -1,4 +1,4 @@
# Note: Intended to be run as "make test-sync-harness" or "make test-sync-harness-verbose"
# Note: Intended to be run as "make run-blackbox-tests" or "make run-blackbox-sync-nightly"
# Makefile target installs & checks all necessary tooling
# Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()

View File

@ -1,4 +1,4 @@
# Note: Intended to be run as "make test-bats-sync" or "make test-bats-sync-verbose"
# Note: Intended to be run as "make run-blackbox-tests" or "make run-blackbox-ci"
# Makefile target installs & checks all necessary tooling
# Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()