ci(tests): add a new workflow for running integration tests against a zot server (#322)

Integration tests will use the latest zot on main
The test data consists of images:
- downloaded from dockerhub
- converted to OCI format
- having all needed annotations
- having a logo as an OCI artifact

Signed-off-by: Andrei Aaron <aaaron@luxoft.com>
This commit is contained in:
Andrei Aaron 2023-03-17 09:54:08 +02:00 committed by GitHub
parent ecff33fe01
commit 2b3058fb14
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 544 additions and 0 deletions

110
.github/workflows/end-to-end-test.yml vendored Normal file
View File

@ -0,0 +1,110 @@
on:
push:
branches:
- main
pull_request:
branches:
- main
release:
types:
- published
name: end-to-end-test
permissions:
contents: read
jobs:
build-and-test:
name: Test zui/zot integration
env:
CI: ""
REGISTRY_HOST: "localhost"
REGISTRY_PORT: "8080"
runs-on: ubuntu-latest
steps:
- name: Checkout zui repository
uses: actions/checkout@v3
with:
fetch-depth: 2
- name: Set up Node.js 16.x
uses: actions/setup-node@v3
with:
node-version: 16.x
cache: 'npm'
- name: Build zui
run: |
cd $GITHUB_WORKSPACE
make install
make build
- name: Install container image tooling
run: |
cd $GITHUB_WORKSPACE
sudo apt-get update
sudo apt-get install libgpgme-dev libassuan-dev libbtrfs-dev libdevmapper-dev pkg-config rpm snapd jq
git clone https://github.com/containers/skopeo -b v1.9.0 $GITHUB_WORKSPACE/src/github.com/containers/skopeo
cd $GITHUB_WORKSPACE/src/github.com/containers/skopeo && make bin/skopeo
chmod +x bin/skopeo
sudo mv bin/skopeo /usr/local/bin/skopeo
which skopeo
skopeo -v
curl -L https://github.com/regclient/regclient/releases/download/v0.4.7/regctl-linux-amd64 -o regctl
chmod +x regctl
sudo mv regctl /usr/local/bin/regctl
which regctl
regctl version
curl -L https://github.com/sigstore/cosign/releases/download/v1.13.0/cosign-linux-amd64 -o cosign
chmod +x cosign
sudo mv cosign /usr/local/bin/cosign
which cosign
cosign version
cd $GITHUB_WORKSPACE
- name: Install go
uses: actions/setup-go@v3
with:
go-version: 1.19.x
- name: Checkout zot repo
uses: actions/checkout@v3
with:
fetch-depth: 2
repository: project-zot/zot
ref: main
path: zot
- name: Build zot
run: |
cd $GITHUB_WORKSPACE/zot
make binary
ls -l bin/
- name: Bringup zot server
run: |
cd $GITHUB_WORKSPACE/zot
mkdir /tmp/zot
./bin/zot-linux-amd64 serve examples/config-ui.json &
while true; do x=0; curl -f http://$REGISTRY_HOST:$REGISTRY_PORT/v2/ || x=1; if [ $x -eq 0 ]; then break; fi; sleep 1; done
- name: Load image test data from cache into a local folder
id: restore-cache
uses: actions/cache@v3
with:
path: tests/data/images
key: image-config-${{ hashFiles('**/tests/data/config.yaml') }}
restore-keys: |
image-config-
- name: Load image test data into zot server
run: |
cd $GITHUB_WORKSPACE
regctl registry set --tls disabled $REGISTRY_HOST:$REGISTRY_PORT
make test-data REGISTRY_HOST=$REGISTRY_HOST REGISTRY_PORT=$REGISTRY_PORT
- name: Run integration tests
run: |
cd $GITHUB_WORKSPACE
make integration-tests REGISTRY_HOST=$REGISTRY_HOST REGISTRY_PORT=$REGISTRY_PORT

1
.gitignore vendored
View File

@ -8,6 +8,7 @@
# testing
/coverage
/tests/data/
# production
/build

View File

@ -1,3 +1,6 @@
REGISTRY_HOST ?= localhost
REGISTRY_PORT ?= 8080
.PHONY: all
all: install audit build
@ -20,3 +23,15 @@ audit:
.PHONY: run
run:
npm start
.PHONY: test-data
test-data:
./tests/scripts/load_test_data.py \
--registry $(REGISTRY_HOST):$(REGISTRY_PORT) \
--data-dir tests/data \
--config-file tests/data/config.yaml \
--metadata-file tests/data/image_metadata.json
.PHONY: integration-tests
integration-tests: # Triggering the tests TBD
cat tests/data/image_metadata.json | jq

116
tests/data/config.yaml Normal file
View File

@ -0,0 +1,116 @@
images:
- name: alpine
tags:
- "3.17"
- "3.17.2"
- "3.17.1"
- "3.16"
- "3.16.4"
- "3.16.3"
- "3.16.2"
- "3.16.1"
- "3.15"
- "3.15.7"
- "3.15.6"
- "3.15.5"
- "3.15.4"
- "3.15.3"
- "3.15.2"
- "3.15.1"
- "3.14"
- "3.14.9"
multiarch: "all"
- name: ubuntu
tags:
- "18.04"
- "bionic-20230301"
- "bionic"
- "22.04"
- "jammy-20230301"
- "jammy"
- "latest"
multiarch: ""
- name: debian
tags:
- "bullseye-slim"
- "bullseye-20230227-slim"
- "11.6-slim"
- "11-slim"
multiarch: ""
- name: centos # Not supported since 2020
tags:
- "centos7"
- "7"
- "centos7.9.2009"
- "7.9.2009"
multiarch: ""
- name: node
tags:
- "18-alpine3.16" # Alpine here an below
- "18.15-alpine3.16"
- "18.15.0-alpine3.16"
- "lts-alpine3.16"
- "18-alpine"
- "18-alpine3.17"
- "18.15-alpine"
- "18.15-alpine3.17"
- "18.15.0-alpine3.17"
- "lts-alpine3.17"
- "18-bullseye-slim" # Debian here and below
- "18-slim"
- "18.15-bullseye-slim"
- "18.15.0-bullseye-slim"
- "18.15.0-slim"
multiarch: "all"
- name: nginx
tags:
- "1.23.3" # debian:bullseye-slim based here and below
- "mainline"
- "1.23"
- "1.23.3-alpine" # Depends on alpine slim tags
- "mainline-alpine"
- "1.23-alpine"
- "1.23.3-alpine-slim" # Based on alpine
- "mainline-alpine-slim"
- "1.23-alpine-slim"
multiarch: ""
- name: python
tags:
- "3.8.16-alpine3.17"
- "3.8.16-alpine3.16"
- "3.8.16-bullseye"
multiarch: ""
- name: golang
tags:
- "1.20.2-bullseye"
- "1.20.2-alpine3.17"
multiarch: ""
- name: perl
tags:
- "5.36.0-slim" # debian:bullseye-slim based
multiarch: ""
- name: ruby
tags:
- "3.2.1-slim-bullseye" # debian:bullseye-slim based
multiarch: ""
- name: busybox
tags:
- "1.36.0" # From scratch
- "1.35.0"
multiarch: ""
- name: httpd
tags:
- "2.4.56-alpine3.17"
multiarch: ""
- name: hello-world # From scratch
tags:
- "linux"
multiarch: ""
- name: bash
tags:
- "5.2.15-alpine3.16"
multiarch: ""
- name: rust
tags:
- "1.68-slim-bullseye"
multiarch: ""

137
tests/scripts/load_test_data.py Executable file
View File

@ -0,0 +1,137 @@
#!/usr/bin/env python3
import argparse
import json
import logging
import os
import subprocess
import sys
import tempfile
import yaml
def init_logger(debug=False):
logger = logging.getLogger(__name__)
if debug:
logger.setLevel(logging.DEBUG)
else:
logger.setLevel(logging.INFO)
# log format
log_fmt = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
date_fmt = "%Y-%m-%d %H:%M:%S"
formatter = logging.Formatter(log_fmt, date_fmt)
# create streamHandler and set log fmt
stream_handler = logging.StreamHandler()
stream_handler.setFormatter(formatter)
# add the streamHandler to logger
logger.addHandler(stream_handler)
return logger
def parse_args():
p = argparse.ArgumentParser()
p.add_argument('-r', '--registry', default='localhost:8080', help='Registry address')
p.add_argument('-u', '--username', default="", help='registry username')
p.add_argument('-p', '--password', default="", help='registry password')
p.add_argument('-c', '--cosign-password', default="", help='cosign key password')
p.add_argument('-d', '--debug', action='store_true', help='enable debug logs')
p.add_argument('-f', '--config-file', default="config.yaml", help='config file containing information about images to upload')
p.add_argument('-m', '--metadata-file', default="image_metadata.json", help='file containing metadata on uploaded images')
p.add_argument('--data-dir', default="", help='location where to store image related data')
return p.parse_args()
def fetch_tags(logger, image_name):
cmd = "skopeo list-tags docker://docker.io/{}".format(image_name)
logger.info("running command: '{}'".format(cmd))
result = subprocess.run(cmd, capture_output=True, shell=True)
if result.returncode != 0:
logger.error("running command `{}` exited with code: {}".format(cmd, str(result.returncode)))
logger.error(result.stderr)
sys.exit(1)
return json.loads(result.stdout)["Tags"]
def pull_modify_push_image(logger, registry, image_name, tag, cosign_password,
multiarch, username, password, debug, data_dir):
logger.info("image '{}:{}' will be processed and pushed".format(image_name, tag))
image_update_script_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "pull_update_push_image.sh")
with tempfile.TemporaryDirectory() as meta_dir_name:
metafile = '{}_{}_metadata.json'.format(image_name, tag)
metafile = os.path.join(meta_dir_name, metafile)
cmd = [image_update_script_path, "-r", registry, "-i", image_name, "-t", tag, "-c", cosign_password,
"-f", metafile, "-m", multiarch, "-u", username, "-p", password, "--data-dir", data_dir]
if debug:
cmd.append("-d")
logger.info("running command: '{}'".format(" ".join(cmd)))
result = subprocess.run(cmd, stderr=sys.stderr, stdout=sys.stdout)
if result.returncode != 0:
logger.error("pushing image `{}:{}` exited with code: ".format(image_name, tag) + str(result.returncode))
sys.exit(1)
with open(metafile) as f:
image_metadata = json.load(f)
image_metadata[image_name][tag]["multiarch"] = multiarch
return image_metadata
def main():
args = parse_args()
registry = args.registry
username = args.username
password = args.password
cosign_password = args.cosign_password
config_file = args.config_file
debug = args.debug
metadata_file = args.metadata_file
data_dir= args.data_dir
logger = init_logger(debug)
with open(config_file, "r") as f:
config = yaml.load(f, Loader=yaml.SafeLoader)
metadata = {}
for image in config["images"]:
image_name = image["name"]
multiarch = image["multiarch"]
if not multiarch:
multiarch = ""
actual_tags = fetch_tags(logger, image_name)
expected_tags = image["tags"]
logger.debug("image '{}' has the following tags specified in the config file: '{}'".format(image_name, ",".join(expected_tags)))
logger.debug("image '{}' has the following tags on source registry: '{}'".format(image_name, ",".join(actual_tags)))
for tag in expected_tags:
found = False
for actual_tag in actual_tags:
if actual_tag == tag:
found = True
break
if not found:
logger.error("image '{}:{}' not found".format(image_name, tag))
sys.exit(1)
for tag in expected_tags:
image_metadata = pull_modify_push_image(logger, registry, image_name, tag, cosign_password, multiarch, username, password, debug, data_dir)
metadata.setdefault(image_name, {})
metadata[image_name][tag] = image_metadata[image_name][tag]
with open(metadata_file, "w") as f:
json.dump(metadata, f, indent=2)
logger.info("Done loading images, see more details in '{}'".format(metadata_file))
if __name__ == "__main__":
main()

View File

@ -0,0 +1,165 @@
#!/bin/bash
registry=""
image=""
tag=""
cosign_password=""
metafile=""
multiarch=""
username=""
username=""
debug=0
data_dir=$(pwd)
options=$(getopt -o dr:i:t:u:p:c:m:f: -l debug,registry:,image:,tag:,username:,password:,cosign-password:,multiarch:,file:,data-dir: -- "$@")
if [ $? -ne 0 ]; then
usage $0
exit 0
fi
eval set -- "$options"
while :; do
case "$1" in
-r|--registry) registry=$2; shift 2;;
-i|--image) image=$2; shift 2;;
-t|--tag) tag=$2; shift 2;;
-u|--username) username=$2; shift 2;;
-p|--password) username=$2; shift 2;;
-c|--cosign-password) cosign_password=$2; shift 2;;
-m|--multiarch) multiarch=$2; shift 2;;
-f|--file) metafile=$2; shift 2;;
--data-dir) data_dir=$2; shift 2;;
-d|--debug) debug=1; shift 1;;
--) shift 1; break;;
*) usage $0; exit 1;;
esac
done
if [ ${debug} -eq 1 ]; then
set -x
fi
images_dir=${data_dir}/images
docker_docs_dir=${data_dir}/docs
cosign_key_path=${data_dir}/cosign.key
function verify_prerequisites {
mkdir -p ${data_dir}
if [ ! command -v regctl ] &>/dev/null; then
echo "you need to install regctl as a prerequisite" >&3
return 1
fi
if [ ! command -v skopeo ] &>/dev/null; then
echo "you need to install skopeo as a prerequisite" >&3
return 1
fi
if [ ! command -v cosign ] &>/dev/null; then
echo "you need to install cosign as a prerequisite" >&3
return 1
fi
if [ ! command -v jq ] &>/dev/null; then
echo "you need to install jq as a prerequisite" >&3
return 1
fi
if [ ! -f "${cosign_key_path}" ]; then
COSIGN_PASSWORD=${cosign_password} cosign generate-key-pair
key_dir=$(dirname ${cosign_key_path})
mv cosign.key ${cosign_key_path}
mv cosign.pub ${key_dir}
fi
# pull docker docs repo
if [ ! -d ${docker_docs_dir} ]
then
git clone https://github.com/docker-library/docs.git ${docker_docs_dir}
fi
return 0
}
verify_prerequisites
repo=$(cat ${docker_docs_dir}/${image}/github-repo)
description="$(cat ${docker_docs_dir}/${image}/README-short.txt)"
license="$(cat ${docker_docs_dir}/${image}/license.md)"
vendor="$(cat ${docker_docs_dir}/${image}/maintainer.md)"
logo=$(base64 -w 0 ${docker_docs_dir}/${image}/logo.png)
echo ${repo}
sed -i "s|%%GITHUB-REPO%%|${repo}|g" ${docker_docs_dir}/${image}/maintainer.md
sed -i "s|%%IMAGE%%|${image}|g" ${docker_docs_dir}/${image}/content.md
doc=$(cat ${docker_docs_dir}/${image}/content.md)
local_image_ref_skopeo=oci:${images_dir}:${image}-${tag}
local_image_ref_regtl=ocidir://${images_dir}:${image}-${tag}
remote_src_image_ref=docker://${image}:${tag}
remote_dest_image_ref=${registry}/${image}:${tag}
multiarch_arg=""
if [ ! -z "${multiarch}" ]; then
multiarch_arg="--multi-arch=${multiarch}"
fi
# Verify if image is already present in local oci layout
skopeo inspect ${local_image_ref_skopeo}
if [ $? -eq 0 ]; then
echo "Image ${local_image_ref_skopeo} found locally"
else
echo "Image ${local_image_ref_skopeo} will be copied"
skopeo --insecure-policy copy --format=oci ${multiarch_arg} ${remote_src_image_ref} ${local_image_ref_skopeo}
if [ $? -ne 0 ]; then
exit 1
fi
fi
# Mofify image in local oci layout and update the old reference to point to the new index
regctl image mod --replace --annotation org.opencontainers.image.title=${image} ${local_image_ref_regtl}
regctl image mod --replace --annotation org.opencontainers.image.description="${description}" ${local_image_ref_regtl}
regctl image mod --replace --annotation org.opencontainers.image.url=${repo} ${local_image_ref_regtl}
regctl image mod --replace --annotation org.opencontainers.image.source=${repo} ${local_image_ref_regtl}
regctl image mod --replace --annotation org.opencontainers.image.licenses="${license}" ${local_image_ref_regtl}
regctl image mod --replace --annotation org.opencontainers.image.vendor="${vendor}" ${local_image_ref_regtl}
regctl image mod --replace --annotation org.opencontainers.image.documentation="${description}" ${local_image_ref_regtl}
credentials_args=""
if [ ! -z "${username}" ]; then
credentials_args="--dest-creds ${username}:${username}"
fi
# Upload image to target registry
skopeo copy --dest-tls-verify=false ${multiarch_arg} ${credentials_args} ${local_image_ref_skopeo} docker://${remote_dest_image_ref}
if [ $? -ne 0 ]; then
exit 1
fi
# Upload image logo as image media type
regctl artifact put --annotation artifact.type=com.zot.logo.image --annotation format=oci \
--artifact-type "application/vnd.zot.logo.v1" --subject ${remote_dest_image_ref} ${remote_dest_image_ref}-logo-image << EOF
${logo}
EOF
if [ $? -ne 0 ]; then
exit 1
fi
# Sign new updated image
COSIGN_PASSWORD=${cosign_password} cosign sign ${remote_dest_image_ref} --key ${cosign_key_path} --allow-insecure-registry
if [ $? -ne 0 ]; then
exit 1
fi
details=$(jq -n \
--arg org.opencontainers.image.title "${image}" \
--arg org.opencontainers.image.description " $description" \
--arg org.opencontainers.image.url "${repo}" \
--arg org.opencontainers.image.source "${repo}" \
--arg org.opencontainers.image.licenses "${license}" \
--arg org.opencontainers.image.vendor "${vendor}" \
--arg org.opencontainers.image.documentation "${description}" \
'$ARGS.named'
)
jq -n --arg image "${image}" --arg tag "${tag}" --argjson details "${details}" '.[$image][$tag]=$details' > ${metafile}