From 2b3058fb1414a254440fba2e4f6a9de413714374 Mon Sep 17 00:00:00 2001
From: Andrei Aaron <aaaron@luxoft.com>
Date: Fri, 17 Mar 2023 09:54:08 +0200
Subject: [PATCH] ci(tests): add a new workflow for running integration tests
 against a zot server (#322)

Integration tests will use the latest zot on main
The test data consists of images:
- downloaded from dockerhub
- converted to OCI format
- having all needed annotations
- having a logo as an OCI artifact

Signed-off-by: Andrei Aaron <aaaron@luxoft.com>
---
 .github/workflows/end-to-end-test.yml   | 110 ++++++++++++++++
 .gitignore                              |   1 +
 Makefile                                |  15 +++
 tests/data/config.yaml                  | 116 +++++++++++++++++
 tests/scripts/load_test_data.py         | 137 ++++++++++++++++++++
 tests/scripts/pull_update_push_image.sh | 165 ++++++++++++++++++++++++
 6 files changed, 544 insertions(+)
 create mode 100644 .github/workflows/end-to-end-test.yml
 create mode 100644 tests/data/config.yaml
 create mode 100755 tests/scripts/load_test_data.py
 create mode 100755 tests/scripts/pull_update_push_image.sh

diff --git a/.github/workflows/end-to-end-test.yml b/.github/workflows/end-to-end-test.yml
new file mode 100644
index 00000000..4250dd74
--- /dev/null
+++ b/.github/workflows/end-to-end-test.yml
@@ -0,0 +1,110 @@
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+  release:
+    types:
+      - published
+name: end-to-end-test
+
+permissions:
+  contents: read
+
+jobs:
+  build-and-test:
+    name: Test zui/zot integration
+    env:
+      CI: ""
+      REGISTRY_HOST: "localhost"
+      REGISTRY_PORT: "8080"
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout zui repository
+      uses: actions/checkout@v3
+      with:
+        fetch-depth: 2
+
+    - name: Set up Node.js 16.x
+      uses: actions/setup-node@v3
+      with:
+        node-version: 16.x
+        cache: 'npm'
+
+    - name: Build zui
+      run:  |
+        cd $GITHUB_WORKSPACE
+        make install
+        make build
+
+    - name: Install container image tooling
+      run: |
+        cd $GITHUB_WORKSPACE
+        sudo apt-get update
+        sudo apt-get install libgpgme-dev libassuan-dev libbtrfs-dev libdevmapper-dev pkg-config rpm snapd jq
+        git clone https://github.com/containers/skopeo -b v1.9.0 $GITHUB_WORKSPACE/src/github.com/containers/skopeo
+        cd $GITHUB_WORKSPACE/src/github.com/containers/skopeo && make bin/skopeo
+        chmod +x bin/skopeo
+        sudo mv bin/skopeo /usr/local/bin/skopeo
+        which skopeo
+        skopeo -v
+        curl -L https://github.com/regclient/regclient/releases/download/v0.4.7/regctl-linux-amd64 -o regctl
+        chmod +x regctl
+        sudo mv regctl /usr/local/bin/regctl
+        which regctl
+        regctl version
+        curl -L https://github.com/sigstore/cosign/releases/download/v1.13.0/cosign-linux-amd64 -o cosign
+        chmod +x cosign
+        sudo mv cosign /usr/local/bin/cosign
+        which cosign
+        cosign version
+        cd $GITHUB_WORKSPACE
+
+    - name: Install go
+      uses: actions/setup-go@v3
+      with:
+        go-version: 1.19.x
+
+    - name: Checkout zot repo
+      uses: actions/checkout@v3
+      with:
+        fetch-depth: 2
+        repository: project-zot/zot
+        ref: main
+        path: zot
+
+    - name: Build zot
+      run: |
+        cd $GITHUB_WORKSPACE/zot
+        make binary
+        ls -l bin/
+
+    - name: Bringup zot server
+      run: |
+        cd $GITHUB_WORKSPACE/zot
+        mkdir /tmp/zot
+        ./bin/zot-linux-amd64 serve examples/config-ui.json &
+        while true; do x=0; curl -f http://$REGISTRY_HOST:$REGISTRY_PORT/v2/ || x=1; if [ $x -eq 0 ]; then break; fi; sleep 1; done
+
+    - name: Load image test data from cache into a local folder
+      id: restore-cache
+      uses: actions/cache@v3
+      with:
+        path: tests/data/images
+        key: image-config-${{ hashFiles('**/tests/data/config.yaml') }}
+        restore-keys: |
+          image-config-
+
+    - name: Load image test data into zot server
+      run: |
+        cd $GITHUB_WORKSPACE
+        regctl registry set --tls disabled $REGISTRY_HOST:$REGISTRY_PORT
+        make test-data REGISTRY_HOST=$REGISTRY_HOST REGISTRY_PORT=$REGISTRY_PORT
+
+    - name: Run integration tests
+      run: |
+        cd $GITHUB_WORKSPACE
+        make integration-tests REGISTRY_HOST=$REGISTRY_HOST REGISTRY_PORT=$REGISTRY_PORT
diff --git a/.gitignore b/.gitignore
index 711323b8..1241da04 100644
--- a/.gitignore
+++ b/.gitignore
@@ -8,6 +8,7 @@
 
 # testing
 /coverage
+/tests/data/
 
 # production
 /build
diff --git a/Makefile b/Makefile
index ae9148cd..947528d4 100644
--- a/Makefile
+++ b/Makefile
@@ -1,3 +1,6 @@
+REGISTRY_HOST ?= localhost
+REGISTRY_PORT ?= 8080
+
 .PHONY: all
 all: install audit build
 
@@ -20,3 +23,15 @@ audit:
 .PHONY: run
 run:
 	npm start
+
+.PHONY: test-data
+test-data:
+	./tests/scripts/load_test_data.py \
+		--registry $(REGISTRY_HOST):$(REGISTRY_PORT) \
+		--data-dir tests/data \
+		--config-file tests/data/config.yaml \
+		--metadata-file tests/data/image_metadata.json
+
+.PHONY: integration-tests
+integration-tests: # Triggering the tests TBD
+	cat tests/data/image_metadata.json | jq
diff --git a/tests/data/config.yaml b/tests/data/config.yaml
new file mode 100644
index 00000000..15bd1790
--- /dev/null
+++ b/tests/data/config.yaml
@@ -0,0 +1,116 @@
+images:
+  - name: alpine
+    tags:
+      - "3.17"
+      - "3.17.2"
+      - "3.17.1"
+      - "3.16"
+      - "3.16.4"
+      - "3.16.3"
+      - "3.16.2"
+      - "3.16.1"
+      - "3.15"
+      - "3.15.7"
+      - "3.15.6"
+      - "3.15.5"
+      - "3.15.4"
+      - "3.15.3"
+      - "3.15.2"
+      - "3.15.1"
+      - "3.14"
+      - "3.14.9"
+    multiarch: "all"
+  - name: ubuntu
+    tags:
+      - "18.04"
+      - "bionic-20230301"
+      - "bionic"
+      - "22.04"
+      - "jammy-20230301"
+      - "jammy"
+      - "latest"
+    multiarch: ""
+  - name: debian
+    tags:
+      - "bullseye-slim"
+      - "bullseye-20230227-slim"
+      - "11.6-slim"
+      - "11-slim"
+    multiarch: ""
+  - name: centos # Not supported since 2020
+    tags:
+      - "centos7"
+      - "7"
+      - "centos7.9.2009"
+      - "7.9.2009"
+    multiarch: ""
+  - name: node
+    tags:
+      - "18-alpine3.16" # Alpine here an below
+      - "18.15-alpine3.16"
+      - "18.15.0-alpine3.16"
+      - "lts-alpine3.16"
+      - "18-alpine"
+      - "18-alpine3.17"
+      - "18.15-alpine"
+      - "18.15-alpine3.17"
+      - "18.15.0-alpine3.17"
+      - "lts-alpine3.17"
+      - "18-bullseye-slim" # Debian here and below
+      - "18-slim"
+      - "18.15-bullseye-slim"
+      - "18.15.0-bullseye-slim"
+      - "18.15.0-slim"
+    multiarch: "all"
+  - name: nginx
+    tags:
+      - "1.23.3" # debian:bullseye-slim based here and below
+      - "mainline"
+      - "1.23"
+      - "1.23.3-alpine" # Depends on alpine slim tags
+      - "mainline-alpine"
+      - "1.23-alpine"
+      - "1.23.3-alpine-slim" # Based on alpine
+      - "mainline-alpine-slim"
+      - "1.23-alpine-slim"
+    multiarch: ""
+  - name: python
+    tags:
+      - "3.8.16-alpine3.17"
+      - "3.8.16-alpine3.16"
+      - "3.8.16-bullseye"
+    multiarch: ""
+  - name: golang
+    tags:
+      - "1.20.2-bullseye"
+      - "1.20.2-alpine3.17"
+    multiarch: ""
+  - name: perl
+    tags:
+      - "5.36.0-slim" # debian:bullseye-slim based
+    multiarch: ""
+  - name: ruby
+    tags:
+      - "3.2.1-slim-bullseye" # debian:bullseye-slim based
+    multiarch: ""
+  - name: busybox
+    tags:
+      - "1.36.0" # From scratch
+      - "1.35.0"
+    multiarch: ""
+  - name: httpd
+    tags:
+      - "2.4.56-alpine3.17"
+    multiarch: ""
+  - name: hello-world # From scratch
+    tags:
+      - "linux"
+    multiarch: ""
+  - name: bash
+    tags:
+      - "5.2.15-alpine3.16"
+    multiarch: ""
+  - name: rust
+    tags:
+      - "1.68-slim-bullseye"
+    multiarch: ""
diff --git a/tests/scripts/load_test_data.py b/tests/scripts/load_test_data.py
new file mode 100755
index 00000000..e0840ac2
--- /dev/null
+++ b/tests/scripts/load_test_data.py
@@ -0,0 +1,137 @@
+#!/usr/bin/env python3
+import argparse
+import json
+import logging
+import os
+import subprocess
+import sys
+import tempfile
+import yaml
+
+def init_logger(debug=False):
+    logger = logging.getLogger(__name__)
+    if debug:
+        logger.setLevel(logging.DEBUG)
+    else:
+        logger.setLevel(logging.INFO)
+    # log format
+    log_fmt = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+    date_fmt = "%Y-%m-%d %H:%M:%S"
+    formatter = logging.Formatter(log_fmt, date_fmt)
+    # create streamHandler and set log fmt
+    stream_handler = logging.StreamHandler()
+    stream_handler.setFormatter(formatter)
+    # add the streamHandler to logger
+    logger.addHandler(stream_handler)
+    return logger
+
+def parse_args():
+    p = argparse.ArgumentParser()
+    p.add_argument('-r', '--registry', default='localhost:8080', help='Registry address')
+    p.add_argument('-u', '--username', default="", help='registry username')
+    p.add_argument('-p', '--password', default="", help='registry password')
+    p.add_argument('-c', '--cosign-password', default="", help='cosign key password')
+    p.add_argument('-d', '--debug', action='store_true', help='enable debug logs')
+    p.add_argument('-f', '--config-file', default="config.yaml", help='config file containing information about images to upload')
+    p.add_argument('-m', '--metadata-file', default="image_metadata.json", help='file containing metadata on uploaded images')
+    p.add_argument('--data-dir', default="", help='location where to store image related data')
+
+    return p.parse_args()
+
+def fetch_tags(logger, image_name):
+    cmd = "skopeo list-tags docker://docker.io/{}".format(image_name)
+    logger.info("running command: '{}'".format(cmd))
+
+    result = subprocess.run(cmd, capture_output=True, shell=True)
+    if result.returncode != 0:
+        logger.error("running command `{}` exited with code: {}".format(cmd, str(result.returncode)))
+        logger.error(result.stderr)
+        sys.exit(1)
+
+    return json.loads(result.stdout)["Tags"]
+
+def pull_modify_push_image(logger, registry, image_name, tag, cosign_password,
+                           multiarch, username, password, debug, data_dir):
+    logger.info("image '{}:{}' will be processed and pushed".format(image_name, tag))
+
+    image_update_script_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "pull_update_push_image.sh")
+
+    with tempfile.TemporaryDirectory() as meta_dir_name:
+        metafile = '{}_{}_metadata.json'.format(image_name, tag)
+        metafile = os.path.join(meta_dir_name, metafile)
+
+        cmd = [image_update_script_path, "-r", registry, "-i", image_name, "-t", tag, "-c", cosign_password,
+               "-f", metafile, "-m", multiarch, "-u", username, "-p", password, "--data-dir", data_dir]
+
+        if debug:
+            cmd.append("-d")
+
+        logger.info("running command: '{}'".format(" ".join(cmd)))
+        result = subprocess.run(cmd, stderr=sys.stderr, stdout=sys.stdout)
+        if result.returncode != 0:
+            logger.error("pushing image `{}:{}` exited with code: ".format(image_name, tag) + str(result.returncode))
+            sys.exit(1)
+
+        with open(metafile) as f:
+            image_metadata = json.load(f)
+            image_metadata[image_name][tag]["multiarch"] = multiarch
+
+    return image_metadata
+
+def main():
+    args = parse_args()
+
+    registry = args.registry
+    username = args.username
+    password = args.password
+    cosign_password = args.cosign_password
+    config_file = args.config_file
+    debug = args.debug
+    metadata_file = args.metadata_file
+    data_dir= args.data_dir
+
+    logger = init_logger(debug)
+
+    with open(config_file, "r") as f:
+        config = yaml.load(f, Loader=yaml.SafeLoader)
+ 
+    metadata = {}
+
+    for image in config["images"]:
+        image_name = image["name"]
+
+        multiarch = image["multiarch"]
+        if not multiarch:
+            multiarch = ""
+
+        actual_tags = fetch_tags(logger, image_name)
+        expected_tags = image["tags"]
+
+        logger.debug("image '{}' has the following tags specified in the config file: '{}'".format(image_name, ",".join(expected_tags)))
+        logger.debug("image '{}' has the following tags on source registry: '{}'".format(image_name, ",".join(actual_tags)))
+
+        for tag in expected_tags:
+            found = False
+
+            for actual_tag in actual_tags:
+                if actual_tag == tag:
+                    found = True
+                    break
+
+            if not found:
+                logger.error("image '{}:{}' not found".format(image_name, tag))
+                sys.exit(1)
+
+        for tag in expected_tags:
+            image_metadata = pull_modify_push_image(logger, registry, image_name, tag, cosign_password, multiarch, username, password, debug, data_dir)
+
+            metadata.setdefault(image_name, {})
+            metadata[image_name][tag] = image_metadata[image_name][tag]
+
+    with open(metadata_file, "w") as f:
+        json.dump(metadata, f, indent=2)
+
+    logger.info("Done loading images, see more details in '{}'".format(metadata_file))
+
+if __name__ == "__main__":
+    main()
diff --git a/tests/scripts/pull_update_push_image.sh b/tests/scripts/pull_update_push_image.sh
new file mode 100755
index 00000000..021e4e62
--- /dev/null
+++ b/tests/scripts/pull_update_push_image.sh
@@ -0,0 +1,165 @@
+#!/bin/bash
+
+registry=""
+image=""
+tag=""
+cosign_password=""
+metafile=""
+multiarch=""
+username=""
+username=""
+debug=0
+data_dir=$(pwd)
+
+options=$(getopt -o dr:i:t:u:p:c:m:f: -l debug,registry:,image:,tag:,username:,password:,cosign-password:,multiarch:,file:,data-dir: -- "$@")
+if [ $? -ne 0 ]; then
+   usage $0
+   exit 0
+fi
+
+eval set -- "$options"
+while :; do
+    case "$1" in
+        -r|--registry)  registry=$2; shift 2;;
+        -i|--image)   image=$2; shift 2;;
+        -t|--tag)   tag=$2; shift 2;;
+        -u|--username) username=$2; shift 2;;
+        -p|--password) username=$2; shift 2;;
+        -c|--cosign-password) cosign_password=$2; shift 2;;
+        -m|--multiarch) multiarch=$2; shift 2;;
+        -f|--file) metafile=$2; shift 2;;
+        --data-dir) data_dir=$2; shift 2;;
+        -d|--debug) debug=1; shift 1;;
+        --)         shift 1; break;;
+        *)          usage $0; exit 1;;
+    esac
+done
+
+if [ ${debug} -eq 1 ]; then
+    set -x
+fi
+
+images_dir=${data_dir}/images
+docker_docs_dir=${data_dir}/docs
+cosign_key_path=${data_dir}/cosign.key
+
+function verify_prerequisites {
+    mkdir -p ${data_dir}
+
+    if [ ! command -v regctl ] &>/dev/null; then
+        echo "you need to install regctl as a prerequisite" >&3
+        return 1
+    fi
+
+    if [ ! command -v skopeo ] &>/dev/null; then
+        echo "you need to install skopeo as a prerequisite" >&3
+        return 1
+    fi
+
+    if [ ! command -v cosign ] &>/dev/null; then
+        echo "you need to install cosign as a prerequisite" >&3
+        return 1
+    fi
+
+    if [ ! command -v jq ] &>/dev/null; then
+        echo "you need to install jq as a prerequisite" >&3
+        return 1
+    fi
+
+    if [ ! -f "${cosign_key_path}" ]; then
+        COSIGN_PASSWORD=${cosign_password} cosign generate-key-pair
+        key_dir=$(dirname ${cosign_key_path})
+        mv cosign.key ${cosign_key_path}
+        mv cosign.pub ${key_dir}
+    fi
+
+    # pull docker docs repo
+    if [ ! -d ${docker_docs_dir} ]
+    then
+        git clone https://github.com/docker-library/docs.git ${docker_docs_dir}
+    fi
+
+    return 0
+}
+
+verify_prerequisites
+
+repo=$(cat ${docker_docs_dir}/${image}/github-repo)
+description="$(cat ${docker_docs_dir}/${image}/README-short.txt)"
+license="$(cat ${docker_docs_dir}/${image}/license.md)"
+vendor="$(cat ${docker_docs_dir}/${image}/maintainer.md)"
+logo=$(base64 -w 0 ${docker_docs_dir}/${image}/logo.png)
+echo ${repo}
+sed -i "s|%%GITHUB-REPO%%|${repo}|g" ${docker_docs_dir}/${image}/maintainer.md
+sed -i "s|%%IMAGE%%|${image}|g" ${docker_docs_dir}/${image}/content.md
+doc=$(cat ${docker_docs_dir}/${image}/content.md)
+
+local_image_ref_skopeo=oci:${images_dir}:${image}-${tag}
+local_image_ref_regtl=ocidir://${images_dir}:${image}-${tag}
+remote_src_image_ref=docker://${image}:${tag}
+remote_dest_image_ref=${registry}/${image}:${tag}
+
+multiarch_arg=""
+if [ ! -z "${multiarch}" ]; then
+    multiarch_arg="--multi-arch=${multiarch}"
+fi
+
+# Verify if image is already present in local oci layout
+skopeo inspect ${local_image_ref_skopeo}
+if [ $? -eq 0 ]; then
+    echo "Image ${local_image_ref_skopeo} found locally"
+else
+    echo "Image ${local_image_ref_skopeo} will be copied"
+    skopeo --insecure-policy copy --format=oci ${multiarch_arg} ${remote_src_image_ref} ${local_image_ref_skopeo}
+    if [ $? -ne 0 ]; then
+        exit 1
+    fi
+fi
+
+# Mofify image in local oci layout and update the old reference to point to the new index
+regctl image mod --replace --annotation org.opencontainers.image.title=${image} ${local_image_ref_regtl}
+regctl image mod --replace --annotation org.opencontainers.image.description="${description}" ${local_image_ref_regtl}
+regctl image mod --replace --annotation org.opencontainers.image.url=${repo} ${local_image_ref_regtl}
+regctl image mod --replace --annotation org.opencontainers.image.source=${repo} ${local_image_ref_regtl}
+regctl image mod --replace --annotation org.opencontainers.image.licenses="${license}" ${local_image_ref_regtl}
+regctl image mod --replace --annotation org.opencontainers.image.vendor="${vendor}" ${local_image_ref_regtl}
+regctl image mod --replace --annotation org.opencontainers.image.documentation="${description}" ${local_image_ref_regtl}
+
+credentials_args=""
+if [ ! -z "${username}" ]; then
+    credentials_args="--dest-creds ${username}:${username}"
+fi
+
+# Upload image to target registry
+skopeo copy --dest-tls-verify=false ${multiarch_arg} ${credentials_args} ${local_image_ref_skopeo} docker://${remote_dest_image_ref}
+if [ $? -ne 0 ]; then
+    exit 1
+fi
+
+# Upload image logo as image media type
+regctl artifact put --annotation artifact.type=com.zot.logo.image --annotation format=oci \
+    --artifact-type "application/vnd.zot.logo.v1" --subject ${remote_dest_image_ref} ${remote_dest_image_ref}-logo-image << EOF
+${logo}
+EOF
+if [ $? -ne 0 ]; then
+    exit 1
+fi
+
+# Sign new updated image
+COSIGN_PASSWORD=${cosign_password} cosign sign ${remote_dest_image_ref} --key ${cosign_key_path} --allow-insecure-registry
+if [ $? -ne 0 ]; then
+    exit 1
+fi
+
+details=$(jq -n \
+    --arg org.opencontainers.image.title "${image}" \
+    --arg org.opencontainers.image.description " $description" \
+    --arg org.opencontainers.image.url "${repo}" \
+    --arg org.opencontainers.image.source "${repo}" \
+    --arg org.opencontainers.image.licenses "${license}" \
+    --arg org.opencontainers.image.vendor "${vendor}" \
+    --arg org.opencontainers.image.documentation "${description}" \
+    '$ARGS.named'
+)
+
+jq -n --arg image "${image}" --arg tag "${tag}"  --argjson details "${details}" '.[$image][$tag]=$details' > ${metafile}