fix: quick metrics setup in kind with prometheus (#1897)

Signed-off-by: Alexei Dodon <adodon@cisco.com>
2023-10-16 11:51:00 +03:00 · 2023-10-16 11:51:00 +03:00 · 12227b4484
commit 12227b4484
parent fc2380b57b
9 changed files with 118 additions and 139 deletions
--- a/19
+++ b/19
@ -259,7 +259,7 @@ swagger:
 .PHONY: update-licenses
 # note: for predictable output of below sort command we use locale LC_ALL=C
 update-licenses: LC_ALL=C
-update-licenses:
+update-licenses: check-linux
 	@echo "Detecting and updating licenses ... please be patient!"
 	go install github.com/google/go-licenses@latest
 	$(shell echo "Module | License URL | License" > THIRD-PARTY-LICENSES.md; echo "---|---|---" >> THIRD-PARTY-LICENSES.md; for i in $$(go list -m all  | awk '{print $$1}'); do l=$$(go-licenses csv $$i 2>/dev/null); if [ $$? -ne 0 ]; then continue; fi; echo $$l | tr \, \| | tr ' ' '\n'; done | sort -u >> THIRD-PARTY-LICENSES.md)
@ -362,6 +362,23 @@ run-container:
 	${CONTAINER_RUNTIME} run --rm --security-opt label=disable -v $$(pwd):/go/src/github.com/project-zot/zot \
 		zot-build:latest

+.PHONY: binary-minimal-container
+binary-minimal-container:
+	${CONTAINER_RUNTIME} build ${BUILD_ARGS} -f build/Dockerfile-minimal -t zot-minimal:latest .
+
+.PHONY: run-minimal-container
+run-minimal-container:
+	${CONTAINER_RUNTIME} run --rm --security-opt label=disable -v $$(pwd):/go/src/github.com/project-zot/zot \
+		zot-minimal:latest
+
+.PHONY: binary-exporter-container
+binary-exporter-container:
+	${CONTAINER_RUNTIME} build ${BUILD_ARGS} -f build/Dockerfile-zxp -t zxp:latest .
+
+.PHONY: run-exporter-container
+run-exporter-container:
+	${CONTAINER_RUNTIME} run --rm --security-opt label=disable zxp:latest
+
 .PHONY: oci-image
 oci-image: $(STACKER)
 	${STACKER} --debug build \
--- a/examples/metrics/Dockerfile
+++ b/examples/metrics/Dockerfile
@ -1,40 +0,0 @@
-# ---
-# Stage 1: Install certs, build binary, create default config file
-# ---
-FROM ghcr.io/project-zot/golang:1.19 AS builder
-RUN mkdir -p /go/src/github.com/project-zot/zot
-WORKDIR /go/src/github.com/project-zot/zot
-COPY . .
-RUN make clean binary
-RUN echo '{\n\
-    "storage": {\n\
-        "rootDirectory": "/var/lib/registry"\n\
-    },\n\
-    "http": {\n\
-        "address": "0.0.0.0",\n\
-        "port": "5000"\n\
-    },\n\
-    "log": {\n\
-        "level": "debug"\n\
-    },\n\
-    "extensions": {\n\
-        "metrics": {\n\
-            "enable": true,\n\
-            "prometheus": {\n\
-                "path": "/metrics"\n\
-            }\n\
-        }\n\
-    }\n\
-}\n' > config.json && cat config.json
-
-# ---
-# Stage 2: Final image with nothing but certs, binary, and default config file
-# ---
-FROM gcr.io/distroless/base AS final
-COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
-COPY --from=builder /go/src/github.com/project-zot/zot/bin/zot-linux-amd64 /usr/bin/zot
-COPY --from=builder /go/src/github.com/project-zot/zot/config.json /etc/zot/config.json
-ENTRYPOINT ["/usr/bin/zot"]
-EXPOSE 5000
-VOLUME ["/var/lib/registry"]
-CMD ["serve", "/etc/zot/config.json"]
--- a/examples/metrics/Dockerfile-minimal
+++ b/examples/metrics/Dockerfile-minimal
@ -1,32 +0,0 @@
-# ---
-# Stage 1: Install certs, build binary, create default config file
-# ---
-FROM ghcr.io/project-zot/golang:1.19 AS builder
-RUN mkdir -p /go/src/github.com/project-zot/zot
-WORKDIR /go/src/github.com/project-zot/zot
-COPY . .
-RUN make clean binary-minimal
-RUN echo '{\n\
-    "storage": {\n\
-        "rootDirectory": "/var/lib/registry"\n\
-    },\n\
-    "http": {\n\
-        "address": "0.0.0.0",\n\
-        "port": "5050"\n\
-    },\n\
-    "log": {\n\
-        "level": "debug"\n\
-    }\n\
-}\n' > config.json && cat config.json
-
-# ---
-# Stage 2: Final image with nothing but certs, binary, and default config file
-# ---
-FROM gcr.io/distroless/base AS final
-COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
-COPY --from=builder /go/src/github.com/project-zot/zot/bin/zot-linux-amd64-minimal /usr/bin/zot
-COPY --from=builder /go/src/github.com/project-zot/zot/config.json /etc/zot/config.json
-ENTRYPOINT ["/usr/bin/zot"]
-EXPOSE 5050
-VOLUME ["/var/lib/registry"]
-CMD ["serve", "/etc/zot/config.json"]
--- a/examples/metrics/Dockerfile-zxp
+++ b/examples/metrics/Dockerfile-zxp
@ -1,31 +0,0 @@
-# ---
-# Stage 1: Build binary, create default config file
-# ---
-FROM ghcr.io/project-zot/golang:1.19 AS builder
-RUN mkdir -p /go/src/github.com/project-zot/zot
-WORKDIR /go/src/github.com/project-zot/zot
-COPY . .
-RUN make clean exporter-minimal
-RUN echo '{\n\
-    "Server": {\n\
-        "protocol": "http",\n\
-        "host": "127.0.0.1",\n\
-        "port": "5050"\n\
-    },\n\
-    "Exporter": {\n\
-        "port": "5051",\n\
-        "log": {\n\
-            "level": "debug"\n\
-        }\n\
-    }\n\
-}\n' > config.json && cat config.json
-
-# ---
-# Stage 2: Final image with nothing but binary and default config file
-# ---
-FROM gcr.io/distroless/base AS final
-COPY --from=builder /go/src/github.com/project-zot/zot/bin/zxp-linux-amd64 /usr/bin/zxp
-COPY --from=builder /go/src/github.com/project-zot/zot/config.json /etc/zxp/config.json
-ENTRYPOINT ["/usr/bin/zxp"]
-EXPOSE 5051
-CMD ["config", "/etc/zxp/config.json"]
--- a/examples/metrics/Makefile
+++ b/examples/metrics/Makefile
@ -1,27 +0,0 @@
-CONTAINER_RUNTIME := docker
-
-.PHONY: binary-container
-binary-container:
-	${CONTAINER_RUNTIME} build -f build/Dockerfile -t zot-build:latest ../../.
-
-.PHONY: run-container
-run-container:
-	${CONTAINER_RUNTIME} run --rm --security-opt label=disable -v $$(pwd)/../..:/go/src/github.com/project-zot/zot \
-		zot-build:latest
-
-.PHONY: binary-minimal-container
-binary-minimal-container:
-	${CONTAINER_RUNTIME} build -f build/Dockerfile-minimal -t zot-minimal:latest ../../.
-
-.PHONY: run-minimal-container
-run-minimal-container:
-	${CONTAINER_RUNTIME} run --rm --security-opt label=disable -v $$(pwd)/../..:/go/src/github.com/project-zot/zot \
-		zot-minimal:latest
-
-.PHONY: binary-exporter-container
-binary-exporter-container:
-	${CONTAINER_RUNTIME} build -f build/Dockerfile-zxp -t zxp:latest ../../.
-
-.PHONY: run-exporter-container
-run-exporter-container:
-	${CONTAINER_RUNTIME} run --rm --security-opt label=disable zxp:latest
--- a/examples/metrics/kind_setup.sh
+++ b/examples/metrics/kind_setup.sh
@ -6,8 +6,8 @@ set -e
 CLUSTER_NAME=zot

 # Script tested with below kubectl & kind versions
-KUBECTL_VERSION=v1.17.5
-KIND_VERSION=v0.7.0
+KUBECTL_VERSION=v1.27.3
+KIND_VERSION=v0.20.0

 function install_bin() {
    if [ "$EUID" -ne 0 ]
@ -41,9 +41,9 @@ kind load docker-image quay.io/prometheus-operator/prometheus-config-reloader:v0
 kind load docker-image quay.io/prometheus/prometheus:v2.22.1 --name ${CLUSTER_NAME}

 ## Build zot & zxp images
-make binary-container
-make binary-minimal-container
-make binary-exporter-container
+make -C ../../ binary-container
+make -C ../../ binary-minimal-container
+make -C ../../ binary-exporter-container

 kind load docker-image zot-build:latest --name ${CLUSTER_NAME}
 kind load docker-image zot-minimal:latest --name ${CLUSTER_NAME}
--- a/examples/metrics/kubernetes/prometheus/prom_rbac.yaml
+++ b/examples/metrics/kubernetes/prometheus/prom_rbac.yaml
@ -3,7 +3,7 @@ kind: ServiceAccount
 metadata:
  name: prometheus
 ---
-apiVersion: rbac.authorization.k8s.io/v1beta1
+apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
  name: prometheus
@ -28,7 +28,7 @@ rules:
 - nonResourceURLs: ["/metrics"]
  verbs: ["get"]
 ---
-apiVersion: rbac.authorization.k8s.io/v1beta1
+apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
  name: prometheus
--- a/examples/metrics/kubernetes/zot-extended/deployment.yaml
+++ b/examples/metrics/kubernetes/zot-extended/deployment.yaml
@ -18,7 +18,76 @@ spec:
      - name: zot-extended
        image: zot-build:latest
        imagePullPolicy: IfNotPresent
+        command: ["/usr/bin/zot"]
+        args: ["serve", "/zot-config/config.json"]
        ports:
        - name: zot-extended
          containerPort: 5000
-
+          protocol: TCP
+        volumeMounts:
+          - name: zot-config
+            mountPath: /zot-config
+            readOnly: false
+      volumes:
+        - name: zot-config
+          configMap:
+            name: zot-config
+            items:
+            - key: zot_config.json
+              path: config.json
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: zot-config
+data:
+  zot_config.json: |
+    {
+      "distSpecVersion":"1.0.1",
+      "storage": {
+          "rootDirectory": "/var/lib/registry"
+      },
+      "http": {
+          "address": "0.0.0.0",
+          "port": "5000"
+      },
+      "log": {
+          "level": "debug"
+      },
+      "extensions": {
+          "metrics": {
+              "enable": true,
+              "prometheus": {
+                  "path": "/metrics"
+              }
+          }
+      }
+    }
+  zot_minimal.json: |
+    {
+      "distSpecVersion":"1.0.1",
+      "storage": {
+          "rootDirectory": "/var/lib/registry"
+      },
+      "http": {
+          "address": "0.0.0.0",
+          "port": "5050"
+      },
+      "log": {
+          "level": "debug"
+      }
+    }
+  zxp.json: |
+    {
+      "Server": {
+        "protocol": "http",
+        "host": "127.0.0.1",
+        "port": "5050"
+      },
+      "Exporter": {
+        "port": "5051",
+        "log": {
+            "level": "debug"
+        }
+      }
+    }
--- a/examples/metrics/kubernetes/zot-minimal/deployment.yaml
+++ b/examples/metrics/kubernetes/zot-minimal/deployment.yaml
@ -18,13 +18,36 @@ spec:
      - name: zot-minimal
        image: zot-minimal:latest
        imagePullPolicy: IfNotPresent
+        command: ["/usr/bin/zot"]
+        args: ["serve", "/zot-config/config.json"]
        ports:
        - name: zot-minimal
          containerPort: 5050
+          protocol: TCP
+        volumeMounts:
+          - name: zot-config
+            mountPath: /zot-config
+            readOnly: false
      - name: zot-exporter
        image: zxp:latest
        imagePullPolicy: IfNotPresent
+        command: ["/zxp"]
+        args: ["config", "/zot-config/zxp-config.json"]
        ports:
        - name: zot-exporter
          containerPort: 5051
+          protocol: TCP
+        volumeMounts:
+          - name: zot-config
+            mountPath: /zot-config
+            readOnly: false
+      volumes:
+        - name: zot-config
+          configMap:
+            name: zot-config
+            items:
+            - key: zot_minimal.json
+              path: config.json
+            - key: zxp.json
+              path: zxp-config.json