drm/ci: uprev mesa version

zlib.net is not allowing tarball download anymore and results in below error in kernel+rootfs_arm32 container build, urllib.error.HTTPError: HTTP Error 403: Forbidden urllib.error.HTTPError: HTTP Error 415: Unsupported Media Type Uprev mesa to latest version which includes a fix for this issue. https://gitlab.freedesktop.org/mesa/mesa/-/commit/908f444e Use id_tokens for JWT authentication. Since s3 bucket is migrated to mesa-rootfs, update the variables accordingly. Also copy helper scripts to install, so that the ci jobs can use these scripts for logging. Signed-off-by: Vignesh Raman <vignesh.raman@collabora.com> Reviewed-by: Dmitry Baryshkov <dmitry.baryshkov@linaro.org> Reviewed-by: Jessica Zhang <quic_jesszhan@quicinc.com> Acked-by: Helen Koike <helen.koike@collabora.com> Signed-off-by: Helen Koike <helen.koike@collabora.com> Link: https://patchwork.freedesktop.org/patch/msgid/20240529024049.356327-2-vignesh.raman@collabora.com
2024-05-29 08:10:44 +05:30 · 2024-05-29 08:10:44 +05:30 · cc806b7446
commit cc806b7446
parent 3ed96977a3
6 changed files with 42 additions and 28 deletions
--- a/drivers/gpu/drm/ci/build-igt.sh
+++ b/drivers/gpu/drm/ci/build-igt.sh
@ -32,4 +32,4 @@ tar -cf artifacts/igt.tar /igt
 # Pass needed files to the test stage
 S3_ARTIFACT_NAME="igt.tar.gz"
 gzip -c artifacts/igt.tar > ${S3_ARTIFACT_NAME}
-ci-fairy s3cp --token-file "${CI_JOB_JWT_FILE}" ${S3_ARTIFACT_NAME} https://${PIPELINE_ARTIFACTS_BASE}/${KERNEL_ARCH}/${S3_ARTIFACT_NAME}
+ci-fairy s3cp --token-file "${S3_JWT_FILE}" ${S3_ARTIFACT_NAME} https://${PIPELINE_ARTIFACTS_BASE}/${KERNEL_ARCH}/${S3_ARTIFACT_NAME}
--- a/drivers/gpu/drm/ci/build.sh
+++ b/drivers/gpu/drm/ci/build.sh
@ -131,6 +131,7 @@ fi
 # Pass needed files to the test stage
 mkdir -p install
 cp -rfv .gitlab-ci/* install/.
+cp -rfv ci/*  install/.
 cp -rfv install/common install/ci-common
 cp -rfv drivers/gpu/drm/ci/* install/.

@ -144,14 +145,15 @@ if [[ "$UPLOAD_TO_MINIO" = "1" ]]; then
        FILES_TO_UPLOAD="$FILES_TO_UPLOAD $(basename -a $DEVICE_TREES)"
    fi

+    ls -l "${S3_JWT_FILE}"
    for f in $FILES_TO_UPLOAD; do
-        ci-fairy s3cp --token-file "${CI_JOB_JWT_FILE}" /lava-files/$f \
+        ci-fairy s3cp --token-file "${S3_JWT_FILE}" /lava-files/$f \
                https://${PIPELINE_ARTIFACTS_BASE}/${DEBIAN_ARCH}/$f
    done

    S3_ARTIFACT_NAME="kernel-files.tar.zst"
    tar --zstd -cf $S3_ARTIFACT_NAME install
-    ci-fairy s3cp --token-file "${CI_JOB_JWT_FILE}" ${S3_ARTIFACT_NAME} https://${PIPELINE_ARTIFACTS_BASE}/${DEBIAN_ARCH}/${S3_ARTIFACT_NAME}
+    ci-fairy s3cp --token-file "${S3_JWT_FILE}" ${S3_ARTIFACT_NAME} https://${PIPELINE_ARTIFACTS_BASE}/${DEBIAN_ARCH}/${S3_ARTIFACT_NAME}

    echo "Download vmlinux.xz from https://${PIPELINE_ARTIFACTS_BASE}/${DEBIAN_ARCH}/vmlinux.xz"
 fi
--- a/drivers/gpu/drm/ci/container.yml
+++ b/drivers/gpu/drm/ci/container.yml
@ -36,15 +36,15 @@ debian/android_build:
  rules:
    - when: never

-debian/x86_64_test-android:
+.debian/x86_64_test-android:
  rules:
    - when: never

-windows_build_vs2019:
+windows_build_msvc:
  rules:
    - when: never

-windows_test_vs2019:
+windows_test_msvc:
  rules:
    - when: never

@ -56,10 +56,6 @@ rustfmt:
   rules:
    - when: never

-windows_vs2019:
-   rules:
-    - when: never
-
-clang-format:
+windows_msvc:
   rules:
    - when: never
--- a/drivers/gpu/drm/ci/gitlab-ci.yml
+++ b/drivers/gpu/drm/ci/gitlab-ci.yml
@ -1,6 +1,6 @@
 variables:
  DRM_CI_PROJECT_PATH: &drm-ci-project-path mesa/mesa
-  DRM_CI_COMMIT_SHA: &drm-ci-commit-sha 9d162de9a05155e1c4041857a5848842749164cf
+  DRM_CI_COMMIT_SHA: &drm-ci-commit-sha e2b9c5a9e3e4f9b532067af8022eaef8d6fc6c00

  UPSTREAM_REPO: git://anongit.freedesktop.org/drm/drm
  TARGET_BRANCH: drm-next
@ -19,33 +19,47 @@ variables:
          bash download-git-cache.sh
          rm download-git-cache.sh
          set +o xtrace
+  S3_JWT_FILE: /s3_jwt
  S3_HOST: s3.freedesktop.org
+  # This bucket is used to fetch the kernel image
+  S3_KERNEL_BUCKET: mesa-rootfs
+  # Bucket for git cache
+  S3_GITCACHE_BUCKET: git-cache
+  # Bucket for the pipeline artifacts pushed to S3
+  S3_ARTIFACTS_BUCKET: artifacts
  # per-pipeline artifact storage on MinIO
-  PIPELINE_ARTIFACTS_BASE: ${S3_HOST}/artifacts/${CI_PROJECT_PATH}/${CI_PIPELINE_ID}
+  PIPELINE_ARTIFACTS_BASE: ${S3_HOST}/${S3_ARTIFACTS_BUCKET}/${CI_PROJECT_PATH}/${CI_PIPELINE_ID}
  # per-job artifact storage on MinIO
  JOB_ARTIFACTS_BASE: ${PIPELINE_ARTIFACTS_BASE}/${CI_JOB_ID}
  # default kernel for rootfs before injecting the current kernel tree
  KERNEL_REPO: "gfx-ci/linux"
-  KERNEL_TAG: "v6.6.4-for-mesa-ci-e4f4c500f7fb"
-  KERNEL_IMAGE_BASE: https://${S3_HOST}/mesa-lava/${KERNEL_REPO}/${KERNEL_TAG}
+  KERNEL_TAG: "v6.6.21-mesa-f8ea"
+  KERNEL_IMAGE_BASE: https://${S3_HOST}/${S3_KERNEL_BUCKET}/${KERNEL_REPO}/${KERNEL_TAG}
+  PKG_REPO_REV: "3cc12a2a"
  LAVA_TAGS: subset-1-gfx
  LAVA_JOB_PRIORITY: 30
+  ARTIFACTS_BASE_URL: https://${CI_PROJECT_ROOT_NAMESPACE}.${CI_PAGES_DOMAIN}/-/${CI_PROJECT_NAME}/-/jobs/${CI_JOB_ID}/artifacts
+  # Python scripts for structured logger
+  PYTHONPATH: "$PYTHONPATH:$CI_PROJECT_DIR/install"

 default:
+  id_tokens:
+    S3_JWT:
+      aud: https://s3.freedesktop.org
  before_script:
    - export SCRIPTS_DIR=$(mktemp -d)
    - curl -L -s --retry 4 -f --retry-all-errors --retry-delay 60 -O --output-dir "${SCRIPTS_DIR}" "${DRM_CI_PROJECT_URL}/-/raw/${DRM_CI_COMMIT_SHA}/.gitlab-ci/setup-test-env.sh"
    - source ${SCRIPTS_DIR}/setup-test-env.sh
    - echo -e "\e[0Ksection_start:$(date +%s):unset_env_vars_section[collapsed=true]\r\e[0KUnsetting vulnerable environment variables"
-    - export CI_JOB_JWT_FILE="${CI_JOB_JWT_FILE:-$(mktemp)}"
-    - echo -n "${CI_JOB_JWT}" > "${CI_JOB_JWT_FILE}"
-    - unset CI_JOB_JWT
+    - echo -n "${S3_JWT}" > "${S3_JWT_FILE}"
+    - unset CI_JOB_JWT S3_JWT
    - echo -e "\e[0Ksection_end:$(date +%s):unset_env_vars_section\r\e[0K"

    - echo -e "\e[0Ksection_start:$(date +%s):drm_ci_download_section[collapsed=true]\r\e[0KDownloading mesa from $DRM_CI_PROJECT_URL/-/archive/$DRM_CI_COMMIT_SHA/mesa-$DRM_CI_COMMIT_SHA.tar.gz"
    - cd $CI_PROJECT_DIR
    - curl --output - $DRM_CI_PROJECT_URL/-/archive/$DRM_CI_COMMIT_SHA/mesa-$DRM_CI_COMMIT_SHA.tar.gz | tar -xz
    - mv mesa-$DRM_CI_COMMIT_SHA/.gitlab-ci* .
+    - mv mesa-$DRM_CI_COMMIT_SHA/bin/ci .
    - rm -rf mesa-$DRM_CI_COMMIT_SHA/
    - echo -e "\e[0Ksection_end:$(date +%s):drm_ci_download_section\r\e[0K"

@ -53,9 +67,9 @@ default:
    - >
      set +x

-      test -e "${CI_JOB_JWT_FILE}" &&
-      export CI_JOB_JWT="$(<${CI_JOB_JWT_FILE})" &&
-      rm "${CI_JOB_JWT_FILE}"
+      test -e "${S3_JWT_FILE}" &&
+      export S3_JWT="$(<${S3_JWT_FILE})" &&
+      rm "${S3_JWT_FILE}"

 include:
  - project: 'freedesktop/ci-templates'
@ -87,6 +101,7 @@ include:
      - '/src/intel/ci/gitlab-ci-inc.yml'
      - '/src/freedreno/ci/gitlab-ci-inc.yml'
      - '/src/amd/ci/gitlab-ci-inc.yml'
+      - '/src/virtio/ci/gitlab-ci-inc.yml'
  - drivers/gpu/drm/ci/image-tags.yml
  - drivers/gpu/drm/ci/container.yml
  - drivers/gpu/drm/ci/static-checks.yml
@ -98,6 +113,7 @@ include:
 stages:
  - sanity
  - container
+  - code-validation
  - git-archive
  - build
  - amdgpu
@ -107,7 +123,6 @@ stages:
  - msm
  - rockchip
  - virtio-gpu
-  - lint

 # YAML anchors for rule conditions
 # --------------------------------
@ -218,14 +233,15 @@ make git archive:
  script:
    # Remove drm-ci files we just added
    - rm -rf .gitlab-ci.*
+    - rm -rf ci

    # Compactify the .git directory
    - git gc --aggressive
    # compress the current folder
    - tar -cvzf ../$CI_PROJECT_NAME.tar.gz .

-    # login with the JWT token file
-    - ci-fairy s3cp --token-file "${CI_JOB_JWT_FILE}" ../$CI_PROJECT_NAME.tar.gz https://$S3_HOST/git-cache/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/$CI_PROJECT_NAME.tar.gz
+    # Use id_tokens for JWT auth
+    - ci-fairy s3cp --token-file "${S3_JWT_FILE}" ../$CI_PROJECT_NAME.tar.gz https://$S3_HOST/${S3_GITCACHE_BUCKET}/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/$CI_PROJECT_NAME.tar.gz


 # Sanity checks of MR settings and commit logs
@ -262,4 +278,4 @@ sanity:

 # Jobs that need to pass before spending hardware resources on further testing
 .required-for-hardware-jobs:
-  needs: []
+  needs: []
--- a/drivers/gpu/drm/ci/image-tags.yml
+++ b/drivers/gpu/drm/ci/image-tags.yml
@ -1,5 +1,5 @@
 variables:
-   CONTAINER_TAG: "2023-10-11-mesa-uprev"
+   CONTAINER_TAG: "2024-05-09-mesa-uprev"
   DEBIAN_X86_64_BUILD_BASE_IMAGE: "debian/x86_64_build-base"
   DEBIAN_BASE_TAG: "${CONTAINER_TAG}"

--- a/drivers/gpu/drm/ci/lava-submit.sh
+++ b/drivers/gpu/drm/ci/lava-submit.sh
@ -27,7 +27,7 @@ KERNEL_IMAGE_BASE="https://${BASE_SYSTEM_HOST_PATH}" \
 section_end variables

 tar zcf job-rootfs-overlay.tar.gz -C results/job-rootfs-overlay/ .
-ci-fairy s3cp --token-file "${CI_JOB_JWT_FILE}" job-rootfs-overlay.tar.gz "https://${JOB_ROOTFS_OVERLAY_PATH}"
+ci-fairy s3cp --token-file "${S3_JWT_FILE}" job-rootfs-overlay.tar.gz "https://${JOB_ROOTFS_OVERLAY_PATH}"

 touch results/lava.log
 tail -f results/lava.log &
@ -45,7 +45,7 @@ PYTHONPATH=artifacts/ artifacts/lava/lava_job_submitter.py \
 	--ci-project-dir "${CI_PROJECT_DIR}" \
 	--device-type "${DEVICE_TYPE}" \
 	--dtb-filename "${DTB}" \
-	--jwt-file "${CI_JOB_JWT_FILE}" \
+	--jwt-file "${S3_JWT_FILE}" \
 	--kernel-image-name "${KERNEL_IMAGE_NAME}" \
 	--kernel-image-type "${KERNEL_IMAGE_TYPE}" \
 	--boot-method "${BOOT_METHOD}" \