diff --git a/.github/workflows/image.yaml b/.github/workflows/image.yaml index 24148bccd..a0b8da726 100644 --- a/.github/workflows/image.yaml +++ b/.github/workflows/image.yaml @@ -18,9 +18,7 @@ name: image on: push: branches: - - main - - "pull-request/[0-9]+" - - release-* + - precompiled-arm-support-no jobs: image: diff --git a/.github/workflows/precompiled.yaml b/.github/workflows/precompiled.yaml index 21f25eab9..55bd69880 100644 --- a/.github/workflows/precompiled.yaml +++ b/.github/workflows/precompiled.yaml @@ -16,6 +16,9 @@ name: Precompiled images on: + push: + branches: + - precompiled-arm-support schedule: - cron: '00 09 * * *' workflow_dispatch: @@ -28,6 +31,7 @@ jobs: kernel_flavors: ${{ steps.extract_driver_branch.outputs.kernel_flavors }} dist: ${{ steps.extract_driver_branch.outputs.dist }} lts_kernel: ${{ steps.extract_driver_branch.outputs.lts_kernel }} + platforms: ${{ steps.extract_driver_branch.outputs.platforms }} steps: - name: Checkout code uses: actions/checkout@v6 @@ -35,25 +39,34 @@ jobs: id: extract_driver_branch run: | # get driver_branch - DRIVER_BRANCH=("535" "570" "580") + # DRIVER_BRANCH=("535" "570" "580") + DRIVER_BRANCH=("580") driver_branch_json=$(printf '%s\n' "${DRIVER_BRANCH[@]}" | jq -R . | jq -cs .) echo "driver_branch=$driver_branch_json" >> $GITHUB_OUTPUT # get kernel flavors - KERNEL_FLAVORS=("aws" "azure" "azure-fde" "generic" "nvidia" "oracle") + # KERNEL_FLAVORS=("aws" "azure" "azure-fde" "generic" "nvidia" "oracle") + KERNEL_FLAVORS=("generic") kernel_flavors_json=$(printf '%s\n' "${KERNEL_FLAVORS[@]}" | jq -R . | jq -cs .) echo "kernel_flavors=$kernel_flavors_json" >> $GITHUB_OUTPUT # get ubuntu distributions - DIST=("ubuntu22.04" "ubuntu24.04") + # DIST=("ubuntu22.04" "ubuntu24.04") + DIST=("ubuntu24.04") dist_json=$(printf '%s\n' "${DIST[@]}" | jq -R . | jq -cs .) echo "dist=$dist_json" >> $GITHUB_OUTPUT # LTS_KERNEL setup - LTS_KERNEL=("5.15" "6.8") + # LTS_KERNEL=("5.15" "6.8") + LTS_KERNEL=("6.8") lts_kernel_json=$(printf '%s\n' "${LTS_KERNEL[@]}" | jq -R . | jq -cs .) echo "lts_kernel=$lts_kernel_json" >> $GITHUB_OUTPUT + # platforms for precompiled build (amd64 always; arm64 for ubuntu24.04) + PLATFORMS=("amd64" "arm64") + platforms_json=$(printf '%s\n' "${PLATFORMS[@]}" | jq -R . | jq -cs .) + echo "platforms=$platforms_json" >> $GITHUB_OUTPUT + precompiled-build-image: needs: set-driver-version-matrix runs-on: linux-amd64-cpu4 @@ -63,6 +76,7 @@ jobs: flavor: ${{ fromJson(needs.set-driver-version-matrix.outputs.kernel_flavors) }} dist: ${{ fromJson(needs.set-driver-version-matrix.outputs.dist) }} lts_kernel: ${{ fromJson(needs.set-driver-version-matrix.outputs.lts_kernel) }} + platform: ${{ fromJson(needs.set-driver-version-matrix.outputs.platforms) }} exclude: - dist: ubuntu24.04 driver_branch: 535 @@ -70,6 +84,8 @@ jobs: dist: ubuntu24.04 - flavor: azure-fde dist: ubuntu22.04 + - dist: ubuntu22.04 + platform: arm64 steps: - uses: actions/checkout@v6 name: Check out code @@ -82,7 +98,8 @@ jobs: GENERATE_ARTIFACTS="false" echo "PUSH_ON_BUILD=${GENERATE_ARTIFACTS}" >> $GITHUB_ENV - echo "BUILD_MULTI_ARCH_IMAGES=${GENERATE_ARTIFACTS}" >> $GITHUB_ENV + echo "BUILD_MULTI_ARCH_IMAGES=false" >> $GITHUB_ENV + echo "DOCKER_BUILD_PLATFORM_OPTIONS=--platform=linux/${{ matrix.platform }}" >> $GITHUB_ENV - name: Set up QEMU uses: docker/setup-qemu-action@v3 @@ -100,6 +117,7 @@ jobs: env: IMAGE_NAME: ghcr.io/nvidia/driver LTS_KERNEL: ${{ matrix.lts_kernel }} + DOCKER_BUILD_PLATFORM_OPTIONS: --platform=linux/${{ matrix.platform }} run: | if [[ "${{ matrix.dist }}" == "ubuntu22.04" ]]; then BASE_TARGET="jammy" @@ -115,51 +133,66 @@ jobs: docker cp base-${BASE_TARGET}-${{ matrix.flavor }}:/var/kernel_version.txt kernel_version.txt && break sleep 10 done + docker image ls - name: Build image env: IMAGE_NAME: ghcr.io/nvidia/driver PRECOMPILED: "true" DIST: signed_${{ matrix.dist }} + VERSION: "" + DOCKER_BUILD_PLATFORM_OPTIONS: --platform=linux/${{ matrix.platform }} run: | source kernel_version.txt && \ make DRIVER_VERSIONS=${DRIVER_VERSIONS} DRIVER_BRANCH=${{ matrix.driver_branch }} build-${DIST}-${DRIVER_VERSION} - + docker image ls + - name: Save base image, build image and kernel version file env: DIST: ${{ matrix.dist }} PRIVATE_REGISTRY: "ghcr.io" LTS_KERNEL: ${{ matrix.lts_kernel }} run: | + cat kernel_version.txt source kernel_version.txt if [[ "${{ matrix.dist }}" == "ubuntu22.04" ]]; then BASE_TARGET="jammy" elif [[ "${{ matrix.dist }}" == "ubuntu24.04" ]]; then BASE_TARGET="noble" fi - tar -cvf kernel-version-${{ matrix.driver_branch }}-${KERNEL_VERSION}-${DIST}.tar kernel_version.txt + # Artifact name suffix: amd64 has no suffix (e2e uses these), arm64 uses -arm64 + if [[ "${{ matrix.platform }}" == "amd64" ]]; then + echo "PLATFORM_SUFFIX=" >> $GITHUB_ENV + else + echo "PLATFORM_SUFFIX=-${{ matrix.platform }}" >> $GITHUB_ENV + fi + ls -la + docker image ls + tar -cvf kernel-version-${{ matrix.driver_branch }}-${KERNEL_VERSION}-${DIST}${PLATFORM_SUFFIX}.tar kernel_version.txt docker save "${PRIVATE_REGISTRY}/nvidia/driver:base-${BASE_TARGET}-${LTS_KERNEL}-${{ matrix.flavor }}-${{ matrix.driver_branch }}" \ - -o ./base-images-${{ matrix.driver_branch }}-${KERNEL_VERSION}-${DIST}.tar + -o ./base-images-${{ matrix.driver_branch }}-${KERNEL_VERSION}-${DIST}${PLATFORM_SUFFIX}.tar docker save "${PRIVATE_REGISTRY}/nvidia/driver:${{ matrix.driver_branch }}-${KERNEL_VERSION}-${DIST}" \ - -o ./driver-images-${{ matrix.driver_branch }}-${KERNEL_VERSION}-${DIST}.tar - # set env for artifacts upload + -o ./driver-images-${{ matrix.driver_branch }}-${KERNEL_VERSION}-${DIST}${PLATFORM_SUFFIX}.tar echo "KERNEL_VERSION=$KERNEL_VERSION" >> $GITHUB_ENV echo "DIST=$DIST" >> $GITHUB_ENV + ls -la + docker image ls - name: Upload base image as an artifact uses: actions/upload-artifact@v6 with: - name: base-images-${{ matrix.driver_branch }}-${{ env.KERNEL_VERSION }}-${{ env.DIST }} - path: ./base-images-${{ matrix.driver_branch }}-${{ env.KERNEL_VERSION }}-${{ env.DIST }}.tar + name: base-images-${{ matrix.driver_branch }}-${{ env.KERNEL_VERSION }}-${{ env.DIST }}${{ env.PLATFORM_SUFFIX }} + path: ./base-images-${{ matrix.driver_branch }}-${{ env.KERNEL_VERSION }}-${{ env.DIST }}${{ env.PLATFORM_SUFFIX }}.tar retention-days: 1 - name: Upload build image as an artifact uses: actions/upload-artifact@v6 with: - name: driver-images-${{ matrix.driver_branch }}-${{ env.KERNEL_VERSION }}-${{ env.DIST }} - path: ./driver-images-${{ matrix.driver_branch }}-${{ env.KERNEL_VERSION }}-${{ env.DIST }}.tar + name: driver-images-${{ matrix.driver_branch }}-${{ env.KERNEL_VERSION }}-${{ env.DIST }}${{ env.PLATFORM_SUFFIX }} + path: ./driver-images-${{ matrix.driver_branch }}-${{ env.KERNEL_VERSION }}-${{ env.DIST }}${{ env.PLATFORM_SUFFIX }}.tar retention-days: 1 - name: Upload kernel version as an artifact + if: matrix.platform == 'amd64' uses: actions/upload-artifact@v6 with: name: kernel-version-${{ matrix.driver_branch }}-${{ env.KERNEL_VERSION }}-${{ env.DIST }} @@ -263,8 +296,8 @@ jobs: LTS_KERNEL=($(echo "$lts_kernel_json" | jq -r '.[]')) for dist in "${DIST[@]}"; do for kernel in "${LTS_KERNEL[@]}"; do - artifact_name="matrix-values-${dist}-${kernel}" - file_path="./matrix-values-artifacts/${artifact_name}/matrix_values_${dist}_${kernel}.json" + artifact_name="matrix_values_${dist}_${kernel}.json" + file_path="./matrix-values-artifacts/${artifact_name}" if [ -f "$file_path" ]; then echo "Successfully found artifact: $artifact_name at $file_path" value=$(jq -r '.[]' "$file_path") @@ -289,18 +322,18 @@ jobs: # slack notification for new kernel release before e2e tests starts # as e2e tests may fail for new kernel release - - name: Slack notification - if: steps.set_kernel_version.outputs.matrix_values_not_empty == '1' - uses: slackapi/slack-github-action@v2.1.1 - with: - token: ${{ secrets.SLACK_BOT_TOKEN }} - method: chat.postMessage - payload: | - { - "channel": "${{ secrets.SLACK_CHANNEL_ID }}", - "text": "${{ secrets.DETECTED_PRECOMPILED_KERNEL_MESSAGE }} ${{ steps.set_kernel_version.outputs.published_kernels }} \n - Details: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" - } + # - name: Slack notification + # if: steps.set_kernel_version.outputs.matrix_values_not_empty == '1' + # uses: slackapi/slack-github-action@v2.1.1 + # with: + # token: ${{ secrets.SLACK_BOT_TOKEN }} + # method: chat.postMessage + # payload: | + # { + # "channel": "${{ secrets.SLACK_CHANNEL_ID }}", + # "text": "${{ secrets.DETECTED_PRECOMPILED_KERNEL_MESSAGE }} ${{ steps.set_kernel_version.outputs.published_kernels }} \n + # Details: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" + # } e2e-tests-nvidiadriver: runs-on: linux-amd64-cpu4 @@ -458,7 +491,7 @@ jobs: image_path="./base-images-${{ matrix.driver_branch }}-${{ matrix.kernel_version }}.tar" echo "uploading $image_path" docker load -i $image_path - docker push ${PRIVATE_REGISTRY}/nvidia/driver:base-${BASE_TARGET}-${LTS_KERNEL}-${KERNEL_FLAVOR}-${{ matrix.driver_branch }} + # docker push ${PRIVATE_REGISTRY}/nvidia/driver:base-${BASE_TARGET}-${LTS_KERNEL}-${KERNEL_FLAVOR}-${{ matrix.driver_branch }} - name: Download built image artifact if: ${{ ! (matrix.driver_branch == 535 && contains(matrix.kernel_version, 'ubuntu24.04')) }} @@ -473,16 +506,16 @@ jobs: image_path="./driver-images-${{ matrix.driver_branch }}-${{ matrix.kernel_version }}.tar" echo "uploading $image_path" docker load -i $image_path - docker push ${PRIVATE_REGISTRY}/nvidia/driver:${{ matrix.driver_branch }}-${{ matrix.kernel_version }} + # docker push ${PRIVATE_REGISTRY}/nvidia/driver:${{ matrix.driver_branch }}-${{ matrix.kernel_version }} - - name: Slack notification - if: ${{ ! (matrix.driver_branch == 535 && contains(matrix.kernel_version, 'ubuntu24.04')) }} - uses: slackapi/slack-github-action@v2.1.1 - with: - token: ${{ secrets.SLACK_BOT_TOKEN }} - method: chat.postMessage - payload: | - { - "channel": "${{ secrets.SLACK_CHANNEL_ID }}", - "text": "${{ secrets.PUBLISHED_PRECOMPILED_IMAGE_MESSAGE }} ${{ env.PRIVATE_REGISTRY }}/nvidia/driver:${{ matrix.driver_branch }}-${{ matrix.kernel_version }}" - } + # - name: Slack notification + # if: ${{ ! (matrix.driver_branch == 535 && contains(matrix.kernel_version, 'ubuntu24.04')) }} + # uses: slackapi/slack-github-action@v2.1.1 + # with: + # token: ${{ secrets.SLACK_BOT_TOKEN }} + # method: chat.postMessage + # payload: | + # { + # "channel": "${{ secrets.SLACK_CHANNEL_ID }}", + # "text": "${{ secrets.PUBLISHED_PRECOMPILED_IMAGE_MESSAGE }} ${{ env.PRIVATE_REGISTRY }}/nvidia/driver:${{ matrix.driver_branch }}-${{ matrix.kernel_version }}" + # } diff --git a/Makefile b/Makefile index 5572cf13b..ff097d4f4 100644 --- a/Makefile +++ b/Makefile @@ -215,6 +215,7 @@ $(BASE_BUILD_TARGETS): DOCKER_BUILDKIT=1 \ $(DOCKER) $(BUILDX) build --pull --no-cache \ $(DOCKER_BUILD_OPTIONS) \ + $(DOCKER_BUILD_PLATFORM_OPTIONS) \ --tag $(IMAGE) \ --target $(TARGET) \ --build-arg CUDA_VERSION="$(CUDA_VERSION)" \ diff --git a/multi-arch.mk b/multi-arch.mk index 263fa88fb..a9cf25035 100644 --- a/multi-arch.mk +++ b/multi-arch.mk @@ -27,5 +27,4 @@ $(DRIVER_PUSH_TARGETS): push-%: build-ubuntu18.04%: DOCKER_BUILD_PLATFORM_OPTIONS = --platform=linux/amd64 build-signed_ubuntu20.04%: DOCKER_BUILD_PLATFORM_OPTIONS = --platform=linux/amd64 build-signed_ubuntu22.04%: DOCKER_BUILD_PLATFORM_OPTIONS = --platform=linux/amd64 -build-signed_ubuntu24.04%: DOCKER_BUILD_PLATFORM_OPTIONS = --platform=linux/amd64 build-sles%: DOCKER_BUILD_PLATFORM_OPTIONS = --platform=linux/amd64 diff --git a/tests/scripts/findkernelversion.sh b/tests/scripts/findkernelversion.sh index 9731a39c7..9093726e1 100755 --- a/tests/scripts/findkernelversion.sh +++ b/tests/scripts/findkernelversion.sh @@ -18,19 +18,15 @@ export PATH=$(pwd)/bin:${PATH} # calculate kernel version of latest image prefix="kernel-version-${DRIVER_BRANCH}-${LTS_KERNEL}" -suffix="${kernel_flavor}-${DIST}" +suffix="${KERNEL_FLAVOR}-${DIST}" artifact_dir="./kernel-version-artifacts" -artifact=$(find "$artifact_dir" -maxdepth 1 -type d -name "${prefix}*-${suffix}" | head -1) -if [ -n "$artifact" ]; then - artifact_name=$(basename "$artifact") - if [ -f "$artifact/${artifact_name}.tar" ]; then - tar -xf "$artifact/${artifact_name}.tar" -C ./ - export $(grep -oP 'KERNEL_VERSION=[^ ]+' ./kernel_version.txt) - rm -f kernel_version.txt - fi +artifact_file=$(find "$artifact_dir" -maxdepth 1 -type f -name "${prefix}*-${suffix}.tar" | head -1) +if [ -n "$artifact_file" ]; then + tar -xf "$artifact_file" -C ./ + export $(grep -oE 'KERNEL_VERSION=[^ ]+' ./kernel_version.txt) + rm -f kernel_version.txt fi - # calculate driver tag status_nvcr=0 status_ghcr=0 diff --git a/ubuntu24.04/precompiled/Dockerfile b/ubuntu24.04/precompiled/Dockerfile index c36043c7d..387f9c535 100644 --- a/ubuntu24.04/precompiled/Dockerfile +++ b/ubuntu24.04/precompiled/Dockerfile @@ -17,7 +17,8 @@ ENV NVIDIA_VISIBLE_DEVICES=void RUN echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections -RUN dpkg --add-architecture i386 && \ +# Add i386 architecture only for amd64 builds (not available on ARM) +RUN if [ "$TARGETARCH" = "amd64" ]; then dpkg --add-architecture i386; fi && \ apt-get update && apt-get install -y --no-install-recommends \ apt-utils \ build-essential \ @@ -31,11 +32,12 @@ RUN dpkg --add-architecture i386 && \ pkg-config && \ rm -rf /var/lib/apt/lists/* -# Fetch GPG keys for CUDA repo -RUN apt-key del 3bf863cc && \ +# Fetch GPG keys for CUDA repo (architecture-specific) +RUN CUDA_ARCH=$([ "$TARGETARCH" = "arm64" ] && echo "sbsa" || echo "x86_64") && \ + apt-key del 3bf863cc && \ rm /etc/apt/sources.list.d/cuda.list && \ - curl -fsSL https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64/3bf863cc.pub | gpg --dearmor -o /etc/apt/keyrings/cuda.pub && \ - echo "deb [signed-by=/etc/apt/keyrings/cuda.pub] https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64 /" > /etc/apt/sources.list.d/cuda.list + curl -fsSL https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/${CUDA_ARCH}/3bf863cc.pub | gpg --dearmor -o /etc/apt/keyrings/cuda.pub && \ + echo "deb [signed-by=/etc/apt/keyrings/cuda.pub] https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/${CUDA_ARCH} /" > /etc/apt/sources.list.d/cuda.list RUN usermod -o -u 0 -g 0 _apt @@ -50,7 +52,10 @@ COPY nvidia-driver /usr/local/bin ADD local-repo.sh /tmp -RUN mkdir -p /usr/local/repos && \ +RUN apt-get clean && \ + apt-get autoclean && \ + rm -rf /var/lib/apt/lists/* && \ + mkdir -p /usr/local/repos && \ /tmp/local-repo.sh download_driver_package_deps && \ /tmp/local-repo.sh build_local_apt_repo && \ /tmp/local-repo.sh fetch_nvidia_installer && \ diff --git a/ubuntu24.04/precompiled/local-repo.sh b/ubuntu24.04/precompiled/local-repo.sh index 1e35de722..0df3e57d0 100755 --- a/ubuntu24.04/precompiled/local-repo.sh +++ b/ubuntu24.04/precompiled/local-repo.sh @@ -68,7 +68,12 @@ download_driver_package_deps () { pushd ${LOCAL_REPO_DIR} download_apt_with_dep linux-objects-nvidia-${DRIVER_BRANCH}-server-${KERNEL_VERSION} - download_apt_with_dep linux-signatures-nvidia-${KERNEL_VERSION} + + # linux-signatures-nvidia (secure boot signatures) is not available for arm64 + if [ "$TARGETARCH" = "amd64" ]; then + download_apt_with_dep linux-signatures-nvidia-${KERNEL_VERSION} + fi + download_apt_with_dep linux-modules-nvidia-${DRIVER_BRANCH}-server-${KERNEL_VERSION} download_apt_with_dep linux-modules-nvidia-${DRIVER_BRANCH}-server-open-${KERNEL_VERSION} download_apt_with_dep nvidia-utils-${DRIVER_BRANCH}-server @@ -76,7 +81,12 @@ download_driver_package_deps () { download_apt_with_dep libnvidia-decode-${DRIVER_BRANCH}-server download_apt_with_dep libnvidia-extra-${DRIVER_BRANCH}-server download_apt_with_dep libnvidia-encode-${DRIVER_BRANCH}-server - download_apt_with_dep libnvidia-fbc1-${DRIVER_BRANCH}-server + + # SHIVAKU: TODO: Remove this once libnvidia-fbc1 is available for arm64 + # libnvidia-fbc1 (FrameBuffer Capture) is not available for arm64 + if [ "$TARGETARCH" = "amd64" ]; then + download_apt_with_dep libnvidia-fbc1-${DRIVER_BRANCH}-server + fi fabricmanager_download nscq_download