From dcf7b9f2226b280e1e0b1ecca26097dcaf07a77c Mon Sep 17 00:00:00 2001 From: Abdullah Ahmed <12223088+geektype@users.noreply.github.com> Date: Thu, 15 Feb 2024 20:45:54 +0000 Subject: [PATCH 01/10] pr dummy commit to create PR --- src/ci/docker/run.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ci/docker/run.sh b/src/ci/docker/run.sh index bd5447ac835d9..e2d19d89b7cbe 100755 --- a/src/ci/docker/run.sh +++ b/src/ci/docker/run.sh @@ -7,6 +7,7 @@ export MSYS_NO_PATHCONV=1 script=`cd $(dirname $0) && pwd`/`basename $0` image="" + dev=0 while [[ $# -gt 0 ]] From ddc7311b9de3542a647b314a9d8aa4fd0475d1f4 Mon Sep 17 00:00:00 2001 From: Abdullah Ahmed <12223088+geektype@users.noreply.github.com> Date: Thu, 15 Feb 2024 21:40:02 +0000 Subject: [PATCH 02/10] Local CI --- .github/workflows/ci.yml | 90 ++++++++++++++++-------------------- src/ci/github-actions/ci.yml | 27 ++++++----- 2 files changed, 55 insertions(+), 62 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f95551d679b98..77e170ab122ac 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,18 +52,8 @@ jobs: strategy: matrix: include: - - name: mingw-check - os: ubuntu-20.04-4core-16gb - env: {} - - name: mingw-check-tidy - os: ubuntu-20.04-4core-16gb - env: {} - - name: x86_64-gnu-llvm-16 - env: - ENABLE_GCC_CODEGEN: "1" - os: ubuntu-20.04-16core-64gb - - name: x86_64-gnu-tools - os: ubuntu-20.04-16core-64gb + - name: temp + os: ubuntu-20.04 env: {} timeout-minutes: 600 runs-on: "${{ matrix.os }}" @@ -194,128 +184,128 @@ jobs: - ARM64 - linux - name: arm-android - os: ubuntu-20.04-8core-32gb + os: ubuntu-20.04 env: {} - name: armhf-gnu - os: ubuntu-20.04-8core-32gb + os: ubuntu-20.04 env: {} - name: dist-aarch64-linux env: CODEGEN_BACKENDS: "llvm,cranelift" - os: ubuntu-20.04-8core-32gb + os: ubuntu-20.04 - name: dist-android - os: ubuntu-20.04-8core-32gb + os: ubuntu-20.04 env: {} - name: dist-arm-linux - os: ubuntu-20.04-16core-64gb + os: ubuntu-20.04 env: {} - name: dist-armhf-linux - os: ubuntu-20.04-8core-32gb + os: ubuntu-20.04 env: {} - name: dist-armv7-linux - os: ubuntu-20.04-8core-32gb + os: ubuntu-20.04 env: {} - name: dist-i586-gnu-i586-i686-musl - os: ubuntu-20.04-8core-32gb + os: ubuntu-20.04 env: {} - name: dist-i686-linux - os: ubuntu-20.04-8core-32gb + os: ubuntu-20.04 env: {} - name: dist-loongarch64-linux - os: ubuntu-20.04-8core-32gb + os: ubuntu-20.04 env: {} - name: dist-powerpc-linux - os: ubuntu-20.04-8core-32gb + os: ubuntu-20.04 env: {} - name: dist-powerpc64-linux - os: ubuntu-20.04-8core-32gb + os: ubuntu-20.04 env: {} - name: dist-powerpc64le-linux - os: ubuntu-20.04-8core-32gb + os: ubuntu-20.04 env: {} - name: dist-riscv64-linux - os: ubuntu-20.04-8core-32gb + os: ubuntu-20.04 env: {} - name: dist-s390x-linux - os: ubuntu-20.04-8core-32gb + os: ubuntu-20.04 env: {} - name: dist-various-1 - os: ubuntu-20.04-8core-32gb + os: ubuntu-20.04 env: {} - name: dist-various-2 - os: ubuntu-20.04-8core-32gb + os: ubuntu-20.04 env: {} - name: dist-x86_64-freebsd - os: ubuntu-20.04-8core-32gb + os: ubuntu-20.04 env: {} - name: dist-x86_64-illumos - os: ubuntu-20.04-8core-32gb + os: ubuntu-20.04 env: {} - name: dist-x86_64-linux env: CODEGEN_BACKENDS: "llvm,cranelift" - os: ubuntu-20.04-16core-64gb + os: ubuntu-20.04 - name: dist-x86_64-linux-alt env: IMAGE: dist-x86_64-linux CODEGEN_BACKENDS: "llvm,cranelift" - os: ubuntu-20.04-16core-64gb + os: ubuntu-20.04 - name: dist-x86_64-musl env: CODEGEN_BACKENDS: "llvm,cranelift" - os: ubuntu-20.04-8core-32gb + os: ubuntu-20.04 - name: dist-x86_64-netbsd - os: ubuntu-20.04-8core-32gb + os: ubuntu-20.04 env: {} - name: i686-gnu - os: ubuntu-20.04-8core-32gb + os: ubuntu-20.04 env: {} - name: i686-gnu-nopt - os: ubuntu-20.04-8core-32gb + os: ubuntu-20.04 env: {} - name: mingw-check - os: ubuntu-20.04-4core-16gb + os: ubuntu-20.04 env: {} - name: test-various - os: ubuntu-20.04-8core-32gb + os: ubuntu-20.04 env: {} - name: x86_64-gnu - os: ubuntu-20.04-4core-16gb + os: ubuntu-20.04 env: {} - name: x86_64-gnu-stable env: IMAGE: x86_64-gnu RUST_CI_OVERRIDE_RELEASE_CHANNEL: stable CI_ONLY_WHEN_CHANNEL: nightly - os: ubuntu-20.04-4core-16gb + os: ubuntu-20.04 - name: x86_64-gnu-aux - os: ubuntu-20.04-4core-16gb + os: ubuntu-20.04 env: {} - name: x86_64-gnu-integration env: CI_ONLY_WHEN_CHANNEL: nightly - os: ubuntu-20.04-8core-32gb + os: ubuntu-20.04 - name: x86_64-gnu-debug - os: ubuntu-20.04-8core-32gb + os: ubuntu-20.04 env: {} - name: x86_64-gnu-distcheck - os: ubuntu-20.04-8core-32gb + os: ubuntu-20.04 env: {} - name: x86_64-gnu-llvm-17 env: RUST_BACKTRACE: 1 - os: ubuntu-20.04-8core-32gb + os: ubuntu-20.04 - name: x86_64-gnu-llvm-16 env: RUST_BACKTRACE: 1 - os: ubuntu-20.04-8core-32gb + os: ubuntu-20.04 - name: x86_64-gnu-nopt - os: ubuntu-20.04-4core-16gb + os: ubuntu-20.04 env: {} - name: x86_64-gnu-tools env: DEPLOY_TOOLSTATES_JSON: toolstates-linux.json - os: ubuntu-20.04-8core-32gb + os: ubuntu-20.04 - name: dist-x86_64-apple env: SCRIPT: "./x.py dist bootstrap --include-default-paths --host=x86_64-apple-darwin --target=x86_64-apple-darwin" @@ -588,7 +578,7 @@ jobs: - name: dist-x86_64-linux env: CODEGEN_BACKENDS: "llvm,cranelift" - os: ubuntu-20.04-16core-64gb + os: ubuntu-20.04 timeout-minutes: 600 runs-on: "${{ matrix.os }}" steps: diff --git a/src/ci/github-actions/ci.yml b/src/ci/github-actions/ci.yml index 9b0f477409a93..0f097066a4842 100644 --- a/src/ci/github-actions/ci.yml +++ b/src/ci/github-actions/ci.yml @@ -77,15 +77,15 @@ x--expand-yaml-anchors--remove: env: {} - &job-linux-4c - os: ubuntu-20.04-4core-16gb + os: ubuntu-20.04 <<: *base-job - &job-linux-8c - os: ubuntu-20.04-8core-32gb + os: ubuntu-20.04 <<: *base-job - &job-linux-16c - os: ubuntu-20.04-16core-64gb + os: ubuntu-20.04 <<: *base-job - &job-macos-xl @@ -330,19 +330,22 @@ jobs: strategy: matrix: include: - - name: mingw-check + - name: temp <<: *job-linux-4c + + # - name: mingw-check + # <<: *job-linux-4c - - name: mingw-check-tidy - <<: *job-linux-4c + # - name: mingw-check-tidy + # <<: *job-linux-4c - - name: x86_64-gnu-llvm-16 - env: - ENABLE_GCC_CODEGEN: "1" - <<: *job-linux-16c + # - name: x86_64-gnu-llvm-16 + # env: + # ENABLE_GCC_CODEGEN: "1" + # <<: *job-linux-16c - - name: x86_64-gnu-tools - <<: *job-linux-16c + # - name: x86_64-gnu-tools + # <<: *job-linux-16c auto: <<: *base-ci-job From 9ee04b7a2f69187e1c1200495c03eb8a69d2d751 Mon Sep 17 00:00:00 2001 From: Abdullah Ahmed <12223088+geektype@users.noreply.github.com> Date: Thu, 15 Feb 2024 21:43:56 +0000 Subject: [PATCH 03/10] branch --- .github/workflows/ci.yml | 1 + src/ci/github-actions/ci.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 77e170ab122ac..4f7cbd6a998f5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,6 +23,7 @@ name: CI - try-perf - automation/bors/try - master + - optimise-docker-cache pull_request: branches: - "**" diff --git a/src/ci/github-actions/ci.yml b/src/ci/github-actions/ci.yml index 0f097066a4842..9bca48c0edd47 100644 --- a/src/ci/github-actions/ci.yml +++ b/src/ci/github-actions/ci.yml @@ -296,6 +296,7 @@ on: - try-perf - automation/bors/try - master + - optimise-docker-cache pull_request: branches: - "**" From 22cb797a54cb12ed3364c6d013a244b81f2fe4a1 Mon Sep 17 00:00:00 2001 From: Abdullah Ahmed <12223088+geektype@users.noreply.github.com> Date: Thu, 15 Feb 2024 21:51:36 +0000 Subject: [PATCH 04/10] Temp dockerfile --- src/ci/docker/host-x86_64/temp/Dockerfile | 29 +++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 src/ci/docker/host-x86_64/temp/Dockerfile diff --git a/src/ci/docker/host-x86_64/temp/Dockerfile b/src/ci/docker/host-x86_64/temp/Dockerfile new file mode 100644 index 0000000000000..cd38b9c67fe69 --- /dev/null +++ b/src/ci/docker/host-x86_64/temp/Dockerfile @@ -0,0 +1,29 @@ +FROM ubuntu:22.04 + +ARG DEBIAN_FRONTEND=noninteractive +RUN apt-get update && apt-get install -y --no-install-recommends \ + g++ \ + make \ + ninja-build \ + file \ + curl \ + ca-certificates \ + python3 \ + git \ + cmake \ + sudo \ + gdb \ + libssl-dev \ + pkg-config \ + xz-utils \ + mingw-w64 \ + && rm -rf /var/lib/apt/lists/* + +COPY scripts/sccache.sh /scripts/ +RUN sh /scripts/sccache.sh + +ENV RUST_CONFIGURE_ARGS \ + --build=x86_64-unknown-linux-gnu \ + --enable-sanitizers \ + --enable-profiler \ + --enable-compiler-docs \ No newline at end of file From 6162d2a290c64dab7abdfcff050f2d50e738b0df Mon Sep 17 00:00:00 2001 From: Abdullah Ahmed <12223088+geektype@users.noreply.github.com> Date: Thu, 15 Feb 2024 22:03:23 +0000 Subject: [PATCH 05/10] Test --- src/ci/scripts/run-build-from-ci.sh | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/src/ci/scripts/run-build-from-ci.sh b/src/ci/scripts/run-build-from-ci.sh index 55e75800d91c4..fdef1d3475421 100755 --- a/src/ci/scripts/run-build-from-ci.sh +++ b/src/ci/scripts/run-build-from-ci.sh @@ -10,14 +10,16 @@ source "$(cd "$(dirname "$0")" && pwd)/../shared.sh" export CI="true" export SRC=. -echo "::add-matcher::src/ci/github-actions/problem_matchers.json" +echo DONE -# Remove any preexisting rustup installation since it can interfere -# with the cargotest step and its auto-detection of things like Clippy in -# the environment -rustup self uninstall -y || true -if [ -z "${IMAGE+x}" ]; then - src/ci/run.sh -else - src/ci/docker/run.sh "${IMAGE}" -fi +# echo "::add-matcher::src/ci/github-actions/problem_matchers.json" + +# # Remove any preexisting rustup installation since it can interfere +# # with the cargotest step and its auto-detection of things like Clippy in +# # the environment +# rustup self uninstall -y || true +# if [ -z "${IMAGE+x}" ]; then +# src/ci/run.sh +# else +# src/ci/docker/run.sh "${IMAGE}" +# fi From 96c7bd9168e5464bb7b7338f14f22e9ef129b9ac Mon Sep 17 00:00:00 2001 From: Abdullah Ahmed <12223088+geektype@users.noreply.github.com> Date: Thu, 15 Feb 2024 22:09:29 +0000 Subject: [PATCH 06/10] test --- src/ci/docker/run.sh | 2 +- src/ci/scripts/run-build-from-ci.sh | 22 ++++++++++------------ 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/src/ci/docker/run.sh b/src/ci/docker/run.sh index e2d19d89b7cbe..7795023f2c4a1 100755 --- a/src/ci/docker/run.sh +++ b/src/ci/docker/run.sh @@ -95,7 +95,7 @@ if [ -f "$docker_dir/$image/Dockerfile" ]; then # On non-CI or PR jobs, we don't have permissions to write to the registry cache, so we should # not use `docker login` nor caching. - if [[ "$CI" == "" ]] || [[ "$PR_CI_JOB" == "1" ]]; + if [[ "$CI" == "" ]]; then retry docker build --rm -t rust-ci -f "$dockerfile" "$context" else diff --git a/src/ci/scripts/run-build-from-ci.sh b/src/ci/scripts/run-build-from-ci.sh index fdef1d3475421..55e75800d91c4 100755 --- a/src/ci/scripts/run-build-from-ci.sh +++ b/src/ci/scripts/run-build-from-ci.sh @@ -10,16 +10,14 @@ source "$(cd "$(dirname "$0")" && pwd)/../shared.sh" export CI="true" export SRC=. -echo DONE +echo "::add-matcher::src/ci/github-actions/problem_matchers.json" -# echo "::add-matcher::src/ci/github-actions/problem_matchers.json" - -# # Remove any preexisting rustup installation since it can interfere -# # with the cargotest step and its auto-detection of things like Clippy in -# # the environment -# rustup self uninstall -y || true -# if [ -z "${IMAGE+x}" ]; then -# src/ci/run.sh -# else -# src/ci/docker/run.sh "${IMAGE}" -# fi +# Remove any preexisting rustup installation since it can interfere +# with the cargotest step and its auto-detection of things like Clippy in +# the environment +rustup self uninstall -y || true +if [ -z "${IMAGE+x}" ]; then + src/ci/run.sh +else + src/ci/docker/run.sh "${IMAGE}" +fi From 09ec2a07c9b0c45bc82307973ed7c8bbadd7a25e Mon Sep 17 00:00:00 2001 From: Abdullah Ahmed <12223088+geektype@users.noreply.github.com> Date: Thu, 15 Feb 2024 22:24:44 +0000 Subject: [PATCH 07/10] test --- src/ci/docker/run.sh | 70 +++++++++++++++++++++++--------------------- 1 file changed, 36 insertions(+), 34 deletions(-) diff --git a/src/ci/docker/run.sh b/src/ci/docker/run.sh index 7795023f2c4a1..142615742de3f 100755 --- a/src/ci/docker/run.sh +++ b/src/ci/docker/run.sh @@ -294,37 +294,39 @@ if [ "$ENABLE_GCC_CODEGEN" = "1" ]; then echo "Setting extra environment values for docker: $extra_env" fi -docker \ - run \ - --workdir /checkout/obj \ - --env SRC=/checkout \ - $extra_env \ - $args \ - --env CARGO_HOME=/cargo \ - --env DEPLOY \ - --env DEPLOY_ALT \ - --env CI \ - --env GITHUB_ACTIONS \ - --env GITHUB_REF \ - --env GITHUB_STEP_SUMMARY="/checkout/obj/${SUMMARY_FILE}" \ - --env TOOLSTATE_REPO_ACCESS_TOKEN \ - --env TOOLSTATE_REPO \ - --env TOOLSTATE_PUBLISH \ - --env RUST_CI_OVERRIDE_RELEASE_CHANNEL \ - --env CI_JOB_NAME="${CI_JOB_NAME-$IMAGE}" \ - --env BASE_COMMIT="$BASE_COMMIT" \ - --env DIST_TRY_BUILD \ - --env PR_CI_JOB \ - --env OBJDIR_ON_HOST="$objdir" \ - --env CODEGEN_BACKENDS \ - --init \ - --rm \ - rust-ci \ - "${command[@]}" - -cat $objdir/${SUMMARY_FILE} >> "${GITHUB_STEP_SUMMARY}" - -if [ -f /.dockerenv ]; then - rm -rf $objdir - docker cp checkout:/checkout/obj $objdir -fi +echo "${command[@]}" + +# docker \ +# run \ +# --workdir /checkout/obj \ +# --env SRC=/checkout \ +# $extra_env \ +# $args \ +# --env CARGO_HOME=/cargo \ +# --env DEPLOY \ +# --env DEPLOY_ALT \ +# --env CI \ +# --env GITHUB_ACTIONS \ +# --env GITHUB_REF \ +# --env GITHUB_STEP_SUMMARY="/checkout/obj/${SUMMARY_FILE}" \ +# --env TOOLSTATE_REPO_ACCESS_TOKEN \ +# --env TOOLSTATE_REPO \ +# --env TOOLSTATE_PUBLISH \ +# --env RUST_CI_OVERRIDE_RELEASE_CHANNEL \ +# --env CI_JOB_NAME="${CI_JOB_NAME-$IMAGE}" \ +# --env BASE_COMMIT="$BASE_COMMIT" \ +# --env DIST_TRY_BUILD \ +# --env PR_CI_JOB \ +# --env OBJDIR_ON_HOST="$objdir" \ +# --env CODEGEN_BACKENDS \ +# --init \ +# --rm \ +# rust-ci \ +# "${command[@]}" + +# cat $objdir/${SUMMARY_FILE} >> "${GITHUB_STEP_SUMMARY}" + +# if [ -f /.dockerenv ]; then +# rm -rf $objdir +# docker cp checkout:/checkout/obj $objdir +# fi From a9960d8f1d5e3cda2ca6dda70cc1267b5e4de2bb Mon Sep 17 00:00:00 2001 From: Abdullah Ahmed <12223088+geektype@users.noreply.github.com> Date: Fri, 16 Feb 2024 20:13:43 +0000 Subject: [PATCH 08/10] test --- src/ci/docker/run.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ci/docker/run.sh b/src/ci/docker/run.sh index 142615742de3f..bd26bd8f30e7a 100755 --- a/src/ci/docker/run.sh +++ b/src/ci/docker/run.sh @@ -133,6 +133,7 @@ if [ -f "$docker_dir/$image/Dockerfile" ]; then # Tag the built image and push it to the registry docker tag rust-ci "${IMAGE_TAG}" + echo ">>>>>>>>>>>>>>>>>> ${IMAGE_TAG}" docker push "${IMAGE_TAG}" # Record the container registry tag/url for reuse, e.g. by rustup.rs builds From 2ace70a54a1165c5535873888684fad080f0f814 Mon Sep 17 00:00:00 2001 From: Abdullah Ahmed <12223088+geektype@users.noreply.github.com> Date: Fri, 16 Feb 2024 20:39:28 +0000 Subject: [PATCH 09/10] kick --- kick | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 kick diff --git a/kick b/kick new file mode 100644 index 0000000000000..e69de29bb2d1d From 2c7f99ef64b9b8d3eac5e6f0f9a111e3d2f2295d Mon Sep 17 00:00:00 2001 From: Abdullah Ahmed <12223088+geektype@users.noreply.github.com> Date: Fri, 16 Feb 2024 21:28:03 +0000 Subject: [PATCH 10/10] kick --- kick | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 kick diff --git a/kick b/kick deleted file mode 100644 index e69de29bb2d1d..0000000000000