From 3e9d6525c7a760a430eb17b766a833af40b8c53b Mon Sep 17 00:00:00 2001
From: Richard Si <sichard26@gmail.com>
Date: Sat, 21 Dec 2024 20:21:19 -0500
Subject: [PATCH] CI: use a Dev Drive to improve Windows I/O performance

Dev Drives are a modern Windows feature based on the ReFS file system
which offers significantly better performance for developer-focused
workloads. This is perfect for pip's Windows CI which is still slower
than the Unix jobs.

Most of the implementation was borrowed from the uv project which also
uses a Dev Drive to improve their Windows CI times. There is a community
action (samypr100/setup-dev-drive) that can set up a Dev Drive for us,
but the Powershell script needed for our simple needs is really not that
bad. The small maintenance burden of doing it ourselves is perferable
over the risks of using yet another 3rd party action.

NB: We used to use a RAM disk to improve I/O performance, but the creation
of the RAM disk started to fail intermittently, annoying everyone and
eliminating any speed ups gained by the constant retrying needed.

See also: https://learn.microsoft.com/en-us/windows/dev-drive/
---
 .github/workflows/ci.yml | 33 ++++++++++++++-------------------
 tools/ci/devdrive.ps1    | 39 +++++++++++++++++++++++++++++++++++++++
 2 files changed, 53 insertions(+), 19 deletions(-)
 create mode 100644 tools/ci/devdrive.ps1

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 2af281c7027..eced817ec35 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -112,14 +112,15 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        os: [ubuntu-latest, macos-13, macos-latest]
+        os: [ubuntu-latest]
+        # os: [ubuntu-latest, macos-13, macos-latest]
         python:
           - "3.8"
-          - "3.9"
-          - "3.10"
-          - "3.11"
-          - "3.12"
-          - "3.13"
+        #   - "3.9"
+        #   - "3.10"
+        #   - "3.11"
+        #   - "3.12"
+        #   - "3.13"
 
     steps:
       - uses: actions/checkout@v4
@@ -162,7 +163,7 @@ jobs:
     name: tests / ${{ matrix.python }} / ${{ matrix.os }} / ${{ matrix.group }}
     runs-on: ${{ matrix.os }}-latest
 
-    needs: [packaging, determine-changes]
+    needs: [determine-changes]
     if: >-
       needs.determine-changes.outputs.tests == 'true' ||
       github.event_name != 'pull_request'
@@ -189,13 +190,13 @@ jobs:
           python-version: ${{ matrix.python }}
           allow-prereleases: true
 
-      # We use C:\Temp (which is already available on the worker)
-      # as a temporary directory for all of the tests because the
-      # default value (under the user dir) is more deeply nested
-      # and causes tests to fail with "path too long" errors.
+      - name: Create a Dev Drive
+        run: |
+          ./tools/ci/devdrive.ps1 -Drive R -Size 5GB
+          mkdir R:\Temp
+          echo "TEMP=R:\\Temp" >> $env:GITHUB_ENV
+
       - run: pip install nox
-        env:
-          TEMP: "C:\\Temp"
 
       # Main check
       - name: Run unit tests
@@ -204,8 +205,6 @@ jobs:
           nox -s test-${{ matrix.python }} --
           -m unit
           --verbose --numprocesses auto --showlocals
-        env:
-          TEMP: "C:\\Temp"
 
       - name: Run integration tests (group 1)
         if: matrix.group == 1
@@ -213,8 +212,6 @@ jobs:
           nox -s test-${{ matrix.python }} --
           -m integration -k "not test_install"
           --verbose --numprocesses auto --showlocals
-        env:
-          TEMP: "C:\\Temp"
 
       - name: Run integration tests (group 2)
         if: matrix.group == 2
@@ -222,8 +219,6 @@ jobs:
           nox -s test-${{ matrix.python }} --
           -m integration -k "test_install"
           --verbose --numprocesses auto --showlocals
-        env:
-          TEMP: "C:\\Temp"
 
   tests-zipapp:
     name: tests / zipapp
diff --git a/tools/ci/devdrive.ps1 b/tools/ci/devdrive.ps1
new file mode 100644
index 00000000000..7e95c7aa84d
--- /dev/null
+++ b/tools/ci/devdrive.ps1
@@ -0,0 +1,39 @@
+# A Powershell script to create a Dev Drive which tends to have significantly better
+# performance in developer workloads. The goal is improve pip's Windows CI times.
+#
+# The implementation was borrowed from the uv project which also use a Dev Drive for
+# better CI performance: https://github.com/astral-sh/uv/pull/3522
+#
+# Windows docs:
+#   https://learn.microsoft.com/en-us/windows/dev-drive/
+# Related GHA reports:
+#   https://github.com/actions/runner-images/issues/7320 (Windows slowness report)
+#   https://github.com/actions/runner-images/issues/8698 (feature request for
+#     preprovisioned Dev Drives)
+
+[CmdletBinding()]
+param(
+    [Parameter(Mandatory=$true,
+    HelpMessage="Drive letter to use for the Dev Drive")]
+    [String]$drive,
+    [Parameter(Mandatory=$true,
+    HelpMessage="Size to allocate to the Dev Drive")]
+    [UInt64]$size
+)
+$ErrorActionPreference = "Stop"
+
+$OSVersion = [Environment]::OSVersion.Version
+$Partition = New-VHD -Path C:/pip_dev_drive.vhdx -SizeBytes $size |
+          Mount-VHD -Passthru |
+          Initialize-Disk -Passthru |
+          New-Partition -DriveLetter $drive -UseMaximumSize
+# Dev Drives aren't supported on all GHA Windows runners, in which case fallback to
+# a ReFS disk which still offer performance gains.
+if ($OSVersion -ge (New-Object 'Version' 10.0.22621)) {
+    Write-Output "Dev Drives are supported, one will be created at ${drive}:"
+    $Volume = ($Partition | Format-Volume -DevDrive -Confirm:$false -Force)
+} else {
+    Write-Output "Dev Drives are unsupported, only creating a ReFS disk at ${drive}:"
+    $Volume = ($Partition | Format-Volume -FileSystem ReFS -Confirm:$false -Force)
+}
+Write-Output $Volume