PYTHON-5379 Run more variants on pull requests (#2340)

Author: blink1073

.evergreen/generated_configs/tasks.yml

@@ -178,26 +178,30 @@ tasks:
         vars:
           TEST_NAME: kms
           SUB_TEST_NAME: gcp
+    tags: []
   - name: test-gcpkms-fail
     commands:
       - func: run server
       - func: run tests
         vars:
           TEST_NAME: kms
           SUB_TEST_NAME: gcp-fail
+    tags: [pr]
   - name: test-azurekms
     commands:
       - func: run tests
         vars:
           TEST_NAME: kms
           SUB_TEST_NAME: azure
+    tags: []
   - name: test-azurekms-fail
     commands:
       - func: run server
       - func: run tests
         vars:
           TEST_NAME: kms
           SUB_TEST_NAME: azure-fail
+    tags: [pr]
 
   # Mod wsgi tests
   - name: mod-wsgi-replica-set-python3.9

.evergreen/generated_configs/variants.yml

@@ -22,6 +22,7 @@ buildvariants:
       VERSION: latest
       NO_EXT: "1"
       REQUIRE_FIPS: "1"
+    tags: []
   - name: other-hosts-rhel8-zseries-latest
     tasks:
       - name: .test-no-toolchain
@@ -32,6 +33,7 @@ buildvariants:
     expansions:
       VERSION: latest
       NO_EXT: "1"
+    tags: []
   - name: other-hosts-rhel8-power8-latest
     tasks:
       - name: .test-no-toolchain
@@ -42,6 +44,7 @@ buildvariants:
     expansions:
       VERSION: latest
       NO_EXT: "1"
+    tags: []
   - name: other-hosts-rhel8-arm64-latest
     tasks:
       - name: .test-no-toolchain
@@ -52,6 +55,7 @@ buildvariants:
     expansions:
       VERSION: latest
       NO_EXT: "1"
+    tags: []
   - name: other-hosts-amazon2023-latest
     tasks:
       - name: .test-no-toolchain
@@ -62,6 +66,7 @@ buildvariants:
     expansions:
       VERSION: latest
       NO_EXT: "1"
+    tags: [pr]
 
   # Atlas connect tests
   - name: atlas-connect-rhel8
@@ -70,6 +75,7 @@ buildvariants:
     display_name: Atlas connect RHEL8
     run_on:
       - rhel87-small
+    tags: [pr]
 
   # Atlas data lake tests
   - name: atlas-data-lake-ubuntu-22
@@ -80,6 +86,7 @@ buildvariants:
       - ubuntu2204-small
     expansions:
       TEST_NAME: data_lake
+    tags: [pr]
 
   # Aws auth tests
   - name: auth-aws-ubuntu-20
@@ -88,18 +95,21 @@ buildvariants:
     display_name: Auth AWS Ubuntu-20
     run_on:
       - ubuntu2004-small
+    tags: []
   - name: auth-aws-win64
     tasks:
       - name: .auth-aws !.auth-aws-ecs
     display_name: Auth AWS Win64
     run_on:
       - windows-64-vsMulti-small
+    tags: []
   - name: auth-aws-macos
     tasks:
       - name: .auth-aws !.auth-aws-web-identity !.auth-aws-ecs !.auth-aws-ec2
     display_name: Auth AWS macOS
     run_on:
       - macos-14
+    tags: [pr]
 
   # Aws lambda tests
   - name: faas-lambda
@@ -288,6 +298,7 @@ buildvariants:
       - rhel87-small
     expansions:
       PYTHON_BINARY: /opt/python/3.13t/bin/python3
+    tags: [pr]
   - name: free-threaded-macos-python3.13t
     tasks:
       - name: .free-threading
@@ -296,6 +307,7 @@ buildvariants:
       - macos-14
     expansions:
       PYTHON_BINARY: /Library/Frameworks/PythonT.Framework/Versions/3.13/bin/python3t
+    tags: []
   - name: free-threaded-macos-arm64-python3.13t
     tasks:
       - name: .free-threading
@@ -304,6 +316,7 @@ buildvariants:
       - macos-14-arm64
     expansions:
       PYTHON_BINARY: /Library/Frameworks/PythonT.Framework/Versions/3.13/bin/python3t
+    tags: []
 
   # Green framework tests
   - name: green-eventlet-rhel8
@@ -372,6 +385,7 @@ buildvariants:
       - rhel87-small
     expansions:
       TEST_NAME: mockupdb
+    tags: [pr]
 
   # Mod wsgi tests
   - name: mod_wsgi-ubuntu-22
@@ -398,6 +412,7 @@ buildvariants:
     display_name: No server RHEL8
     run_on:
       - rhel87-small
+    tags: [pr]
 
   # Ocsp tests
   - name: ocsp-rhel8
@@ -427,21 +442,29 @@ buildvariants:
   # Oidc auth tests
   - name: auth-oidc-ubuntu-22
     tasks:
-      - name: .auth_oidc
+      - name: .auth_oidc_remote
     display_name: Auth OIDC Ubuntu-22
     run_on:
       - ubuntu2204-small
     batchtime: 10080
+  - name: auth-oidc-local-ubuntu-22
+    tasks:
+      - name: "!.auth_oidc_remote .auth_oidc"
+    display_name: Auth OIDC Local Ubuntu-22
+    run_on:
+      - ubuntu2204-small
+    batchtime: 10080
+    tags: [pr]
   - name: auth-oidc-macos
     tasks:
-      - name: .auth_oidc !.auth_oidc_remote
+      - name: "!.auth_oidc_remote .auth_oidc"
     display_name: Auth OIDC macOS
     run_on:
       - macos-14
     batchtime: 10080
   - name: auth-oidc-win64
     tasks:
-      - name: .auth_oidc !.auth_oidc_remote
+      - name: "!.auth_oidc_remote .auth_oidc"
     display_name: Auth OIDC Win64
     run_on:
       - windows-64-vsMulti-small

.evergreen/scripts/generate_config.py

@@ -112,10 +112,13 @@ def create_free_threaded_variants() -> list[BuildVariant]:
             # TODO: PYTHON-5027
             continue
         tasks = [".free-threading"]
+        tags = []
+        if host_name == "rhel8":
+            tags.append("pr")
         host = HOSTS[host_name]
         python = "3.13t"
         display_name = get_variant_name("Free-threaded", host, python=python)
-        variant = create_variant(tasks, display_name, python=python, host=host)
+        variant = create_variant(tasks, display_name, tags=tags, python=python, host=host)
         variants.append(variant)
     return variants
 
@@ -329,7 +332,7 @@ def create_atlas_data_lake_variants():
     tasks = [".test-no-orchestration"]
     expansions = dict(TEST_NAME="data_lake")
     display_name = get_variant_name("Atlas Data Lake", host)
-    return [create_variant(tasks, display_name, host=host, expansions=expansions)]
+    return [create_variant(tasks, display_name, tags=["pr"], host=host, expansions=expansions)]
 
 
 def create_mod_wsgi_variants():
@@ -370,9 +373,9 @@ def create_oidc_auth_variants():
     variants = []
     for host_name in ["ubuntu22", "macos", "win64"]:
         if host_name == "ubuntu22":
-            tasks = [".auth_oidc"]
+            tasks = [".auth_oidc_remote"]
         else:
-            tasks = [".auth_oidc !.auth_oidc_remote"]
+            tasks = ["!.auth_oidc_remote .auth_oidc"]
         host = HOSTS[host_name]
         variants.append(
             create_variant(
@@ -382,6 +385,18 @@ def create_oidc_auth_variants():
                 batchtime=BATCHTIME_WEEK,
             )
         )
+        # Add a specific local test to run on PRs.
+        if host_name == "ubuntu22":
+            tasks = ["!.auth_oidc_remote .auth_oidc"]
+            variants.append(
+                create_variant(
+                    tasks,
+                    get_variant_name("Auth OIDC Local", host),
+                    tags=["pr"],
+                    host=host,
+                    batchtime=BATCHTIME_WEEK,
+                )
+            )
     return variants
 
 
@@ -406,6 +421,7 @@ def create_mockupdb_variants():
             [".test-no-orchestration"],
             get_variant_name("MockupDB", host),
             host=host,
+            tags=["pr"],
             expansions=expansions,
         )
     ]
@@ -430,6 +446,7 @@ def create_atlas_connect_variants():
         create_variant(
             [".test-no-orchestration"],
             get_variant_name("Atlas connect", host),
+            tags=["pr"],
             host=DEFAULT_HOST,
         )
     ]
@@ -469,15 +486,18 @@ def create_aws_auth_variants():
     for host_name in ["ubuntu20", "win64", "macos"]:
         expansions = dict()
         tasks = [".auth-aws"]
+        tags = []
         if host_name == "macos":
             tasks = [".auth-aws !.auth-aws-web-identity !.auth-aws-ecs !.auth-aws-ec2"]
+            tags = ["pr"]
         elif host_name == "win64":
             tasks = [".auth-aws !.auth-aws-ecs"]
         host = HOSTS[host_name]
         variant = create_variant(
             tasks,
             get_variant_name("Auth AWS", host),
             host=host,
+            tags=tags,
             expansions=expansions,
         )
         variants.append(variant)
@@ -487,7 +507,7 @@ def create_aws_auth_variants():
 def create_no_server_variants():
     host = HOSTS["rhel8"]
     name = get_variant_name("No server", host=host)
-    return [create_variant([".test-no-orchestration"], name, host=host)]
+    return [create_variant([".test-no-orchestration"], name, host=host, tags=["pr"])]
 
 
 def create_alternative_hosts_variants():
@@ -512,14 +532,18 @@ def create_alternative_hosts_variants():
         expansions = dict(VERSION="latest")
         handle_c_ext(C_EXTS[0], expansions)
         host = HOSTS[host_name]
+        tags = []
         if "fips" in host_name.lower():
             expansions["REQUIRE_FIPS"] = "1"
+        if "amazon" in host_name.lower():
+            tags.append("pr")
         variants.append(
             create_variant(
                 [".test-no-toolchain"],
                 display_name=get_variant_name("Other hosts", host, version=version),
                 batchtime=batchtime,
                 host=host,
+                tags=tags,
                 expansions=expansions,
             )
         )
@@ -693,16 +717,18 @@ def create_kms_tasks():
         for success in [True, False]:
             name = f"test-{kms_type}kms"
             sub_test_name = kms_type
+            tags = []
             if not success:
                 name += "-fail"
                 sub_test_name += "-fail"
+                tags.append("pr")
             commands = []
             if not success:
                 commands.append(FunctionCall(func="run server"))
             test_vars = dict(TEST_NAME="kms", SUB_TEST_NAME=sub_test_name)
             test_func = FunctionCall(func="run tests", vars=test_vars)
             commands.append(test_func)
-            tasks.append(EvgTask(name=name, commands=commands))
+            tasks.append(EvgTask(name=name, tags=tags, commands=commands))
     return tasks
 
 
@@ -756,6 +782,7 @@ def create_oidc_tasks():
         if sub_test != "default":
             tags.append("auth_oidc_remote")
         tasks.append(EvgTask(name=task_name, tags=tags, commands=[test_func]))
+
     return tasks
 
 
@@ -802,6 +829,8 @@ def _create_ocsp_tasks(algo, variant, server_type, base_task_name):
         tags = ["ocsp", f"ocsp-{algo}", version]
         if "disableStapling" not in variant:
             tags.append("ocsp-staple")
+        if algo == "valid-cert-server-staples" and version == "latest":
+            tags.append("pr")
 
         task_name = get_task_name(
             f"test-ocsp-{algo}-{base_task_name}", python=python, version=version