diff --git a/tools/pipelines/templates/include-conditionally-run-stress-tests.yml b/tools/pipelines/templates/include-conditionally-run-stress-tests.yml index 22d45092cce7..b2c446a0f5d7 100644 --- a/tools/pipelines/templates/include-conditionally-run-stress-tests.yml +++ b/tools/pipelines/templates/include-conditionally-run-stress-tests.yml @@ -30,6 +30,14 @@ parameters: - name: artifactBuildId type: string +# A valid git ref to use for the microsoft/FluidFramework repository checkout (e.g. refs/heads/main, a commit SHA, tag, etc) +# Pipelines that are triggered on completion of the "Build - client packages" pipeline should generally use +# the same source commit as what was used for "Build - client packages". This is typically available via +# `$(resources.pipeline.client.sourceCommit)` (where "client" is actually the name of the resource declared +# in the current pipeline's root yaml file) +- name: fluidFrameworkSourceVersion + type: string + stages: - stage: CheckAffectedPaths displayName: Determine changed packages @@ -73,6 +81,7 @@ stages: condition: eq(dependencies.CheckAffectedPaths.outputs['Job.Check${{ replace(package.testFileTarName, '-', '') }}.AffectedFilesModified'],'true') poolBuild: ${{ parameters.pool }} artifactBuildId: ${{ parameters.artifactBuildId }} + fluidFrameworkSourceVersion: ${{ parameters.fluidFrameworkSourceVersion }} testPackage: ${{ package.name }} timeoutInMinutes: 120 testFileTarName: ${{ package.testFileTarName }} diff --git a/tools/pipelines/templates/include-install-pnpm.yml b/tools/pipelines/templates/include-install-pnpm.yml index 23f3509f4af4..6207eb48abe4 100644 --- a/tools/pipelines/templates/include-install-pnpm.yml +++ b/tools/pipelines/templates/include-install-pnpm.yml @@ -31,6 +31,17 @@ parameters: type: string default: $(ado-feeds-primary-registry) +# An optional extra segment appended to the pnpm store cache key (separated by |). +# Note that Azure pipelines interpret unquoted cache keys as files rather than literal strings. E.g. the following specification: +# +# additionalCacheKey: 'foo | "bar"' +# +# will use the hashed contents of the file "foo" (hashing nothing if that file does not exist) as well as the string "bar". +# Use this parameter if the set of dependencies to be installed may be be better distinguished using files other than the root pnpm-lock. +- name: additionalCacheKey + type: string + default: '' + steps: - ${{ if eq(parameters.enableCache, true) }}: - task: Cache@2 @@ -43,7 +54,10 @@ steps: inputs: # Caches are already scoped to individual pipelines, so no need to include the release group name or tag # in the cache key - key: '"pnpm-store" | "$(Agent.OS)" | $(Pipeline.Workspace)/${{ parameters.buildDirectory }}/pnpm-lock.yaml' + ${{ if ne(parameters.additionalCacheKey, '') }}: + key: '"pnpm-store" | "$(Agent.OS)" | $(Pipeline.Workspace)/${{ parameters.buildDirectory }}/pnpm-lock.yaml | ${{ parameters.additionalCacheKey }}' + ${{ else }}: + key: '"pnpm-store" | "$(Agent.OS)" | $(Pipeline.Workspace)/${{ parameters.buildDirectory }}/pnpm-lock.yaml' path: ${{ parameters.pnpmStorePath }} restoreKeys: | "pnpm-store" | "$(Agent.OS)" diff --git a/tools/pipelines/templates/include-test-real-service.yml b/tools/pipelines/templates/include-test-real-service.yml index 1faeeceb0c7f..6ebd3b694398 100644 --- a/tools/pipelines/templates/include-test-real-service.yml +++ b/tools/pipelines/templates/include-test-real-service.yml @@ -71,6 +71,15 @@ parameters: - name: artifactBuildId type: string +# A valid git ref to use for the microsoft/FluidFramework repository checkout (e.g. refs/heads/main, a commit SHA, tag, etc) +# Pipelines that are triggered on completion of the "Build - client packages" pipeline should generally use +# the same source commit as what was used for "Build - client packages". This is typically available via +# `$(resources.pipeline.client.sourceCommit)` (where "client" is actually the name of the resource declared +# in the current pipeline's root yaml file) +- name: fluidFrameworkSourceVersion + type: string + default: $(Build.SourceVersion) + # Name of the Secure File that contains the self-signed cert for the R11s deployment. # If not blank, the pipeline will try to install it to the local cert store. - name: r11sSelfSignedCertSecureFile @@ -88,11 +97,6 @@ parameters: type: boolean default: false -# If true, the versions of our packages installed for compat testing will be cached. -- name: cacheCompatVersionsInstalls - type: boolean - default: false - # If tests need to run against an ODSP tenant, specify the type of tenant here. Steps will be injected to lease one for the duration of the test. - name: odspTenantType type: string @@ -174,6 +178,22 @@ stages: path: $(FFPipelineHostDirectory) clean: true + # ADO doesn't support checking out refs that are determined at pipeline queue time as ours likely would be + # (where the ref is a function of one of the pipeline resource inputs). See docs here: + # https://learn.microsoft.com/en-us/azure/devops/pipelines/repos/multi-repo-checkout?view=azure-devops#checking-out-a-specific-ref + # To work around this, we get to the right revision in a follow-up step. + - checkout: self + path: $(FluidFrameworkDirectory) + clean: true + + - bash: | + git fetch --depth=1 origin $FF_SOURCE_VERSION + git checkout $FF_SOURCE_VERSION + workingDirectory: $(Pipeline.Workspace)/$(FluidFrameworkDirectory) + displayName: Switch FluidFramework checkout to artifact source version + env: + FF_SOURCE_VERSION: ${{ parameters.fluidFrameworkSourceVersion }} + # Install self-signed cert for R11s deployment in local cert store - ${{ if ne(parameters.r11sSelfSignedCertSecureFile, '') }}: - task: DownloadSecureFile@1 @@ -231,15 +251,15 @@ stages: - template: /tools/pipelines/templates/include-install-pnpm.yml@self parameters: buildDirectory: $(FFPipelineHostDirectory) + # The compat-workspaces lockfile is committed to the FluidFramework repo and changes + # whenever legacy package versions are added or removed. Including it in the pnpm store + # cache key ensures a cache miss whenever the set of compat packages changes. + additionalCacheKey: '$(Pipeline.Workspace)/$(FluidFrameworkDirectory)/packages/test/test-version-utils/compat-workspaces/full/pnpm-lock.yaml' - template: /tools/pipelines/templates/include-setup-npmrc-for-download.yml@self # Set up Docker environment if running against docker - ${{ if eq(parameters.runAgainstDocker, true) }}: - # Checks out FluidFramework repo - - checkout: self - path: $(FluidFrameworkDirectory) - clean: true - task: Docker@2 displayName: 'Login to container registry' inputs: @@ -371,43 +391,6 @@ stages: sourcePackageLocation: $(Pipeline.Workspace)/$(FFPipelineHostDirectory)/node_modules/${{ parameters.testPackage }} destPackageLocation: $(Pipeline.Workspace)/$(FFPipelineHostDirectory) - - ${{ if eq(parameters.cacheCompatVersionsInstalls, true) }}: - - task: Bash@3 - displayName: Compute compat versions install location and version - inputs: - targetType: 'inline' - workingDirectory: $(Pipeline.Workspace)/$(FFPipelineHostDirectory)/node_modules/@fluid-private/test-end-to-end-tests - # Using import.meta.resolve to compute this is more resilient to different install tree types. - # Also note that test-version-utils is esm-only, so cannot be loaded with require. - script: | - set -eu -o pipefail - node --input-type=module -e " - import { createRequire } from 'module'; - const require = createRequire(import.meta.url); - const path = require('path'); - const versionUtilsIndexUrl = await import.meta.resolve('@fluid-private/test-version-utils'); - const basePath = new URL(versionUtilsIndexUrl.replace('/lib/index.js', '')).pathname; - const legacyModulesPath = path.join(basePath, 'node_modules', '.legacy'); - console.log('Resolved @fluid-private/test-version-utils legacy modules to ' + legacyModulesPath); - console.log('##vso[task.setvariable variable=compatVersionInstallsPath]' + legacyModulesPath); - const packageJsonPath = path.join(basePath, 'package.json'); - const packageJson = require(packageJsonPath); - const packageVersion = packageJson.version; - const [major, minor] = packageVersion.split('.'); - const cacheKey = 'major:' + major + ',minor:' + minor; - console.log('Computed compat version cache key: ' + cacheKey); - console.log('##vso[task.setvariable variable=compatVersionCacheKey;]'+ cacheKey); - " - - - task: Cache@2 - displayName: Cache compat versions install location - timeoutInMinutes: 3 - continueOnError: true - inputs: - # Increment the number on the cache-bust key (no special semantics) to force a new cache key. - key: '"compat-version-installs" | "cache-bust-1" | "$(Agent.OS)" | "${{ parameters.testCommand }}" | "${{ variant.name }}" | "$(compatVersionCacheKey)"' - path: $(compatVersionInstallsPath) - # Only check out tenants from the tenant pool if we are running tests against ODSP - ${{ if ne(parameters.odspTenantType, 'none') }}: # Retrieve a tenant from the tenant pool @@ -429,6 +412,7 @@ stages: SYSTEM_ACCESSTOKEN: $(System.AccessToken) inputs: targetType: 'inline' + workingDirectory: $(Pipeline.Workspace)/$(FFPipelineHostDirectory) script: | set -eu -o pipefail @@ -528,6 +512,7 @@ stages: SYSTEM_ACCESSTOKEN: $(System.AccessToken) inputs: targetType: 'inline' + workingDirectory: $(Pipeline.Workspace)/$(FFPipelineHostDirectory) script: | set -eu -o pipefail pnpm exec trips-cleanup --odspEndpoint=${{ parameters.odspTenantType }} --useFic diff --git a/tools/pipelines/test-dds-stress.yml b/tools/pipelines/test-dds-stress.yml index ff98abacb2cd..af6dd7f2a5d0 100644 --- a/tools/pipelines/test-dds-stress.yml +++ b/tools/pipelines/test-dds-stress.yml @@ -43,6 +43,7 @@ stages: - template: /tools/pipelines/templates/include-conditionally-run-stress-tests.yml@self parameters: artifactBuildId: $(resources.pipeline.client.runID) + fluidFrameworkSourceVersion: $(resources.pipeline.client.sourceCommit) packages: - name: "@fluidframework/sequence" affectedPaths: diff --git a/tools/pipelines/test-real-service-stress.yml b/tools/pipelines/test-real-service-stress.yml index 89e4844c3299..20ca4f0b8fd6 100644 --- a/tools/pipelines/test-real-service-stress.yml +++ b/tools/pipelines/test-real-service-stress.yml @@ -47,6 +47,7 @@ stages: poolBuild: Large-eastus2 testPackage: ${{ variables.testPackage }} artifactBuildId: $(resources.pipeline.client.runID) + fluidFrameworkSourceVersion: $(resources.pipeline.client.sourceCommit) timeoutInMinutes: 120 testCommand: start:odsp skipTestResultPublishing: true @@ -64,6 +65,7 @@ stages: poolBuild: Large-eastus2 testPackage: ${{ variables.testPackage }} artifactBuildId: $(resources.pipeline.client.runID) + fluidFrameworkSourceVersion: $(resources.pipeline.client.sourceCommit) timeoutInMinutes: 120 testCommand: start:odspdf skipTestResultPublishing: true @@ -80,6 +82,7 @@ stages: poolBuild: Large-eastus2 testPackage: ${{ variables.testPackage }} artifactBuildId: $(resources.pipeline.client.runID) + fluidFrameworkSourceVersion: $(resources.pipeline.client.sourceCommit) timeoutInMinutes: 120 testCommand: start:t9s skipTestResultPublishing: true @@ -99,6 +102,7 @@ stages: poolBuild: Large-eastus2 testPackage: ${{ variables.testPackage }} artifactBuildId: $(resources.pipeline.client.runID) + fluidFrameworkSourceVersion: $(resources.pipeline.client.sourceCommit) timeoutInMinutes: 120 testCommand: start:frs lockBehavior: sequential @@ -119,6 +123,7 @@ stages: poolBuild: Large-eastus2 testPackage: ${{ variables.testPackage }} artifactBuildId: $(resources.pipeline.client.runID) + fluidFrameworkSourceVersion: $(resources.pipeline.client.sourceCommit) timeoutInMinutes: 120 testCommand: start:frs:canary lockBehavior: sequential diff --git a/tools/pipelines/test-real-service.yml b/tools/pipelines/test-real-service.yml index 4c0575c802e1..fda39698becd 100644 --- a/tools/pipelines/test-real-service.yml +++ b/tools/pipelines/test-real-service.yml @@ -58,8 +58,9 @@ stages: poolBuild: Large-eastus2 # Need Large pool for full-compat matrix testPackage: ${{ variables.testPackage }} artifactBuildId: $(resources.pipeline.client.runID) + fluidFrameworkSourceVersion: $(resources.pipeline.client.sourceCommit) testCommand: test:realsvc:local:report:full - cacheCompatVersionsInstalls: true + uploadTestPassRateTelemetry: true pipelineIdentifierForTelemetry: ${{ variables.pipelineIdentifierForTelemetry }} env: @@ -74,8 +75,9 @@ stages: poolBuild: Large-eastus2 # Need Large pool for full-compat matrix testPackage: ${{ variables.testPackage }} artifactBuildId: $(resources.pipeline.client.runID) + fluidFrameworkSourceVersion: $(resources.pipeline.client.sourceCommit) testCommand: test:realsvc:tinylicious:report:full - cacheCompatVersionsInstalls: true + # TODO: AB#8968 tracks figuring out the root cause of the extended delay, and restoring this timeout to 90m or less timeoutInMinutes: 120 uploadTestPassRateTelemetry: true @@ -95,6 +97,7 @@ stages: poolBuild: Large-eastus2 # Need Large pool for full-compat matrix testPackage: ${{ variables.testPackage }} artifactBuildId: $(resources.pipeline.client.runID) + fluidFrameworkSourceVersion: $(resources.pipeline.client.sourceCommit) continueOnError: true timeoutInMinutes: 360 testCommand: test:realsvc:r11s:docker @@ -109,7 +112,7 @@ stages: flags: --compatVersion=OCV - name: Cross-Client flags: --compatVersion=CROSS_CLIENT - cacheCompatVersionsInstalls: true + uploadTestPassRateTelemetry: true pipelineIdentifierForTelemetry: ${{ variables.pipelineIdentifierForTelemetry }} @@ -134,6 +137,7 @@ stages: poolBuild: Small-eastus2 testPackage: ${{ variables.testPackage }} artifactBuildId: $(resources.pipeline.client.runID) + fluidFrameworkSourceVersion: $(resources.pipeline.client.sourceCommit) timeoutInMinutes: 360 continueOnError: true testCommand: test:realsvc:frs:report @@ -150,7 +154,7 @@ stages: flags: --compatVersion=OCV - name: Cross-Client flags: --compatVersion=CROSS_CLIENT - cacheCompatVersionsInstalls: true + uploadTestPassRateTelemetry: true pipelineIdentifierForTelemetry: ${{ variables.pipelineIdentifierForTelemetry }} env: @@ -167,6 +171,7 @@ stages: poolBuild: Small-eastus2 testPackage: ${{ variables.testPackage }} artifactBuildId: $(resources.pipeline.client.runID) + fluidFrameworkSourceVersion: $(resources.pipeline.client.sourceCommit) timeoutInMinutes: 360 continueOnError: true testCommand: test:realsvc:odsp:report @@ -182,7 +187,7 @@ stages: # Tests N-2 to LTS+3 back compat for loader + driver - name: N-2ToLTS+1-back-compat flags: --compatVersion=V2_INT_3 --tenantIndex=3 - cacheCompatVersionsInstalls: true + uploadTestPassRateTelemetry: true pipelineIdentifierForTelemetry: ${{ variables.pipelineIdentifierForTelemetry }} env: diff --git a/tools/pipelines/test-service-clients.yml b/tools/pipelines/test-service-clients.yml index 9a7aec43aa58..56dfea9b634b 100644 --- a/tools/pipelines/test-service-clients.yml +++ b/tools/pipelines/test-service-clients.yml @@ -52,6 +52,7 @@ stages: testPackage: ${{ variables.testPackage }} testCommand: test:realsvc:azure artifactBuildId: $(resources.pipeline.client.runID) + fluidFrameworkSourceVersion: $(resources.pipeline.client.sourceCommit) pipelineIdentifierForTelemetry: ${{ variables.pipelineIdentifierForTelemetry }} env: FLUID_TEST_SCALE: true # run longer duration tests @@ -69,6 +70,7 @@ stages: testPackage: ${{ variables.testPackage }} testCommand: test:realsvc:tinylicious artifactBuildId: $(resources.pipeline.client.runID) + fluidFrameworkSourceVersion: $(resources.pipeline.client.sourceCommit) pipelineIdentifierForTelemetry: ${{ variables.pipelineIdentifierForTelemetry }} env: FLUID_TEST_SCALE: true # run longer duration tests @@ -88,6 +90,7 @@ stages: testPackage: ${{ variables.testOdspPackage }} testCommand: test:realsvc:odsp artifactBuildId: $(resources.pipeline.client.runID) + fluidFrameworkSourceVersion: $(resources.pipeline.client.sourceCommit) pipelineIdentifierForTelemetry: ${{ variables.pipelineIdentifierForTelemetry }} env: FLUID_TEST_SCALE: true # run longer duration tests