build.yml

#
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT license.
#

# run on CI or once per day at 8 pm PST
trigger:
- master

schedules:
- cron: "0 4 * * *"
  displayName: Daily Build
  branches:
    include:
    - master
  always: true

pr:
- master

resources:
- repo: self
  clean: true

jobs:
- job: Build
  pool:
    name: $(WindowsPipelineName)
  timeoutInMinutes: 60
  variables:
    ArtifactOut: $(Build.ArtifactStagingDirectory)/Out/JavaScript
  steps:
  - task: UseNode@1
    displayName: Use Node.js 20
    inputs:
      version: '20.x'
  - bash: ./ci/check-git-head.sh
    displayName: Repository checks
  - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0
    displayName: Component Detection
    condition: >
      and(eq(variables['System.CollectionId'], '26f8e8b1-373f-4f65-96fc-d17a59b38306'),
        eq(variables['System.DefinitionId'], '198'),
        eq(variables['Build.SourceBranch'], 'refs/heads/master'),
        or(eq(variables['Build.Reason'], 'Schedule'), eq(variables['Build.Reason'], 'Manual')))
  - bash: |
      set -euo pipefail
      npm ci
      ./ci/set-version.sh
      echo "COLLECTION_ID=$(System.CollectionId)"
      echo "DEFINITION_ID=$(System.DefinitionId)"
    displayName: Install packages and set version from package.json
  - bash: |
      ./ci/update-version-in-code.sh
    displayName: Update version in source code
  - bash: |
      ./ci/update-package-version.sh
    displayName: Update version in package.json
  - bash: npm pack
    displayName: Build and pack SDK
  - bash: |
      set -u -e -o pipefail -x
      PACKAGE_BASE=microsoft-cognitiveservices-speech-sdk
      PACKAGE_NAME=$PACKAGE_BASE-$SPEECHSDK_SEMVER2NOMETA.tgz
      PACKAGE_IN=$PACKAGE_NAME
      PACKAGE_OUT="$(ArtifactOut)/npm"
      ZIP_OUT="$(ArtifactOut)/SpeechSDK-JavaScript-$SPEECHSDK_SEMVER2NOMETA"
      mkdir -p "$PACKAGE_OUT" "$ZIP_OUT"
      cp --preserve "$PACKAGE_IN" "$PACKAGE_OUT"
      echo SRI hash for microsoft.cognitiveservices.speech.sdk.bundle.js: sha512-"$(openssl dgst -sha512 -binary distrib/browser/microsoft.cognitiveservices.speech.sdk.bundle.js | openssl base64 -A)"
      cp --preserve LICENSE REDIST.txt NOTICE.txt distrib/browser/microsoft.cognitiveservices.speech.sdk.bundle.* distrib/browser/microsoft.cognitiveservices.speech.sdk.bundle-min.* distrib/browser/speech-processor.js "$ZIP_OUT"
    displayName: Create drop
  - task: ArchiveFiles@2
    inputs:
      rootFolderOrFile: $(ArtifactOut)/SpeechSDK-JavaScript-$(SPEECHSDK_SEMVER2NOMETA)
      includeRootFolder: true
      archiveType: zip
      archiveFile: $(ArtifactOut)/SpeechSDK-JavaScript-$(SPEECHSDK_SEMVER2NOMETA).zip
    displayName: Create .zip
  - bash: rm -rf "$(ArtifactOut)/SpeechSDK-JavaScript-$(SPEECHSDK_SEMVER2NOMETA)"
    displayName: Remove temporary directory
  - task: PublishBuildArtifacts@1
    displayName: Publish drop
    inputs:
      PathtoPublish: $(ArtifactOut)
      ArtifactName: JavaScript
      publishLocation: Container
  - bash: "echo '##vso[task.setvariable variable=SPEECHSDK_RUN_TESTS]false'"
    condition: or(failed(), canceled())
    displayName: Skip tests on build failure
- job : RunTest
  pool:
    name: $(WindowsPipelineName)
  timeoutInMinutes: 60
  condition: and(succeeded(), eq(variables['SPEECHSDK_RUN_TESTS'], 'true'))
  variables:
    SPEECHSDK_RUN_TESTS: true
  dependsOn: Build
  steps:
  - task: UseNode@1
    displayName: Use Node.js 20
    inputs:
      version: '20.x'
  - bash: ./ci/check-git-head.sh
    displayName: Repository checks
  - bash: |
      set -euo pipefail
      npm ci
      ./ci/set-version.sh
      echo "COLLECTION_ID=$(System.CollectionId)"
      echo "DEFINITION_ID=$(System.DefinitionId)"
    displayName: Install packages and set version from package.json
  - bash: |
      ./ci/update-version-in-code.sh
    displayName: Update version in source code
  - template: generate-subscription-file.yml
  - bash: |
      set -u -e -x -o pipefail
      npm run test:non-connection
    displayName: Run tests
    condition: and(succeeded(), eq(variables['SPEECHSDK_RUN_TESTS'], 'true'))
  - task: PublishTestResults@2
    displayName: Publish test results
    inputs:
      testRunner: JUnit
      testResultsFiles: 'test-javascript-junit.xml'
      failTaskOnFailedTests: true
    condition: and(succeededOrFailed(), eq(variables['SPEECHSDK_RUN_TESTS'], 'true'))
  - task: DownloadBuildArtifacts@1
    displayName: Download artifacts
    inputs:
      buildType: current
      downloadPath: $(Pipeline.Workspace)
      artifactName: JavaScript
  - bash: |
        set -e
        cd tests/packaging
        echo "SDK version = $SPEECHSDK_SEMVER2NOMETA"
        npm ci
        PACKAGE_PATH="$(Pipeline.Workspace)/JavaScript/npm/microsoft-cognitiveservices-speech-sdk-$SPEECHSDK_SEMVER2NOMETA.tgz"
        # Convert Windows path to proper format
        PACKAGE_PATH=$(echo $PACKAGE_PATH | sed 's/\\/\//g')
        npm install "$PACKAGE_PATH"

        npm run bundle
    displayName: Run Test Bundle
    condition: and(succeeded(), eq(variables['SPEECHSDK_RUN_TESTS'], 'true'))
- job: RunConnectionTests
  dependsOn: Build
  pool:
    name: CarbonUbuntu2204Hosted
  timeoutInMinutes: 60
  condition: and(succeeded(), eq(variables['SPEECHSDK_RUN_TESTS'], 'true'))
  variables:
    SPEECHSDK_RUN_TESTS: true
  steps:
  - task: UseNode@1
    displayName: Use Node.js 20
    inputs:
      version: '20.x'
  - bash: ./ci/check-git-head.sh
    displayName: Repository checks
  - bash: |
      set -euo pipefail
      npm ci
      ./ci/set-version.sh
      echo "COLLECTION_ID=$(System.CollectionId)"
      echo "DEFINITION_ID=$(System.DefinitionId)"
    displayName: Install packages and set version from package.json
  - bash: |
      ./ci/update-version-in-code.sh
    displayName: Update version in source code
  - template: generate-subscription-file.yml
  - bash: |
      set -euo pipefail
      # Move Docker and containerd data directories to the larger /mnt disk to avoid
      # out-of-space errors when pulling images. Docker's data-root alone is not enough
      # because containerd stores image layers and content under /var/lib/containerd/
      # on the (small) OS root disk.
      NEW_DOCKER_DIR="/mnt/docker"
      DOCKER_CONFIG_FILE="/etc/docker/daemon.json"
      NEW_CONTAINERD_ROOT="/mnt/containerd"
      CONTAINERD_CONFIG_FILE="/etc/containerd/config.toml"

      # Only relocate if not already done
      CURRENT_ROOT=$(docker info 2>/dev/null | grep "Docker Root Dir" | awk '{print $4}')
      if [ "$CURRENT_ROOT" = "$NEW_DOCKER_DIR/docker" ]; then
        echo "Docker data already at $CURRENT_ROOT – skipping relocation."
        exit 0
      fi

      echo "Relocating Docker and containerd data to /mnt..."
      sudo systemctl stop docker
      sudo systemctl stop containerd || true

      # --- Docker data-root ---
      sudo mkdir -p "$NEW_DOCKER_DIR"
      sudo mkdir -p "$(dirname "$DOCKER_CONFIG_FILE")"
      sudo touch "$DOCKER_CONFIG_FILE"
      sudo chmod 666 "$DOCKER_CONFIG_FILE"
      cat <<EOL | sudo tee "$DOCKER_CONFIG_FILE"
      {
        "data-root": "$NEW_DOCKER_DIR/docker"
      }
      EOL
      sudo chmod 644 "$DOCKER_CONFIG_FILE"

      # --- containerd root (image layers + content store) ---
      sudo mkdir -p "$NEW_CONTAINERD_ROOT"
      sudo mkdir -p "$(dirname "$CONTAINERD_CONFIG_FILE")"
      if [ ! -f "$CONTAINERD_CONFIG_FILE" ]; then
        containerd config default | sudo tee "$CONTAINERD_CONFIG_FILE" > /dev/null
      fi
      if grep -q '^root\s*=' "$CONTAINERD_CONFIG_FILE"; then
        sudo sed -i "s|^root\s*=.*|root = \"$NEW_CONTAINERD_ROOT\"|" "$CONTAINERD_CONFIG_FILE"
      else
        sudo sed -i "1i root = \"$NEW_CONTAINERD_ROOT\"" "$CONTAINERD_CONFIG_FILE"
      fi

      sudo systemctl restart containerd || true
      sudo systemctl restart docker

      DOCKER_ROOT_DIR=$(docker info | grep "Docker Root Dir" | awk '{print $4}')
      if [ "$DOCKER_ROOT_DIR" == "$NEW_DOCKER_DIR/docker" ]; then
        echo "Docker data successfully moved to $DOCKER_ROOT_DIR"
      else
        echo "Error: Docker data directory is not set correctly."
        exit 1
      fi
      echo "containerd root set to: $NEW_CONTAINERD_ROOT"
      df -h / /mnt
    displayName: (Docker) Relocate data to larger disk
  - template: get-docker-image.yml
    parameters:
      DockerImages: $(DOCKER_IMAGE) mcr.microsoft.com/azure-cognitive-services/speechservices/neural-text-to-speech:latest mcr.microsoft.com/azure-cognitive-services/speechservices/speech-to-text:latest mcr.microsoft.com/azure-cognitive-services/speechservices/language-detection:latest
  - bash: |
      set -u -e -x -o pipefail
      . ci/functions.sh
      . ci/load-build-secrets.sh

      tts_port=$(find_free_port)
      echo "TTS Port: $tts_port"

      host_ip=$(ip -4 addr show docker0 | grep -Po 'inet \K[\d.]+')
      echo "Host IP: $host_ip"

      TTS_URL="ws://$host_ip:$tts_port"
      vsts_setvar TTS_CONTAINER_URL "$TTS_URL"

      docker create -p $tts_port:5000 --memory 14g --cpus 4 --name tts_container mcr.microsoft.com/azure-cognitive-services/speechservices/neural-text-to-speech:latest EULA=accept Billing=https://$SPEECHSDK_SPEECH_REGION.api.cognitive.microsoft.com/ ApiKey=$SPEECHSDK_SPEECH_KEY
      docker start tts_container
    displayName: Create and start TTS Container
  - bash: |
      set -u -e -x -o pipefail
      . ci/functions.sh
      . ci/load-build-secrets.sh

      sr_port=$(find_free_port)
      echo "SR Port: $sr_port"

      host_ip=$(ip -4 addr show docker0 | grep -Po 'inet \K[\d.]+')
      echo "Host IP: $host_ip"
      SR_URL="ws://$host_ip:$sr_port"
      vsts_setvar SR_CONTAINER_URL "$SR_URL"

      docker create -p $sr_port:5000 --memory 6g --cpus 4 --name sr_container mcr.microsoft.com/azure-cognitive-services/speechservices/speech-to-text:latest EULA=accept Billing=https://$SPEECHSDK_SPEECH_REGION.api.cognitive.microsoft.com/ ApiKey=$SPEECHSDK_SPEECH_KEY
      docker start sr_container
    displayName: Create and start ASR Container
  - bash: |
      set -u -e -x -o pipefail
      . ci/functions.sh
      . ci/load-build-secrets.sh

      lid_port=$(find_free_port)
      echo "LID Port: $lid_port"

      host_ip=$(ip -4 addr show docker0 | grep -Po 'inet \K[\d.]+')
      echo "Host IP: $host_ip"
      LID_URL="ws://$host_ip:$lid_port"
      vsts_setvar LID_CONTAINER_URL "$LID_URL"

      docker create -p $lid_port:5003   --memory 1g --cpus 4 --name lid_container mcr.microsoft.com/azure-cognitive-services/speechservices/language-detection:latest EULA=accept Billing=https://$SPEECHSDK_SPEECH_REGION.api.cognitive.microsoft.com/ ApiKey=$SPEECHSDK_SPEECH_KEY
      docker start lid_container
    displayName: Create and start LID Container
  - bash: |
      set -u -e -x -o pipefail
      export ENABLE_PRIVATE_LINK_TESTS=true
      npm run test:connection
    displayName: Run tests
    condition: and(succeeded(), eq(variables['SPEECHSDK_RUN_TESTS'], 'true'))
  - task: PublishTestResults@2
    displayName: Publish test results
    inputs:
      testRunner: JUnit
      testResultsFiles: 'test-javascript-junit.xml'
      failTaskOnFailedTests: true
    condition: and(succeededOrFailed(), eq(variables['SPEECHSDK_RUN_TESTS'], 'true'))
  - bash: |
      set -u -e -x -o pipefail

      docker stop tts_container
      docker stop sr_container
      docker stop lid_container

      docker logs tts_container > tts_container.log
      ls -l tts_container.log

      docker logs sr_container > sr_container.log
      ls -l tts_container.log

      docker logs lid_container > lid_container.log
      ls -l tts_container.log

      docker rm tts_container
      docker rm sr_container
      docker rm lid_container

      DOCKER_LOG_DIR=$(Build.ArtifactStagingDirectory)/DockerLogs
      mkdir -p $DOCKER_LOG_DIR
      # Copy logs to the artifacts directory
      cp tts_container.log $DOCKER_LOG_DIR/tts_container.log
      cp sr_container.log $DOCKER_LOG_DIR/sr_container.log
      cp lid_container.log $DOCKER_LOG_DIR/lid_container.log
    displayName: Stop Containers and export logs
    condition: and(succeededOrFailed(), eq(variables['SPEECHSDK_RUN_TESTS'], 'true'))
  - task: ArchiveFiles@2
    displayName: Archive Docker container logs
    inputs:
      rootFolderOrFile: '$(Build.ArtifactStagingDirectory)/DockerLogs'
      includeRootFolder: false
      archiveFile: '*.*'
      replaceExistingArchive: false
    condition: and(succeededOrFailed(), eq(variables['SPEECHSDK_RUN_TESTS'], 'true'))
  - task: PublishBuildArtifacts@1
    retryCountOnTaskFailure: 5
    inputs:
      PathtoPublish: '$(Build.ArtifactStagingDirectory)/DockerLogs'
      ArtifactName: TestRunBackup
    condition: and(succeededOrFailed(), eq(variables['SPEECHSDK_RUN_TESTS'], 'true'))
    displayName: Publish Docker logs