torchdata/.github/workflows/ci.yml at main · tmbdev-archive/torchdata · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
name: Run DataPipes Tests
on:
  push:
    branches:
      - main
      - release/*
  pull_request:
    types: [opened, synchronize, reopened, labeled]
    branches:
      - main
      # For PR created by ghstack
      - gh/*/*/base
      - release/*

jobs:
  test:
    if:
      ${{ github.repository_owner == 'pytorch' && (github.event.action != 'labeled' ||
      startsWith(github.event.label.name, 'ciflow')) }}
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        os:
          - macos-latest
          - ubuntu-latest
          - windows-latest
        python-version:
          - 3.7
          - 3.8
          - 3.9
        with-s3:
          - 1
          - 0
    steps:
      - name: Setup additional system libraries
        if: startsWith( matrix.os, 'ubuntu' )
        run: |
          sudo add-apt-repository multiverse
          sudo apt update
          sudo apt install rar unrar libssl-dev libcurl4-openssl-dev zlib1g-dev
      - name: Setup Python ${{ matrix.python-version }}
        uses: actions/setup-python@v2
        with:
          python-version: ${{ matrix.python-version }}
      - name: Setup msbuild on Windows
        if: matrix.with-s3 == 1 && matrix.os == 'windows-latest'
        uses: microsoft/setup-msbuild@v1.1
      - name: Set up Visual Studio shell
        if: matrix.with-s3 == 1 && matrix.os == 'windows-latest'
        uses: egor-tensin/vs-shell@v2
        with:
          arch: x64
      - name: Check out source repository
        uses: actions/checkout@v2
      - name: Install dependencies
        run: |
          pip3 install -r requirements.txt
          pip3 install --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html
          pip3 install cmake ninja
          echo "/home/runner/.local/bin" >> $GITHUB_PATH
      - name: Install AWS-SDK-CPP on Windows for S3 IO datapipes
        if: matrix.with-s3 == 1 && matrix.os == 'windows-latest'
        run: |
          git clone --recurse-submodules https://github.com/aws/aws-sdk-cpp
          cd aws-sdk-cpp
          mkdir sdk-lib
          cmake -S . -B build -GNinja -DBUILD_ONLY="s3;transfer" -DBUILD_SHARED_LIBS=OFF -DENABLE_TESTING=OFF -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=sdk-lib
          cmake --build build --config Release
          cmake --install build --config Release
      - name: Install AWS-SDK-CPP on Non-Windows for S3 IO datapipes
        if: matrix.with-s3 == 1 && matrix.os != 'windows-latest'
        run: |
          git clone --recurse-submodules https://github.com/aws/aws-sdk-cpp
          cd aws-sdk-cpp/
          mkdir sdk-build sdk-lib
          cd sdk-build
          cmake .. -DCMAKE_BUILD_TYPE=Release -DBUILD_ONLY="s3;transfer" -DENABLE_TESTING=OFF -DCMAKE_INSTALL_PREFIX=../sdk-lib
          make
          sudo make install
      - name: Export AWS-SDK-CPP
        if: matrix.with-s3 == 1
        shell: bash
        run: |
          if [[ ${{ matrix.os }} == 'windows-latest' ]]; then
            AWSSDK_PATH="$GITHUB_WORKSPACE\\aws-sdk-cpp\\sdk-lib"
          else
            AWSSDK_PATH="$GITHUB_WORKSPACE/aws-sdk-cpp/sdk-lib"
          fi
          echo "::set-output name=awssdk::$AWSSDK_PATH"
        id: export_path
      - name: Build TorchData
        run: |
          python setup.py install
        env:
          BUILD_S3: ${{ matrix.with-s3 }}
          AWSSDK_DIR: ${{ steps.export_path.outputs.awssdk }}
      - name: Install test requirements
        run: pip3 install expecttest fsspec iopath==0.1.9 numpy pytest rarfile protobuf
      - name: Run DataPipes tests with pytest
        if: ${{ ! contains(github.event.pull_request.labels.*.name, 'ciflow/slow') }}
        run:
          pytest --no-header -v test --ignore=test/test_period.py --ignore=test/test_text_examples.py
          --ignore=test/test_audio_examples.py
      - name: Run DataPipes tests with pytest (including slow tests)
        if: ${{ contains(github.event.pull_request.labels.*.name, 'ciflow/slow') }}
        run:
          pytest --no-header -v test --ignore=test/test_period.py --ignore=test/test_text_examples.py
          --ignore=test/test_audio_examples.py
        env:
          PYTORCH_TEST_WITH_SLOW: 1
      - name: Run DataPipes period tests with pytest
        if: ${{ contains(github.event.pull_request.labels.*.name, 'ciflow/period') }}
        run:
          pytest --no-header -v test/test_period.py --ignore=test/test_text_examples.py
          --ignore=test/test_audio_examples.py
        env:
          PYTORCH_TEST_WITH_SLOW: 1