From 0c76b1cdadf4d393e2f71b5dc0554b389f5fa803 Mon Sep 17 00:00:00 2001 From: Prasanth Nunna Date: Fri, 16 Aug 2024 18:59:19 +0000 Subject: [PATCH 1/8] Add messages for Heuristics error --- csrc/ops.hip | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/csrc/ops.hip b/csrc/ops.hip index 157e84629..5c0688b91 100644 --- a/csrc/ops.hip +++ b/csrc/ops.hip @@ -576,6 +576,7 @@ template int igemmlt(hipblasLtHandl if (returnedAlgoCount == 0) { has_error = 1; + printf("Error: Matmul Algo Heurisitic didn't return algorithms\n"); } else { @@ -614,18 +615,26 @@ template int igemmlt(hipblasLtHandl heuristicResult, &returnedAlgoCount)); - if(!SCALE_ROWS) + if (returnedAlgoCount == 0) { - float alpha = 1.0f, beta = 0.0f; - - has_error |= checkHipblasStatus(hipblasLtMatmul(ltHandle, matmulDesc,&alpha, A, Adesc, B, Bdesc, &beta, (int8_t*)C, Cdesc, (int8_t*)C, Cdesc, &heuristicResult[0].algo, nullptr, 0, 0)); + has_error = 1; + printf("Error: Matmul Algo Heurisitic didn't return algorithms\n"); } else { - //has_error |= checkHipblasStatus(hipblasLtMatmulDescSetAttribute(matmulDesc, hipblasLt_MATMUL_DESC_POINTER_MODE, &alphaVec, sizeof(alphaVec))); - float beta = 0.0f; - - has_error |= checkHipblasStatus(hipblasLtMatmul(ltHandle, matmulDesc, row_scale, A, Adesc, B, Bdesc, &beta, (int8_t*)C, Cdesc, (int8_t*)C, Cdesc, &heuristicResult[0].algo, nullptr, 0, 0)); + if(!SCALE_ROWS) + { + float alpha = 1.0f, beta = 0.0f; + + has_error |= checkHipblasStatus(hipblasLtMatmul(ltHandle, matmulDesc,&alpha, A, Adesc, B, Bdesc, &beta, (int8_t*)C, Cdesc, (int8_t*)C, Cdesc, &heuristicResult[0].algo, nullptr, 0, 0)); + } + else + { + //has_error |= checkHipblasStatus(hipblasLtMatmulDescSetAttribute(matmulDesc, hipblasLt_MATMUL_DESC_POINTER_MODE, &alphaVec, sizeof(alphaVec))); + float beta = 0.0f; + + has_error |= checkHipblasStatus(hipblasLtMatmul(ltHandle, matmulDesc, row_scale, A, Adesc, B, Bdesc, &beta, (int8_t*)C, Cdesc, (int8_t*)C, Cdesc, &heuristicResult[0].algo, nullptr, 0, 0)); + } } } @@ -635,7 +644,7 @@ template int igemmlt(hipblasLtHandl if (Adesc) has_error |= checkHipblasStatus(hipblasLtMatrixLayoutDestroy(Adesc)); if (matmulDesc) has_error |= checkHipblasStatus(hipblasLtMatmulDescDestroy(matmulDesc)); if(has_error == 1) - printf("error detected"); + printf("error detected\n"); return has_error; #endif // NO_HIPBLASLT From 714d9e9adb41c5983ae62105fb355186c54255d3 Mon Sep 17 00:00:00 2001 From: Prasanth Nunna Date: Fri, 16 Aug 2024 20:10:10 +0000 Subject: [PATCH 2/8] Remove toolcache for disk space --- .github/workflows/python-package.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 91e6d82a6..1c0488401 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -120,6 +120,7 @@ jobs: sudo rm -rf /opt/ghc sudo rm -rf "/usr/local/share/boost" sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /opt/hostedtoolcache - name: Build C++ run: bash .github/scripts/build-rocm.sh env: From ce77361934721642ab1dc2f6751074cac027762f Mon Sep 17 00:00:00 2001 From: Prasanth Nunna Date: Fri, 16 Aug 2024 20:19:29 +0000 Subject: [PATCH 3/8] print disk usage --- .github/workflows/python-package.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 1c0488401..210f22a9f 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -116,11 +116,13 @@ jobs: uses: docker/setup-qemu-action@v2 - name: Clean up disk space run: | + sudo df -h sudo rm -rf /usr/share/dotnet sudo rm -rf /opt/ghc sudo rm -rf "/usr/local/share/boost" sudo rm -rf "$AGENT_TOOLSDIRECTORY" sudo rm -rf /opt/hostedtoolcache + sudo df -h - name: Build C++ run: bash .github/scripts/build-rocm.sh env: From b87c2b93a7065ae6d893d36695e5e798aee79663 Mon Sep 17 00:00:00 2001 From: Prasanth Nunna Date: Fri, 16 Aug 2024 20:33:55 +0000 Subject: [PATCH 4/8] Clean disk space for linux --- .github/workflows/python-package.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 210f22a9f..68b45affe 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -123,6 +123,11 @@ jobs: sudo rm -rf "$AGENT_TOOLSDIRECTORY" sudo rm -rf /opt/hostedtoolcache sudo df -h + if: startsWith(matrix.os, 'ubuntu') + run: | + sudo rm -rf /swapfile + apt-get clean + sudo df -h - name: Build C++ run: bash .github/scripts/build-rocm.sh env: From 828fdc6e34095da83f2a9174fd2a1ef6ed8386e9 Mon Sep 17 00:00:00 2001 From: Prasanth Nunna Date: Fri, 16 Aug 2024 20:38:29 +0000 Subject: [PATCH 5/8] Fix for ubuntu --- .github/workflows/python-package.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 68b45affe..5d3baa6ee 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -123,6 +123,7 @@ jobs: sudo rm -rf "$AGENT_TOOLSDIRECTORY" sudo rm -rf /opt/hostedtoolcache sudo df -h + - name: Clean up disk space Ubuntu if: startsWith(matrix.os, 'ubuntu') run: | sudo rm -rf /swapfile From 5721601db924739f456a6d4df0386bc9fc8e6eda Mon Sep 17 00:00:00 2001 From: Prasanth Nunna Date: Fri, 16 Aug 2024 20:40:09 +0000 Subject: [PATCH 6/8] Add sudo for apt clean --- .github/workflows/python-package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 5d3baa6ee..1aece2298 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -127,7 +127,7 @@ jobs: if: startsWith(matrix.os, 'ubuntu') run: | sudo rm -rf /swapfile - apt-get clean + sudo apt-get clean sudo df -h - name: Build C++ run: bash .github/scripts/build-rocm.sh From d58303f921a3555644ac2a4ce2e197d757123726 Mon Sep 17 00:00:00 2001 From: Prasanth Nunna Date: Fri, 16 Aug 2024 20:56:30 +0000 Subject: [PATCH 7/8] Update clean up disk list --- .github/workflows/python-package.yml | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 1aece2298..f784446e6 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -117,17 +117,23 @@ jobs: - name: Clean up disk space run: | sudo df -h - sudo rm -rf /usr/share/dotnet - sudo rm -rf /opt/ghc - sudo rm -rf "/usr/local/share/boost" - sudo rm -rf "$AGENT_TOOLSDIRECTORY" - sudo rm -rf /opt/hostedtoolcache - sudo df -h - - name: Clean up disk space Ubuntu - if: startsWith(matrix.os, 'ubuntu') - run: | - sudo rm -rf /swapfile - sudo apt-get clean + sudo rm -rf \ + /usr/share/dotnet \ + /opt/ghc \ + "/usr/local/share/boost" \ + "$AGENT_TOOLSDIRECTORY" \ + /opt/hostedtoolcache \ + /opt/google/chrome \ + /opt/microsoft/msedge \ + /opt/microsoft/powershell \ + /opt/pipx \ + /usr/lib/mono \ + /usr/local/julia* \ + /usr/local/lib/android \ + /usr/local/lib/node_modules \ + /usr/local/share/chromium \ + /usr/local/share/powershell \ + /usr/share/swift sudo df -h - name: Build C++ run: bash .github/scripts/build-rocm.sh From 483e8ca48da03bd9a619d2d7f54db200a4c08748 Mon Sep 17 00:00:00 2001 From: Prasanth Nunna Date: Fri, 16 Aug 2024 21:04:34 +0000 Subject: [PATCH 8/8] remove disk usage print --- .github/workflows/python-package.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index f784446e6..d2da82501 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -116,7 +116,6 @@ jobs: uses: docker/setup-qemu-action@v2 - name: Clean up disk space run: | - sudo df -h sudo rm -rf \ /usr/share/dotnet \ /opt/ghc \ @@ -134,7 +133,6 @@ jobs: /usr/local/share/chromium \ /usr/local/share/powershell \ /usr/share/swift - sudo df -h - name: Build C++ run: bash .github/scripts/build-rocm.sh env: