Add option for conda and pip internal uploads (#62)

Author: brynpickering

.github/workflows/conda-build.yml

@@ -21,6 +21,10 @@ on:
         description: "GitHub environment in which secrets are stored"
         type: string
         default: "release"
+      destination:
+        description: "Upload packaging index destination. Must be one of [anaconda, internal]."
+        type: string
+        default: anaconda
     secrets:
       ANACONDA_TOKEN:
         required: true
@@ -49,6 +53,7 @@ jobs:
   token-exists:
     runs-on: ubuntu-latest
     environment: ${{ inputs.environment }}
+    if: inputs.destination == 'anaconda'
     steps:
     - name: check if ANACONDA_TOKEN exists
       env:
@@ -83,6 +88,10 @@ jobs:
     - name: Add conda channel
       run: conda config --add channels city-modelling-lab
 
+    - name: Add internal conda channel
+      if: inputs.destination == 'internal'
+      run: conda config --add channels https://packages.arup.com/conda
+
     - name: Build conda package
       run: conda mambabuild ${{ inputs.recipe_dir }}
 

.github/workflows/conda-upload.yml

@@ -19,6 +19,10 @@ on:
         description: "GitHub environment in which secrets are stored"
         type: string
         default: "release"
+      destination:
+        description: "Upload packaging index destination. Must be one of [anaconda, internal]."
+        type: string
+        default: anaconda
     secrets:
       ANACONDA_TOKEN:
         required: true
@@ -28,11 +32,12 @@ defaults:
     shell: bash -l {0}
 
 jobs:
-  conda-publish:
+  conda-anaconda-publish:
+    if: inputs.destination == 'anaconda'
     runs-on: ubuntu-latest
     environment: ${{ inputs.environment }}
     env:
-      PACKAGENAME: "conda-build[a-zA-Z0-9-.]*-${{ inputs.package_name }}-${{ inputs.version }}"
+      ARTIFACTNAME: "conda-build[a-zA-Z0-9-.]*-${{ inputs.package_name }}-${{ inputs.version }}"
     steps:
     - uses: actions/checkout@v4
     - uses: mamba-org/setup-micromamba@v2
@@ -49,7 +54,7 @@ jobs:
       if: inputs.build_workflow != ''
       uses: dawidd6/action-download-artifact@v7
       with:
-        name: ${{ env.PACKAGENAME }}
+        name: ${{ env.ARTIFACTNAME }}
         name_is_regexp: true
         workflow: ${{ inputs.build_workflow }}
         path: ${{ runner.temp }}/conda_builds
@@ -58,6 +63,7 @@ jobs:
       if: inputs.build_workflow == ''
       uses: actions/download-artifact@v4
       with:
+        pattern: ${{ env.ARTIFACTNAME }}
         path: ${{ runner.temp }}/conda_builds
 
     - name: Get version without the v
@@ -69,3 +75,45 @@ jobs:
       env:
         ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_TOKEN }}
       run: find ${{ runner.temp }}/conda_builds -path "**/${{ inputs.package_name }}-${{ env.VERSION }}-*.tar.bz2" -exec anaconda upload {} +
+
+  conda-internal-publish:
+    if: inputs.destination == 'internal'
+    runs-on: [self-hosted, linux, packages]
+    env:
+      ARTIFACTNAME: "conda-build[a-zA-Z0-9-.]*-${{ inputs.package_name }}-${{ inputs.version }}"
+      PACKAGENAME: "conda-build-*-${{ inputs.package_name }}-${{ inputs.version }}"
+    steps:
+      - uses: mamba-org/setup-micromamba@v2
+        with:
+          micromamba-version: '1.5.10-0'
+          environment-name: condaindex
+          create-args: conda-index
+          post-cleanup: all
+          cache-environment: true
+          micromamba-binary-path: ${{ runner.temp }}/bin/micromamba
+
+      - name: Download built conda package from another workflow
+        uses: dawidd6/action-download-artifact@v7
+        with:
+          name: ${{ env.ARTIFACTNAME }}
+          name_is_regexp: true
+          workflow: ${{ inputs.build_workflow }}
+          path: ${{ runner.temp }}/conda_builds
+
+      - name: Get version without the v
+        run: |
+          TAG=${{ inputs.version }}
+          echo "VERSION=${TAG#v}" >> $GITHUB_ENV
+
+      # Move to a location that can be accessed at packages.arup.com
+      - name: Move package to download location
+        shell: bash
+        run: |
+          touch ${{ runner.temp }}/.condarc
+          rsync -a -v --prune-empty-dirs --include '*/' --include '${{ inputs.package_name }}-${{ env.VERSION }}-*.tar.bz2' --exclude '*' ${{ runner.temp }}/conda_builds/${{ env.PACKAGENAME }}/ ~/packages/conda/
+
+      - name: Re-index conda
+        env:
+          CONDARC: ${{ runner.temp }}/.condarc
+        shell: bash -l {0}
+        run: python -m conda_index ~/packages/conda

.github/workflows/pip-build.yml

@@ -21,6 +21,10 @@ on:
         required: false
         type: string
         default: "--no-deps"
+      destination:
+        description: "Upload packaging index destination. Must be one of [pypi, internal]."
+        type: string
+        default: pypi
 
     secrets:
       TEST_PYPI_API_TOKEN:
@@ -42,7 +46,7 @@ jobs:
         # cannot use secrets directly in conditionals
         # see https://docs.github.com/en/actions/security-guides/using-secrets-in-github-actions#using-secrets-in-a-workflow
         test_pypi_token: ${{ secrets.TEST_PYPI_API_TOKEN }}
-      if: env.test_pypi_token == ''
+      if: inputs.destination == 'pypi' && env.test_pypi_token == ''
       run: |
         echo "the secret \"TEST_PYPI_API_TOKEN\" is not available, so the pypi package cannot be uploaded to the test index site."
         echo "Please go to \"settings \> secrets \> actions\" to create it before trying to build the pip package."
@@ -98,7 +102,7 @@ jobs:
   pip-test-upload:  # upload to test-pypi and then check install from there succeeds
     needs: [pip-build, pip-test-build]
     environment: ${{ inputs.environment }}
-    if: needs.pip-build.result == 'success' && needs.pip-test-build.result == 'success'
+    if: needs.pip-build.result == 'success' && needs.pip-test-build.result == 'success' && inputs.destination == 'pypi'
     runs-on: ubuntu-latest
     steps:
     - uses: mamba-org/setup-micromamba@v2

.github/workflows/pip-upload.yml

@@ -19,6 +19,10 @@ on:
         description: "GitHub environment in which secrets are stored"
         type: string
         default: "pre-release"
+      destination:
+        description: "Upload packaging index destination. Must be one of [pypi, internal]."
+        type: string
+        default: pypi
     secrets:
       PYPI_API_TOKEN:
         required: true
@@ -28,11 +32,12 @@ defaults:
     shell: bash -l {0}
 
 jobs:
-  pip-publish:
+  pip-pypi-publish:
     runs-on: ubuntu-latest
+    if: inputs.destination == 'pypi'
     environment: ${{ inputs.environment }}
     env:
-      PACKAGENAME: "pip-build-${{ inputs.package_name }}-${{ inputs.version }}"
+      ARTIFACTNAME: "pip-build-${{ inputs.package_name }}-${{ inputs.version }}"
 
     steps:
     - name: check if PYPI_API_TOKEN exists
@@ -50,18 +55,47 @@ jobs:
       if: inputs.build_workflow != ''
       uses: dawidd6/action-download-artifact@v7
       with:
-        name: ${{ env.PACKAGENAME }}
+        name: ${{ env.ARTIFACTNAME }}
         workflow: ${{ inputs.build_workflow }}
         path: dist/
 
     - name: Download built package from same workflow
       if: inputs.build_workflow == ''
       uses: actions/download-artifact@v4
       with:
-        name: ${{ env.PACKAGENAME }}
+        name: ${{ env.ARTIFACTNAME }}
         path: dist/
 
     - name: Publish distribution 📦 to PyPI
       uses: pypa/gh-action-pypi-publish@release/v1
       with:
-        password: ${{ secrets.PYPI_API_TOKEN }}
+        password: ${{ secrets.PYPI_API_TOKEN }}
+
+
+  pip-internal-publish:
+    if: inputs.destination == 'internal'
+    runs-on: [self-hosted, linux, packages]
+    environment: ${{ inputs.environment }}
+    env:
+      ARTIFACTNAME: "pip-build-${{ inputs.package_name }}-${{ inputs.version }}"
+
+    steps:
+    - name: Download built package from another workflow
+      if: inputs.build_workflow != ''
+      uses: dawidd6/action-download-artifact@v7
+      with:
+        name: ${{ env.ARTIFACTNAME }}
+        workflow: ${{ inputs.build_workflow }}
+        path: ${{ runner.temp }}/${{ env.ARTIFACTNAME }}/
+
+    - name: Download built package from same workflow
+      if: inputs.build_workflow == ''
+      uses: actions/download-artifact@v4
+      with:
+        name: ${{ env.ARTIFACTNAME }}
+        path: ${{ runner.temp }}/${{ env.ARTIFACTNAME }}/
+
+    - name: Publish distribution 📦 to internal repository
+      run: |
+        cp ${{ runner.temp }}/${{ env.ARTIFACTNAME }}/${{ inputs.package_name }}-*.tar.gz ~/packages/
+        cp ${{ runner.temp }}/${{ env.ARTIFACTNAME }}/${{ inputs.package_name }}-*.tar.gz ~/packages/${{ inputs.package_name }}.tar.gz

CHANGELOG.md

@@ -33,10 +33,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- Ability to upload pip / conda packages to <https://packages.arup.com> for internal Arup projects.
 - Composite action for building a project-specific conda environment, used across reusable workflows but also available for direct use as a step in other projects (#26).
 - Environment cache directory within the runner working directory (`.cache/envs`) (#29).
 
-
 ## [v1.0.0] - 2024-07-12
 
 ### Fixed

README.md

@@ -108,15 +108,14 @@ jobs:
       message: "AWS upload action"
 ```
 
-!!! note
-
-  You can _only_ use `secrets: inherit` if you are hosting your repository in the `arup-group` organisation.
-  If you have the repo under your own username, you will need to explicitly pass the necessary secrets, e.g.:
-
-  ``` yaml
-  secrets:
-    SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
-  ```
+> [!NOTE]
+> You can _only_ use `secrets: inherit` if you are hosting your repository in the `arup-group` organisation.
+> If you have the repo under your own username, you will need to explicitly pass the necessary secrets, e.g.:
+>
+> ``` yaml
+> secrets:
+>   SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
+> ```
 
 ## Available workflows
 
@@ -146,8 +145,15 @@ _Inputs_:
 - environment (optional, default="pre-release"): GitHub environment in which secrets are stored.
 Environments help to ensure that only certain operations are available to different user types.
 E.g., releasing packages can be given an extra layer of security whereby a maintainer has to approve an action before it can run.
+- destination (optional, default="anaconda"): One of "anaconda" or "internal", to specify what the ultimate destination of the package will be.
+If `internal`, the package will be uploaded to <https://packages.arup.com/conda>.
+If `anaconda`, the package will be uploaded to <https://anaconda.org/[CHANNEL-NAME]/> where `[CHANNEL-NAME]` is linked to the `ANACONDA_TOKEN` secret.
 
 _Required secrets_: `ANACONDA_TOKEN` (required to verify that later upload will not fail) stored in a GitHub actions environment of the same name as `environment`.
+If `destination=internal`, this secret must still be defined, but can be a placeholder string (e.g. "NA").
+
+> [!NOTE]
+> To use this action with `destination=internal`, you must request access to the `packages` self-hosted runner for your repository via an Arup service-now request.
 
 ### Upload a conda package
 
@@ -164,10 +170,17 @@ _Inputs_:
 - environment (optional, default="pre-release"): GitHub environment in which secrets are stored.
 Environments help to ensure that only certain operations are available to different user types.
 E.g., releasing packages can be given an extra layer of security whereby a maintainer has to approve an action before it can run.
+- destination (optional, default="anaconda"): One of "anaconda" or "internal", to specify what the ultimate destination of the package will be.
+If `internal`, the package will be uploaded to <https://packages.arup.com/conda>.
+If `anaconda`, the package will be uploaded to <https://anaconda.org/[CHANNEL-NAME]/> where `[CHANNEL-NAME]` is linked to the `ANACONDA_TOKEN` secret.
 
 _Required secrets_: `ANACONDA_TOKEN` stored in a GitHub actions environment of the same name as `environment`.
+If `destination=internal`, this secret must still be defined, but can be a placeholder string (e.g. "NA").
 
-### Build a pip package for upload to PyPI
+> [!NOTE]
+> To use this action with `destination=internal`, you must request access to the `packages` self-hosted runner for your repository via an Arup service-now request.
+
+### Build a pip package for upload to PyPI or to <https://packages.arup.com>
 
 _URL_: `arup-group/actions-city-modelling-lab/.github/workflows/pip-build.yml`
 
@@ -187,10 +200,17 @@ E.g., releasing packages can be given an extra layer of security whereby a maint
 - pip_args (optional, default="--no-deps"). Any arguments to pass to pip when running test installations.
 Many of our packages have non-python dependencies, so it is useful to use `--no-deps` in the installation.
 However, if you know that your library has purely python dependencies then the pip build process is made more robust by removing this argument (i.e. `pip_args: ""`)
+- destination (optional, default="pypi"): One of "pypi" or "internal", to specify what the ultimate destination of the package will be.
+If `internal`, the package will be uploaded to <https://packages.arup.com>.
+If `pypi`, the package will be uploaded to <https://test.pypi.org/> for testing and <https://pypi.org/> for final upload.
 
 _Required secrets_: `TEST_PYPI_API_TOKEN` stored in a GitHub actions environment of the same name as `environment`.
+If `destination=internal`, this secret must still be defined, but can be a placeholder string (e.g. "NA").
+
+> [!NOTE]
+> To use this action with `destination=internal`, you must request access to the `packages` self-hosted runner for your repository via an Arup service-now request.
 
-### Upload a pip package to PyPI
+### Upload a pip package to PyPI or to <https://packages.arup.com>
 
 _URL_: `arup-group/actions-city-modelling-lab/.github/workflows/pip-upload.yml`
 
@@ -205,8 +225,15 @@ _Inputs_:
 - environment (optional, default="pre-release"): GitHub environment in which secrets are stored.
 Environments help to ensure that only certain operations are available to different user types.
 E.g., releasing packages can be given an extra layer of security whereby a maintainer has to approve an action before it can run.
+- destination (optional, default="pypi"): One of "pypi" or "internal", to specify what the ultimate destination of the package will be.
+If `internal`, the package will be uploaded to <https://packages.arup.com>.
+If `pypi`, the package will be uploaded to <https://test.pypi.org/> for testing and <https://pypi.org/> for final upload.
 
 _Required secrets_: `PYPI_API_TOKEN` stored in a GitHub actions environment of the same name as `environment`.
+If `destination=internal`, this secret must still be defined, but can be a placeholder string (e.g. "NA").
+
+> [!NOTE]
+> To use this action with `destination=internal`, you must request access to the `packages` self-hosted runner for your repository via an Arup service-now request.
 
 ### Deploy documentation
 
@@ -296,4 +323,4 @@ _Required secrets_: `SLACK_WEBHOOK`
 
 _URL_: `arup-group/actions-city-modelling-lab/.github/workflows/template-check.yml`
 
-_description_: If your project was generated using a [cookiecutter](https://github.com/cookiecutter/cookiecutter) template, check whether there are changes to the template that could be pulled into the project.
+_description_: If your project was generated using a [cookiecutter](https://github.com/cookiecutter/cookiecutter) template, check whether there are changes to the template that could be pulled into the project.