diff --git a/.github/workflows/build-book.yaml b/.github/workflows/build-book.yaml index fe842f6..db39fbf 100644 --- a/.github/workflows/build-book.yaml +++ b/.github/workflows/build-book.yaml @@ -3,48 +3,19 @@ name: build-book on: workflow_call: inputs: - environment_name: - description: 'Name of conda environment to activate (NO LONGER USED)' + # Control of build input + build_from_artifact: + description: "Should we try to build from a previously uploaded code artifact?" required: false - default: 'cookbook-dev' - type: string - environment_file: - description: 'Name of conda environment file' - required: false - default: 'environment.yml' - type: string - path_to_notebooks: - description: 'Location of the JupyterBook source relative to repo root' - required: false - default: '.' - type: string - artifact_name: - description: 'The name to assign to the built book artifact.' - required: false - default: 'book-zip' - type: string - build_command: - description: 'The linux command to build the book or site.' - required: false - default: 'myst build --execute --html' - type: string - output_path: - description: 'Path to the built html content relative to `path_to_notebooks`' - required: false - default: '_build/html' - type: string - build_from_code_artifact: - description: 'Should we try to build from a previously uploaded code artifact?' - required: false - default: 'false' + default: "false" type: string code_artifact_name: - description: 'Name of zipped artifact passed in, instead of checking out the repository.' + description: "Name of zipped artifact passed in, instead of checking out the repository." required: false - default: 'code-zip' + default: "code-zip" type: string workflow: - description: 'Identify the workflow that produced the artifact' + description: "Identify the workflow that produced the artifact" required: false default: trigger-book-build.yaml type: string @@ -53,41 +24,10 @@ on: required: false default: success type: string - base_url: - description: 'Determines where the website is served from, including CSS & JS assets (needed for MyST)' - required: false - default: '/${{ github.event.repository.name }}' - type: string - - secrets: - PRIVATE_KEY: - description: 'Google analytics key needed for metrics page on portal site' - required: false - PRIVATE_KEY_ID: - description: 'Google analytics key id needed for metrics page on portal site' - required: false - ARM_USERNAME: - description: 'Username for the ARM Data Discovery portal (https://adc.arm.gov/armlive/)' - required: false - ARM_PASSWORD: - description: 'Password for the ARM Data Discovery portal (https://adc.arm.gov/armlive/)' - required: false - AQS_USERNAME: - description: 'Username for the AQS Data Portal' - required: false - AQS_KEY: - description: 'Key for the AQS Data Portal' - required: false - EARTHDATA_USERNAME: - description: 'NASA Earthdata API Username' - required: false - EARTHDATA_PASSWORD: - description: 'NASA Earthdata API Password' - required: false env: # the BASE_URL environment variable needs to be set if building with myst - BASE_URL: ${{ inputs.base_url }} + BASE_URL: "/${{ github.event.repository.name }}" jobs: build: @@ -95,6 +35,8 @@ jobs: defaults: run: shell: bash -leo pipefail {0} + env: + ENV_FILE: environment.yml steps: - name: Checkout the code from the repo if: inputs.build_from_code_artifact == 'false' @@ -116,140 +58,57 @@ jobs: if: inputs.build_from_code_artifact == 'true' run: | unzip pr_code.zip - rm -f pr_code.zip + rm -f pr_code.zip - - name: Get GitHub environment variables - id: get-env - uses: FranzDiebold/github-env-vars-action@v2 - - - name: Check for config file - id: check_config - uses: andstor/file-existence-action@v3 - with: - files: "${{ inputs.path_to_notebooks }}/_config.yml" - - - name: Parse config file - id: parse_config - if: steps.check_config.outputs.files_exists == 'true' - uses: CumulusDS/get-yaml-paths-action@v1.0.2 + - name: Load cookbook configuration + id: config + uses: CumulusDS/get-yaml-paths-action@v1.0.1 with: - file: ${{ inputs.path_to_notebooks }}/_config.yml - execute_notebooks: execute.execute_notebooks - binderhub_url: sphinx.config.html_theme_options.launch_buttons.binderhub_url - timeout: execute.timeout - - - name: Echo values from config file - if: steps.check_config.outputs.files_exists == 'true' - run: | - echo ${{ steps.parse_config.outputs.execute_notebooks }} - echo ${{ steps.parse_config.outputs.binderhub_url }} - echo ${{ steps.parse_config.outputs.timeout }} + file: cookbook.yml + # Settings JMES paths + execute: execution.enable + cacheExecution: execution.use-cache + executeOnBinder: execution.use-binder - name: Test for environment change id: env_change uses: tj-actions/changed-files@v47 with: - files: ${{ inputs.environment_file }} + files: "${{ env.ENV_FILE }}" - name: Echo environment change test result run: | echo '(DEBUG) The value of steps.env_change.outputs.any_changed is:' echo ${{ steps.env_change.outputs.any_changed }} - + - name: Setup environment with micromamba uses: mamba-org/setup-micromamba@v2 with: - environment-file: ${{ inputs.environment_file }} - - - name: Create book build environment - if: | - (inputs.use_cached_environment != 'true' - || steps.cache.outputs.cache-hit != 'true') - && steps.parse_config.outputs.execute_notebooks == 'binder' - run: | - conda install -c conda-forge jupyter-book pip - conda install sphinx-pythia-theme - pip install git+https://github.com/pangeo-gallery/binderbot.git - conda list - - - name: Get paths to notebook files - if: | - steps.parse_config.outputs.execute_notebooks == 'binder' - # This will find ALL *.ipynb files in the repo - # It would be better to cross-check this against the _toc.yml file - # to avoid unneccesary execution of notebooks that aren't included in the book - shell: python - run: | - import glob - notebooks = glob.glob('**/*.ipynb', recursive=True) - outfile = open("notebook_paths", "w") - for path in notebooks: - outfile.write(path + ' ') - outfile.close() # Writing these out to a file because I can't figure out how to set an environment variable from a python script + environment-file: "${{ env.ENV_FILE }}" - - name: Execute notebooks via binderbot using existing image - if: | - ( steps.parse_config.outputs.execute_notebooks == 'binder' - && steps.env_change.outputs.any_changed != 'true' ) - env: - ARM_USERNAME: ${{ secrets.ARM_USERNAME }} - ARM_PASSWORD: ${{ secrets.ARM_PASSWORD }} - AQS_USERNAME: ${{ secrets.AQS_USERNAME }} - AQS_KEY: ${{ secrets.AQS_KEY }} - EARTHDATA_USERNAME: ${{ secrets.EARTHDATA_USERNAME }} - EARTHDATA_PASSWORD: ${{ secrets.EARTHDATA_PASSWORD }} - run: | - NOTEBOOKS=$(cat notebook_paths) - echo 'Retrieved binder_url: ${{ steps.parse_config.outputs.binderhub_url }}' - echo "We will now execute these notebooks: $NOTEBOOKS" - echo "using the existing binder image from the main branch" - python -m binderbot.cli --binder-url ${{ steps.parse_config.outputs.binderhub_url }} --repo ${{ github.repository }} --ref main --nb-timeout ${{ steps.parse_config.outputs.timeout }} $NOTEBOOKS --pass-env-var ARM_USERNAME --pass-env-var ARM_PASSWORD + - name: Setup execution environment on a BinderHub + if: steps.config.executeOnBinder && steps.config.execute + # This action sets up evironment variables for e.g. mystmd to recognise and use + uses: 2i2c-org/clinder@action-v1 + with: + hub-url: https://binder.projectpythia.org/ - - name: Execute notebooks via binderbot using new image with latest environment - if: | - ( steps.parse_config.outputs.execute_notebooks == 'binder' - && steps.env_change.outputs.any_changed == 'true' ) - env: - ARM_USERNAME: ${{ secrets.ARM_USERNAME }} - ARM_PASSWORD: ${{ secrets.ARM_PASSWORD }} - AQS_USERNAME: ${{ secrets.AQS_USERNAME }} - AQS_KEY: ${{ secrets.AQS_KEY }} - EARTHDATA_USERNAME: ${{ secrets.EARTHDATA_USERNAME }} - EARTHDATA_PASSWORD: ${{ secrets.EARTHDATA_PASSWORD }} - run: | - NOTEBOOKS=$(cat notebook_paths) - echo 'Retrieved binder_url: ${{ steps.parse_config.outputs.binderhub_url }}' - echo "We will now execute these notebooks: $NOTEBOOKS" - echo "using the updated environment file in this branch to build a new image" - python -m binderbot.cli --binder-url ${{ steps.parse_config.outputs.binderhub_url }} --repo ${{ github.actor }}/$CI_REPOSITORY_NAME --ref $CI_ACTION_REF_NAME --nb-timeout ${{ steps.parse_config.outputs.timeout }} $NOTEBOOKS --pass-env-var ARM_USERNAME --pass-env-var ARM_PASSWORD + - name: Cache execution + if: steps.config.cacheExecution && steps.config.execute + uses: actions/cache@v4 + with: + path: ${{ inputs.path_to_notebooks }}/_build/execute + key: ${{ runner.os }}-${{ hashFiles(env.ENV_FILE) }}-execution - - name: Disable notebook execution during jupyterbook build - if: | - steps.parse_config.outputs.execute_notebooks == 'binder' - shell: python + - name: Build the book with execution + if: steps.config.execute run: | - import yaml - with open('${{ inputs.path_to_notebooks }}/_config.yml') as f: - data = yaml.safe_load(f) - data['execute']['execute_notebooks'] = 'off' - with open('${{ inputs.path_to_notebooks }}/_config.yml', 'w') as f: - yaml.dump(data, f) + myst build --execute --html --strict - - name: Build the book - # Assumption is that if execute_notebooks != 'binder' then the _config.yml file must be set to execute notebooks during build - env: - PRIVATE_KEY: ${{ secrets.PRIVATE_KEY }} - PRIVATE_KEY_ID: ${{ secrets.PRIVATE_KEY_ID }} - ARM_USERNAME: ${{ secrets.ARM_USERNAME }} - ARM_PASSWORD: ${{ secrets.ARM_PASSWORD }} - AQS_USERNAME: ${{ secrets.AQS_USERNAME }} - AQS_KEY: ${{ secrets.AQS_KEY }} - EARTHDATA_USERNAME: ${{ secrets.EARTHDATA_USERNAME }} - EARTHDATA_PASSWORD: ${{ secrets.EARTHDATA_PASSWORD }} - SECRETS_VARS: ${{ toJson(secrets) }} + - name: Build the book without execution + if: !steps.config.execute run: | - cd ${{ inputs.path_to_notebooks }} - ${{ inputs.build_command }} + myst build --html --strict - name: Zip the book run: | diff --git a/.github/workflows/config-validator.yaml b/.github/workflows/config-validator.yaml new file mode 100644 index 0000000..68f0ed2 --- /dev/null +++ b/.github/workflows/config-validator.yaml @@ -0,0 +1,35 @@ +name: config-validator + +on: + workflow_call: + +concurrency: + group: ${{ github.workflow }}=${{ github.head_ref}} + cancel-in-progress: true + +jobs: + cookbook-validator: + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@v6 + - name: Validate workflows + uses: dsanders11/json-schema-validate-action@v2.0.0 + with: + # Schema is not versioned by URL — the versioning is defined _in_ the schema + # The latest version of the schema should always be used for validation. + schema: https://raw.githubusercontent.com/ProjectPythia/cookbook-actions/refs/heads/feat/clinder/cookbook.schema.json + files: cookbook.yaml + myst-validator: + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@v6 + - name: Validate workflows + uses: dsanders11/json-schema-validate-action@v2.0.0 + with: + # Assume that MyST schema is unchanging + schema: https://raw.githubusercontent.com/ProjectPythia/cookbook-actions/refs/heads/feat/clinder/myst.schema.json + files: myst.yaml diff --git a/cookbook.schema.json b/cookbook.schema.json new file mode 100644 index 0000000..15cf0b3 --- /dev/null +++ b/cookbook.schema.json @@ -0,0 +1,44 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/projectpythia/pythia-config/cookbook.schema.json", + "$ref": "#/definitions/config", + "definitions": { + "config": { + "$comment": "Allow multiple versions of config to co-exist across a set of repositories by requiring the schema version", + "oneOf": [ + { + "$ref": "#/definitions/configV1" + } + ] + }, + "configV1": { + "type": "object", + "additionalProperties": false, + "required": ["version"], + "properties": { + "version": { + "const": 1 + }, + "execution": { + "$ref": "#/definitions/execution" + } + } + }, + "execution": { + "properties": { + "enable": { + "type": "boolean" + }, + "use-cache": { + "type": "boolean" + }, + "use-binder": { + "type": "boolean" + } + }, + "required": [], + "type": "object", + "additionalProperties": false + } + } +} diff --git a/myst.schema.json b/myst.schema.json new file mode 100644 index 0000000..7da4b2a --- /dev/null +++ b/myst.schema.json @@ -0,0 +1,34 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/projectpythia/pythia-config/myst.schema.json", + "$ref": "#/definitions/myst.yml", + "definitions": { + "myst.yml": { + "type": "object", + "required": ["project"], + "properties": { + "project": { + "$ref": "#/definitions/project" + } + } + }, + "project": { + "type": "object", + "required": ["title", "thumbnail", "tags"], + "properties": { + "title": { + "type": "string" + }, + "thumbnail": { + "type": "string" + }, + "tags": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + } +}