fix(diagrams): adds ast parsing to block import and from import
#9638
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Python | |
| on: | |
| push: | |
| pull_request: | |
| workflow_dispatch: | |
| permissions: {} | |
| jobs: | |
| detect-packages: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| outputs: | |
| packages: ${{ steps.find-packages.outputs.packages }} | |
| changed-directories: ${{ steps.find-changed-directories.outputs.changed-directories }} | |
| changed-files: ${{ steps.find-changed-directories.outputs.changed-files }} | |
| steps: | |
| - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 | |
| with: | |
| fetch-depth: 0 | |
| - name: Fetch base branch | |
| run: git fetch origin ${{ github.base_ref }}:${{ github.base_ref }} | |
| - name: find changed directories | |
| id: find-changed-directories | |
| env: | |
| EVENT_NAME: ${{ github.event_name }} | |
| EVENT_BEFORE: ${{ github.event.before }} | |
| BASE_REF: ${{ github.base_ref }} | |
| run: | | |
| # Push against last commit | |
| # Pull Request against target branch sha | |
| # Otherwise against latest release | |
| if [ "$EVENT_NAME" == "push" ]; then | |
| # Handle null SHA case for new branches | |
| if [ "$EVENT_BEFORE" == "0000000000000000000000000000000000000000" ]; then | |
| SINCE="$(git merge-base HEAD origin/main)" | |
| else | |
| SINCE="$EVENT_BEFORE" | |
| fi | |
| elif [ "$EVENT_NAME" == "pull_request" ]; then | |
| SINCE="$BASE_REF" | |
| else | |
| SINCE="$(gh release list --exclude-drafts --exclude-pre-releases --limit 1 --json tagName | jq -r '.[].tagName')" | |
| fi; | |
| if [ -z "$SINCE" ]; then SINCE="$(git rev-list --max-parents=0 HEAD)"; fi; | |
| echo "$SINCE" | |
| CHANGEDFILES="$(git diff --name-only "$SINCE" HEAD | sed 's/^\.\///' | jq -R -s -c 'split("\n")[:-1]')" | |
| CHANGEDDIRECTORIES="$(echo $CHANGEDFILES | jq -r '.[] | select(. | startswith("src\/"))' | cut -d'/' -f2 | sort -u | sed 's/^\.\///' | jq -R -s -c 'split("\n")[:-1]')" | |
| echo "$CHANGEDDIRECTORIES" | |
| echo "$CHANGEDFILES" | |
| echo "changed-files=$CHANGEDFILES" >> $GITHUB_OUTPUT | |
| echo "changed-directories=$CHANGEDDIRECTORIES" >> $GITHUB_OUTPUT | |
| - name: Find Python packages | |
| id: find-packages | |
| working-directory: src | |
| run: | | |
| PACKAGES=$(find . -name pyproject.toml -exec dirname {} \; | sed 's/^\.\///' | jq -R -s -c 'split("\n")[:-1]') | |
| echo "packages=$PACKAGES" >> $GITHUB_OUTPUT | |
| build: | |
| needs: [detect-packages] | |
| if: ${{ needs.detect-packages.outputs.packages != '[]' && needs.detect-packages.outputs.packages != '' }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| package: ${{ fromJson(needs.detect-packages.outputs.packages) }} | |
| name: Build ${{ matrix.package }} | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| pull-requests: read | |
| security-events: write | |
| actions: read | |
| steps: | |
| - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@3259c6206f993105e3a61b142c2d97bf4b9ef83d # v7.1.0 | |
| - name: Set up Python | |
| uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 | |
| with: | |
| python-version-file: "src/${{ matrix.package }}/.python-version" | |
| # cache: uv (not supported) | |
| - name: Cache GraphViz | |
| uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 #v4.3.0 | |
| id: cache-graphviz | |
| with: | |
| path: "~/graphviz" | |
| key: graphviz | |
| - name: Install Graphviz | |
| env: | |
| CACHE_HIT: ${{steps.cache-graphviz.outputs.cache-hit}} | |
| run: | | |
| if [[ "$CACHE_HIT" == 'true' ]]; then | |
| sudo cp --verbose --force --recursive ~/graphviz/* / | |
| else | |
| sudo apt-get update && sudo apt-get install -y graphviz | |
| mkdir -p ~/graphviz | |
| sudo dpkg -L graphviz | while IFS= read -r f; do if test -f $f; then echo $f; fi; done | xargs cp --parents --target-directory ~/graphviz/ | |
| fi | |
| - name: Install Bandit | |
| run: | | |
| pip install --require-hashes --requirement .github/workflows/bandit-requirements.txt | |
| - name: Security check - Bandit | |
| id: bandit-check | |
| working-directory: src/${{ matrix.package }} | |
| run: bandit -r --severity-level medium --confidence-level medium -f html -o bandit-report-${{ matrix.package }}.html -c "pyproject.toml" . || echo "status=failure" >> $GITHUB_OUTPUT | |
| - name: Store Bandit as Artifact | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 | |
| with: | |
| name: bandit-report-${{ matrix.package }}.html | |
| path: src/${{ matrix.package }}/bandit-report-${{ matrix.package }}.html | |
| - name: Stop on Bandit failure | |
| if: steps.bandit-check.outputs.status == 'failure' | |
| run: exit 1 | |
| - name: Install dependencies | |
| working-directory: src/${{ matrix.package }} | |
| run: uv sync --frozen --all-extras --dev | |
| - name: Run tests | |
| working-directory: src/${{ matrix.package }} | |
| run: | | |
| if [ -d "tests" ]; then | |
| uv run --frozen pytest --cov --cov-branch --cov-report=term-missing --cov-report=xml:${{ matrix.package }}-coverage.xml | |
| else | |
| echo "No tests directory found, skipping tests" | |
| fi | |
| - name: Upload coverage reports to Codecov | |
| uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 #v5.5.1 | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| files: ${{ matrix.package }}-coverage.xml | |
| - name: Run pyright | |
| working-directory: src/${{ matrix.package }} | |
| run: uv run --frozen pyright | |
| - name: Run ruff format | |
| working-directory: src/${{ matrix.package }} | |
| run: uv run --frozen ruff format . | |
| - name: Run ruff check | |
| working-directory: src/${{ matrix.package }} | |
| run: uv run --frozen ruff check . | |
| - name: Build package | |
| working-directory: src/${{ matrix.package }} | |
| run: uv build | |
| - name: Upload distribution | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 | |
| with: | |
| name: dist-${{ matrix.package }} | |
| path: src/${{ matrix.package }}/dist/ | |
| - name: Generate Software Bill of Materials (SBOM) | |
| working-directory: src/${{ matrix.package }} | |
| run: | | |
| source .venv/bin/activate | |
| echo "Attempt to convert to proper UTF-8 files https://github.com/CycloneDX/cyclonedx-python/issues/868" | |
| find .venv -type f -path '*/*.dist-info/*' > .venv/FILES | |
| # because grep with xargs returns 123 have to do this the long and hard way... | |
| while IFS= read -r line; do | |
| (grep -s -q -axv '.*' $line && | |
| if [[ "$(file -b --mime-encoding $line)" != "binary" ]]; then | |
| echo "illegal utf-8 characters in $line...converting..."; | |
| iconv -f $(file -b --mime-encoding $line) -t utf-8 $line > $line.utf8; | |
| mv $line.utf8 $line; | |
| fi; | |
| ) || echo "good $line" | |
| done < .venv/FILES; | |
| uv tool run --from cyclonedx-bom==6.1.3 cyclonedx-py environment $VIRTUAL_ENV --PEP-639 --gather-license-texts --pyproject pyproject.toml --mc-type library --output-format JSON > sbom.json | |
| - name: Display SBOM | |
| working-directory: src/${{ matrix.package }} | |
| run: | | |
| cat <<EOT | | |
| import re | |
| import json | |
| import importlib.metadata as metadata | |
| def parse_bom(json_file): | |
| # Parse the JSON file | |
| with open(json_file, 'r') as file: | |
| data = json.load(file) | |
| # Extract components | |
| components = [] | |
| for component in data['components']: | |
| comp_info = {} | |
| # Get name, version, description, and purl | |
| comp_info['name'] = component.get('name', 'Unknown') | |
| comp_info['version'] = component.get('version', 'Unknown') | |
| comp_info['description'] = component.get('description', 'Unknown') | |
| comp_info['purl'] = component.get('purl', 'Unknown') | |
| # Get licenses | |
| comp_info['licenses'] = [] | |
| licenses = component.get('licenses', []) | |
| for license in licenses: | |
| if license.get('license', {}).get('id'): | |
| comp_info['licenses'].append(license.get('license').get('id')) | |
| if len(comp_info['licenses']) == 0: | |
| comp_info['licenses'].append("No licenses") | |
| # Extract additional information (copyright, etc.) | |
| copyright_info = extract_copyright_from_metadata(comp_info['name']) | |
| comp_info['copyright'] = copyright_info if copyright_info else "No copyright information" | |
| components.append(comp_info) | |
| return components | |
| def extract_copyright_from_metadata(package_name): | |
| try: | |
| # Use importlib.metadata to retrieve metadata from the installed package | |
| dist = metadata.distribution(package_name) | |
| metadata_info = dist.metadata | |
| # Extract relevant metadata | |
| copyright_info = [] | |
| author = metadata_info.get('Author') | |
| author_email = metadata_info.get('Author-email') | |
| license_info = metadata_info.get('License') | |
| if author: | |
| copyright_info.append(f"Author: {author}") | |
| if author_email: | |
| copyright_info.append(f"Author Email: {author_email}") | |
| if license_info: | |
| copyright_info.append(f"License: {license_info}") | |
| # Check for classifiers or any extra metadata fields | |
| if 'Classifier' in metadata_info: | |
| for classifier in metadata_info.get_all('Classifier'): | |
| if 'copyright' in classifier.lower(): | |
| copyright_info.append(classifier) | |
| return ', '.join(copyright_info) if copyright_info else None | |
| except metadata.PackageNotFoundError: | |
| return None | |
| def main(): | |
| bom_file = 'sbom.json' # Replace with your BOM file path | |
| components = parse_bom(bom_file) | |
| for component in components: | |
| print(f"Name: {component['name']}") | |
| print(f"Version: {component['version']}") | |
| print(f"Description: {component['description']}") | |
| print(f"PURL: {component['purl']}") | |
| print(f"Licenses: {', '.join(component['licenses'])}") | |
| print(f"Copyright: {component['copyright']}") | |
| print("-" * 40) | |
| if __name__ == "__main__": | |
| main() | |
| EOT | |
| python - | |
| - name: Upload Software Bill of Materials | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 | |
| with: | |
| name: sbom-${{ matrix.package }} | |
| path: src/${{ matrix.package }}/sbom.json |