Skip to content
CI/CD Pipeline

CI/CD Pipeline #421

Sign in to view logs

CI/CD Pipeline

CI/CD Pipeline #421

Workflow file for this run

.github/workflows/ci.yml at a99a840
---
#####################################################################################################################################################################################################
# Project: Juniper
# Sub-Project: JuniperData
# Application: juniper_data
# File Name: ci.yml
# Author: Paul Calnon
# Version: 0.4.0
#
# Date Created: 2026-01-30
# Last Modified: 2026-02-06
#
# License: MIT License
# Copyright: Copyright (c) 2024-2026 Paul Calnon
#
# Description:
# GitHub Actions CI/CD Pipeline for JuniperData Dataset Generation Service
# - Pre-commit hooks for code quality (black, isort, flake8, mypy, bandit)
# - Unit tests with coverage enforcement
# - Integration tests
# - Security scanning (Gitleaks, Bandit SARIF, pip-audit)
# - Proper failure handling (no continue-on-error for critical steps)
# - Dependency caching for performance
#
# References:
# - JUNIPER-DATA-001: CI/CD Pipeline Setup
# - Best practices: https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions
#####################################################################################################################################################################################################
name: CI/CD Pipeline
on:
push:
branches:
- main
- develop
- feature/**
- fix/**
pull_request:
branches:
- main
- develop
schedule:
- cron: '0 6 * * *' # Daily at 6 AM UTC (slow tests)
workflow_dispatch:
concurrency:
group: ci-${{ github.ref }}
cancel-in-progress: true
permissions:
contents: read
env:
SERVICE_NAME: JuniperData
PYTHON_TEST_VERSION: "3.14"
COVERAGE_FAIL_UNDER: "80"
jobs:
# ═══════════════════════════════════════════════════════════════════════════════════════════════
# Pre-commit: Code Quality Checks (runs across multiple Python versions)
# ═══════════════════════════════════════════════════════════════════════════════════════════════
pre-commit:
name: Pre-commit (Python ${{ matrix.python-version }})
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
# python-version: ["3.11", "3.12", "3.13", "3.14"]
python-version: ["3.12", "3.13", "3.14"]
steps:
- name: Checkout Code
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: ${{ matrix.python-version }}
cache: pip
- name: Install pre-commit
run: |
python -m pip install --upgrade pip
pip install pre-commit
- name: Cache pre-commit hooks
uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
with:
path: ~/.cache/pre-commit
key: pre-commit-${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('.pre-commit-config.yaml') }}
restore-keys: |
pre-commit-${{ runner.os }}-${{ matrix.python-version }}-
- name: Run pre-commit hooks
run: |
echo "╔════════════════════════════════════════════════════════════╗"
echo "║ JuniperData - Pre-commit Checks ║"
echo "╚════════════════════════════════════════════════════════════╝"
pre-commit run --all-files --show-diff-on-failure
# ═══════════════════════════════════════════════════════════════════════════════════════════════
# Unit Tests: Run unit tests with coverage enforcement
# ═══════════════════════════════════════════════════════════════════════════════════════════════
unit-tests:
name: Unit Tests + Coverage (Python ${{ matrix.python-version }})
runs-on: ubuntu-latest
needs: [pre-commit]
strategy:
fail-fast: false
matrix:
# python-version: ["3.11", "3.12", "3.13", "3.14"]
python-version: ["3.12", "3.13", "3.14"]
steps:
- name: Checkout Code
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
fetch-depth: 1
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: ${{ matrix.python-version }}
cache: pip
- name: Install dependencies
run: |
echo "╔════════════════════════════════════════════════════════════╗"
echo "║ JuniperData - Install Dependencies ║"
echo "╚════════════════════════════════════════════════════════════╝"
python -m pip install --upgrade pip
pip install ".[all]"
- name: Create required directories
run: |
mkdir -p reports/junit reports/htmlcov
- name: Run Unit Tests with Coverage Gate
run: |
echo "╔════════════════════════════════════════════════════════════╗"
echo "║ JuniperData - Unit Tests (Fast Only) ║"
echo "╚════════════════════════════════════════════════════════════╝"
python -m pytest \
-m "unit and not slow" \
juniper_data/tests/unit \
--verbose \
--timeout=60 \
--maxfail=5 \
--junitxml=reports/junit-unit.xml \
--cov=juniper_data \
--cov-report=term-missing \
--cov-report=xml:reports/coverage.xml \
--cov-report=html:reports/htmlcov \
--cov-fail-under=${{ env.COVERAGE_FAIL_UNDER }}
- name: Upload Coverage Artifacts
uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
if: always()
with:
name: coverage-report-py${{ matrix.python-version }}
path: |
reports/coverage.xml
reports/htmlcov/
retention-days: 30
- name: Upload Coverage to Codecov
uses: codecov/codecov-action@ad3126e916f78f00edff4ed0317cf185271ccc2d # v5.4.2
if: matrix.python-version == '3.14'
with:
token: ${{ secrets.CODECOV_TOKEN }}
files: reports/coverage.xml
flags: unittests
name: juniper-data-coverage
fail_ci_if_error: false
- name: Upload Test Results
uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
if: always()
with:
name: unit-test-results-py${{ matrix.python-version }}
path: reports/junit-unit.xml
retention-days: 30
# ═══════════════════════════════════════════════════════════════════════════════════════════════
# Build: Build sdist and wheel artifacts
# ═══════════════════════════════════════════════════════════════════════════════════════════════
build:
name: Build Package
runs-on: ubuntu-latest
needs: [unit-tests]
steps:
- name: Checkout Code
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: Set up Python 3.14
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.14"
cache: pip
- name: Install build
run: |
python -m pip install --upgrade pip
pip install build
- name: Build sdist and wheel
run: |
echo "╔════════════════════════════════════════════════════════════╗"
echo "║ JuniperData - Build Package ║"
echo "╚════════════════════════════════════════════════════════════╝"
python -m build --sdist --wheel
- name: Verify build artifacts
run: |
ls -la dist/
test -f dist/*.tar.gz
test -f dist/*.whl
- name: Upload build artifacts
uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
with:
name: build-artifacts
path: dist/
retention-days: 30
# ═══════════════════════════════════════════════════════════════════════════════════════════════
# Integration Tests: Run integration tests (PRs and main/develop pushes)
# ═══════════════════════════════════════════════════════════════════════════════════════════════
integration-tests:
name: Integration Tests
runs-on: ubuntu-latest
needs: [unit-tests]
if: github.event_name == 'pull_request' || github.ref_name == 'main' || github.ref_name == 'develop'
steps:
- name: Checkout Code
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
fetch-depth: 1
- name: Set up Python ${{ env.PYTHON_TEST_VERSION }}
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: ${{ env.PYTHON_TEST_VERSION }}
cache: pip
- name: Install dependencies
run: |
echo "╔════════════════════════════════════════════════════════════╗"
echo "║ JuniperData - Install Dependencies ║"
echo "╚════════════════════════════════════════════════════════════╝"
python -m pip install --upgrade pip
pip install ".[all]"
- name: Create required directories
run: |
mkdir -p reports/junit
- name: Run Integration Tests (Fast Only)
run: |
echo "╔════════════════════════════════════════════════════════════╗"
echo "║ JuniperData - Integration Tests (Fast Only) ║"
echo "╚════════════════════════════════════════════════════════════╝"
python -m pytest \
-m "integration and not slow" \
juniper_data/tests/integration \
--verbose \
--timeout=120 \
--maxfail=3 \
--junitxml=reports/junit-integration.xml
- name: Upload Integration Test Results
uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
if: always()
with:
name: integration-test-results
path: reports/junit-integration.xml
retention-days: 30
# ═══════════════════════════════════════════════════════════════════════════════════════════════
# Security: Run security scans (Gitleaks, Bandit, pip-audit)
# ═══════════════════════════════════════════════════════════════════════════════════════════════
security:
name: Security Scans
runs-on: ubuntu-latest
needs: [pre-commit]
permissions:
contents: read
security-events: write
steps:
- name: Checkout Code
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
fetch-depth: 0
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
cache: pip
- name: Install security tools
run: |
python -m pip install --upgrade pip
pip install "bandit[sarif]" pip-audit
- name: Create reports directory
run: mkdir -p reports/security
- name: Run Gitleaks (Secrets Detection)
uses: gitleaks/gitleaks-action@v2
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Run Bandit (SAST)
run: |
echo "╔════════════════════════════════════════════════════════════╗"
echo "║ JuniperData - Bandit Security Scan ║"
echo "╚════════════════════════════════════════════════════════════╝"
# Generate SARIF report (always succeeds for upload)
bandit -r juniper_data -x juniper_data/tests -f sarif -o reports/security/bandit.sarif --exit-zero
# Blocking check - fail on medium or higher severity
bandit -r juniper_data -x juniper_data/tests --confidence-level medium --severity-level medium
- name: Upload Bandit SARIF to GitHub Security
uses: github/codeql-action/upload-sarif@48ab28a6f5dbc2a99bf1e0131198dd8f1df78169 # v3.28.0
if: always()
with:
sarif_file: reports/security/bandit.sarif
continue-on-error: true
- name: Run pip-audit (Dependency Vulnerabilities)
run: |
echo "╔════════════════════════════════════════════════════════════╗"
echo "║ JuniperData - pip-audit Dependency Scan ║"
echo "╚════════════════════════════════════════════════════════════╝"
pip install ".[all]"
pip freeze | grep -ivE "^juniper[-_]data" > reports/security/requirements.txt
pip-audit -r reports/security/requirements.txt --strict
- name: Upload Security Reports
uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
if: always()
with:
name: security-reports
path: reports/security/
retention-days: 30
# ═══════════════════════════════════════════════════════════════════════════════════════════════
# Required Checks Aggregator: Final quality gate
# ═══════════════════════════════════════════════════════════════════════════════════════════════
required-checks:
name: Quality Gate
runs-on: ubuntu-latest
if: always()
needs: [pre-commit, unit-tests, build, integration-tests, security]
steps:
- name: Check Quality Gate Status
run: |
echo "╔════════════════════════════════════════════════════════════╗"
echo "║ JuniperData - Quality Gate Status ║"
echo "╚════════════════════════════════════════════════════════════╝"
echo ""
echo "Pre-commit: ${{ needs.pre-commit.result }}"
echo "Unit Tests: ${{ needs.unit-tests.result }}"
echo "Build: ${{ needs.build.result }}"
echo "Integration Tests: ${{ needs.integration-tests.result }}"
echo "Security: ${{ needs.security.result }}"
echo ""
# Pre-commit and unit-tests MUST pass
if [[ "${{ needs.pre-commit.result }}" != "success" ]]; then
echo "::error::Pre-commit checks failed"
exit 1
fi
if [[ "${{ needs.unit-tests.result }}" != "success" ]]; then
echo "::error::Unit tests failed"
exit 1
fi
if [[ "${{ needs.build.result }}" != "success" ]]; then
echo "::error::Build failed"
exit 1
fi
# Security should pass (but SARIF upload failures are allowed)
if [[ "${{ needs.security.result }}" == "failure" ]]; then
echo "::error::Security scans failed"
exit 1
fi
# Integration tests: failure = error, skipped = OK (for feature branches)
if [[ "${{ needs.integration-tests.result }}" == "failure" ]]; then
echo "::error::Integration tests failed"
exit 1
fi
echo "::notice::Quality Gate PASSED ✓"
# ═══════════════════════════════════════════════════════════════════════════════════════════════
# Slow Tests: Run tests marked as slow (weekly or on-demand)
# ═══════════════════════════════════════════════════════════════════════════════════════════════
slow-tests:
name: Slow Tests
runs-on: ubuntu-latest
if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
steps:
- name: Checkout Code
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.14"
cache: pip
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install ".[all]"
- name: Run Slow Tests
run: |
echo "╔════════════════════════════════════════════════════════════╗"
echo "║ JuniperData - Slow Tests ║"
echo "╚════════════════════════════════════════════════════════════╝"
python -m pytest -m "slow" --verbose --timeout=600 || echo "No slow tests found"
# ═══════════════════════════════════════════════════════════════════════════════════════════════
# Notify: Build status notification
# ═══════════════════════════════════════════════════════════════════════════════════════════════
notify:
name: Build Notification
runs-on: ubuntu-latest
needs: [required-checks]
if: always()
steps:
- name: Build Status Summary
run: |
echo "╔════════════════════════════════════════════════════════════╗"
echo "║ JuniperData - Build Complete ║"
echo "╚════════════════════════════════════════════════════════════╝"
echo ""
echo "========== CI/CD Pipeline Summary =============================="
echo "Status: ${{ needs.required-checks.result }}"
echo "Workflow: ${{ github.workflow }}"
echo "Branch: ${{ github.ref_name }}"
echo "Commit: ${{ github.sha }}"
echo "Actor: ${{ github.actor }}"
echo "Event: ${{ github.event_name }}"
echo "================================================================"