Run Nightly Tests #38
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Run Nightly Tests | |
| on: | |
| schedule: | |
| - cron: '0 20 * * *' # 20:00 UTC daily (offset from integration at 18:00 UTC) | |
| workflow_dispatch: # Allow manual trigger | |
| jobs: | |
| run-nightly: | |
| runs-on: self-hosted | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Checkout datus-db-adapters main | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: Datus-ai/datus-db-adapters | |
| ref: main | |
| path: external/datus-db-adapters | |
| - name: Checkout datus-bi-adapters main | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: Datus-ai/datus-bi-adapters | |
| ref: main | |
| path: external/datus-bi-adapters | |
| - name: Checkout datus-scheduler-adapters main | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: Datus-ai/datus-scheduler-adapters | |
| ref: main | |
| path: external/datus-scheduler-adapters | |
| - name: Clean up stale temp files | |
| run: | | |
| echo "Cleaning stale pytest temp directories..." | |
| find /tmp/pytest-of-* -maxdepth 1 -type d -mtime +1 -exec rm -rf {} + 2>/dev/null || true | |
| echo "Cleaning stale .pyc caches..." | |
| find "$GITHUB_WORKSPACE" -type d -name __pycache__ -exec rm -rf {} + 2>/dev/null || true | |
| echo "Disk usage: $(df -h /tmp | tail -1 | awk '{print $5, $4}')" | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v5 | |
| - name: Install dependencies | |
| run: | | |
| uv sync | |
| uv pip install --upgrade --no-cache-dir \ | |
| "datus-semantic-metricflow>=0.1.3" \ | |
| pytest-playwright \ | |
| ./external/datus-db-adapters/datus-db-core \ | |
| ./external/datus-db-adapters/datus-sqlalchemy \ | |
| ./external/datus-db-adapters/datus-postgresql \ | |
| ./external/datus-db-adapters/datus-mysql \ | |
| ./external/datus-db-adapters/datus-clickhouse \ | |
| ./external/datus-db-adapters/datus-starrocks \ | |
| ./external/datus-db-adapters/datus-trino \ | |
| ./external/datus-db-adapters/datus-greenplum \ | |
| ./external/datus-db-adapters/datus-hive \ | |
| ./external/datus-db-adapters/datus-spark \ | |
| ./external/datus-bi-adapters/datus-bi-core \ | |
| ./external/datus-bi-adapters/datus-bi-superset \ | |
| ./external/datus-scheduler-adapters/datus-scheduler-core \ | |
| ./external/datus-scheduler-adapters/datus-scheduler-airflow | |
| uv run playwright install chromium | |
| - name: Setup config file | |
| run: | | |
| cp /home/datus_ci/ci_template/agent.yml conf/agent.yml | |
| cp /home/datus_ci/ci_template/ci.env .env | |
| - name: Restore tracked test config | |
| run: git restore --source=HEAD --worktree --staged tests/conf/agent.yml | |
| - name: Isolate test data directory | |
| run: | | |
| DATUS_TEST_HOME="${GITHUB_WORKSPACE}/.datus_test_data" | |
| echo "DATUS_TEST_HOME=$DATUS_TEST_HOME" >> $GITHUB_ENV | |
| sed -i "s|home: .datus_test_data|home: $DATUS_TEST_HOME|g" tests/conf/agent.yml | |
| sed -i "s|project_root: .datus_test_data/workspace|project_root: $DATUS_TEST_HOME/workspace|g" tests/conf/agent.yml | |
| - name: Load environment variables | |
| run: | | |
| echo "Loading environment variables from .env file..." | |
| while IFS= read -r line || [ -n "$line" ]; do | |
| if [[ ! -z "$line" && ! "$line" =~ ^[[:space:]]*# ]]; then | |
| var_name=$(echo "$line" | cut -d'=' -f1) | |
| var_value=$(echo "$line" | cut -d'=' -f2-) | |
| echo "::add-mask::$var_value" | |
| echo "Loading variable: $var_name" | |
| echo "$line" >> $GITHUB_ENV | |
| fi | |
| done < .env | |
| echo "Environment variables loaded successfully" | |
| - name: Enable strict nightly test requirements | |
| run: | | |
| echo "DATUS_STRICT_NIGHTLY_REQUIREMENTS=1" >> $GITHUB_ENV | |
| echo "ADAPTERS_PG=1" >> $GITHUB_ENV | |
| echo "ADAPTERS_MYSQL=1" >> $GITHUB_ENV | |
| echo "ADAPTERS_CH=1" >> $GITHUB_ENV | |
| echo "ADAPTERS_SR=1" >> $GITHUB_ENV | |
| echo "ADAPTERS_TRINO=1" >> $GITHUB_ENV | |
| echo "ADAPTERS_GP=1" >> $GITHUB_ENV | |
| echo "ADAPTERS_HIVE=1" >> $GITHUB_ENV | |
| echo "ADAPTERS_SPARK=1" >> $GITHUB_ENV | |
| - name: Set up Node.js | |
| uses: actions/setup-node@v6 | |
| with: | |
| node-version: '22' | |
| - name: Restore knowledge base cache | |
| id: kb-cache | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: | | |
| ${{ github.workspace }}/.datus_test_data/data/datus_db_bird_school | |
| ${{ github.workspace }}/.datus_test_data/data/datus_db_ssb_sqlite | |
| key: ${{ runner.os }}-kb-${{ hashFiles('build_scripts/build_test_data.sh', 'sample_data/**', 'datus/storage/**/*.py') }} | |
| restore-keys: | | |
| ${{ runner.os }}-kb- | |
| - name: Build test data | |
| id: build-test-data | |
| if: steps.kb-cache.outputs.cache-hit != 'true' | |
| run: | | |
| echo "Building test data before running nightly tests..." | |
| chmod +x build_scripts/build_test_data.sh | |
| ./build_scripts/build_test_data.sh || exit 1 | |
| echo "Test data build completed" | |
| - name: Save knowledge base cache | |
| if: steps.build-test-data.outcome == 'success' | |
| uses: actions/cache/save@v4 | |
| with: | |
| path: | | |
| ${{ github.workspace }}/.datus_test_data/data/datus_db_bird_school | |
| ${{ github.workspace }}/.datus_test_data/data/datus_db_ssb_sqlite | |
| key: ${{ runner.os }}-kb-${{ hashFiles('build_scripts/build_test_data.sh', 'sample_data/**', 'datus/storage/**/*.py') }} | |
| - name: Verify FEISHU_WEBHOOK_URL | |
| env: | |
| FEISHU_WEBHOOK_URL: ${{ secrets.FEISHU_NOTIFY_CI_URL }} | |
| run: | | |
| if [ -n "$FEISHU_WEBHOOK_URL" ]; then | |
| echo "FEISHU_WEBHOOK_URL is set (length: ${#FEISHU_WEBHOOK_URL} characters)" | |
| else | |
| echo "FEISHU_WEBHOOK_URL is not set or empty" | |
| fi | |
| - name: Run nightly tests | |
| id: nightly_tests | |
| timeout-minutes: 150 | |
| run: | | |
| set -o pipefail | |
| LOG_FILE="test_output_nightly_$(date +%Y%m%d_%H%M%S).log" | |
| test_exit_code=0 | |
| EXTERNAL_REPOS_ROOT="${GITHUB_WORKSPACE}/external" | |
| POSTGRES_COMPOSE="${EXTERNAL_REPOS_ROOT}/datus-db-adapters/datus-postgresql/docker-compose.yml" | |
| MYSQL_COMPOSE="${EXTERNAL_REPOS_ROOT}/datus-db-adapters/datus-mysql/docker-compose.yml" | |
| CLICKHOUSE_COMPOSE="${EXTERNAL_REPOS_ROOT}/datus-db-adapters/datus-clickhouse/docker-compose.yml" | |
| STARROCKS_COMPOSE="${EXTERNAL_REPOS_ROOT}/datus-db-adapters/datus-starrocks/docker-compose.yml" | |
| TRINO_COMPOSE="${EXTERNAL_REPOS_ROOT}/datus-db-adapters/datus-trino/docker-compose.yml" | |
| GREENPLUM_COMPOSE="${EXTERNAL_REPOS_ROOT}/datus-db-adapters/datus-greenplum/docker-compose.yml" | |
| HIVE_COMPOSE="${EXTERNAL_REPOS_ROOT}/datus-db-adapters/datus-hive/docker-compose.yml" | |
| SPARK_COMPOSE="${EXTERNAL_REPOS_ROOT}/datus-db-adapters/datus-spark/docker-compose.yml" | |
| SUPERSET_COMPOSE="${EXTERNAL_REPOS_ROOT}/datus-bi-adapters/datus-bi-superset/tests/integration/docker-compose.yml" | |
| AIRFLOW_COMPOSE="${EXTERNAL_REPOS_ROOT}/datus-scheduler-adapters/datus-scheduler-airflow/docker-compose.yml" | |
| run_logged() { | |
| local group_name="$1" | |
| shift | |
| echo -e "\n=== ${group_name} ===" | tee -a "$LOG_FILE" | |
| "$@" 2>&1 | tee -a "$LOG_FILE" | |
| local cmd_status=${PIPESTATUS[0]} | |
| if [ "$cmd_status" -ne 0 ]; then | |
| test_exit_code="$cmd_status" | |
| fi | |
| return 0 | |
| } | |
| compose_down() { | |
| local compose_file="$1" | |
| docker compose -f "$compose_file" down -v --remove-orphans >/dev/null 2>&1 || true | |
| } | |
| compose_up() { | |
| local compose_file="$1" | |
| shift | |
| if [ ! -f "$compose_file" ]; then | |
| echo "Missing compose file: $compose_file" | tee -a "$LOG_FILE" >&2 | |
| test_exit_code=1 | |
| return 1 | |
| fi | |
| docker compose -f "$compose_file" up -d --build "$@" 2>&1 | tee -a "$LOG_FILE" | |
| local cmd_status=${PIPESTATUS[0]} | |
| if [ "$cmd_status" -ne 0 ]; then | |
| test_exit_code="$cmd_status" | |
| return 1 | |
| fi | |
| return 0 | |
| } | |
| wait_for_service_health() { | |
| local compose_file="$1" | |
| local service_name="$2" | |
| local timeout_seconds="$3" | |
| local container_id="" | |
| local status="" | |
| local deadline=$((SECONDS + timeout_seconds)) | |
| container_id="$(docker compose -f "$compose_file" ps -q "$service_name")" | |
| if [ -z "$container_id" ]; then | |
| echo "No container found for service '$service_name' in $compose_file" | tee -a "$LOG_FILE" >&2 | |
| docker compose -f "$compose_file" ps 2>&1 | tee -a "$LOG_FILE" || true | |
| test_exit_code=1 | |
| return 1 | |
| fi | |
| while [ "$SECONDS" -lt "$deadline" ]; do | |
| status="$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$container_id" 2>/dev/null || echo unknown)" | |
| if [ "$status" = "healthy" ] || [ "$status" = "running" ]; then | |
| echo "Service '$service_name' is $status" | tee -a "$LOG_FILE" | |
| return 0 | |
| fi | |
| sleep 5 | |
| done | |
| echo "Timed out waiting for service '$service_name' from $compose_file" | tee -a "$LOG_FILE" >&2 | |
| docker compose -f "$compose_file" ps 2>&1 | tee -a "$LOG_FILE" || true | |
| docker compose -f "$compose_file" logs --tail=200 2>&1 | tee -a "$LOG_FILE" || true | |
| test_exit_code=1 | |
| return 1 | |
| } | |
| run_compose_suite() { | |
| local group_name="$1" | |
| local compose_file="$2" | |
| shift 2 | |
| local service_specs=() | |
| while [ "$#" -gt 0 ] && [ "$1" != "--" ]; do | |
| service_specs+=("$1") | |
| shift | |
| done | |
| shift | |
| echo -e "\n=== Starting ${group_name} Services ===" | tee -a "$LOG_FILE" | |
| compose_down "$compose_file" | |
| if ! compose_up "$compose_file"; then | |
| compose_down "$compose_file" | |
| return 0 | |
| fi | |
| local spec | |
| for spec in "${service_specs[@]}"; do | |
| local service_name="${spec%%:*}" | |
| local timeout_seconds="${spec##*:}" | |
| if ! wait_for_service_health "$compose_file" "$service_name" "$timeout_seconds"; then | |
| compose_down "$compose_file" | |
| return 0 | |
| fi | |
| done | |
| run_logged "$group_name" "$@" | |
| echo -e "\n=== Stopping ${group_name} Services ===" | tee -a "$LOG_FILE" | |
| compose_down "$compose_file" | |
| return 0 | |
| } | |
| # Run service-free groups first. Compose-backed nightly tests then run | |
| # serially, each with its own docker compose lifecycle. | |
| # Group 0: full unit-test suite | |
| run_logged "Full Unit Tests" uv run pytest tests/unit_tests/ -m "not nightly" --tb=short --verbose --timeout=300 --dist=loadscope -n auto | |
| # Group 1: MCP server tests (event loop sensitive, no xdist) | |
| run_logged "MCP Server Tests" uv run pytest -m nightly tests/integration/tools/test_mcp_server.py --tb=short --verbose --timeout=60 | |
| # Group 2: gen_* agent tests in dependency order | |
| run_logged "Gen Agent Tests" uv run pytest -m nightly tests/integration/agent/test_gen_semantic_model_agentic.py tests/integration/agent/test_gen_metrics_agentic.py tests/integration/agent/test_gen_ext_knowledge_agentic.py --tb=short --verbose --timeout=600 --reruns 1 --reruns-delay 5 | |
| # Group 3: Remaining nightly tests that do not require compose-backed services. | |
| run_logged "Main Nightly Tests" uv run pytest -m nightly tests/ --deselect tests/integration/tools/test_mcp_server.py --deselect tests/integration/agent/test_gen_semantic_model_agentic.py --deselect tests/integration/agent/test_gen_metrics_agentic.py --deselect tests/integration/agent/test_gen_ext_knowledge_agentic.py --deselect tests/integration/agent/test_gen_dashboard_agentic.py --deselect tests/integration/agent/test_scheduler_agentic.py --deselect tests/integration/tools/test_bi_dashboard.py --deselect tests/integration/adapters/test_postgresql.py --deselect tests/integration/adapters/test_mysql.py --deselect tests/integration/adapters/test_clickhouse.py --deselect tests/integration/adapters/test_starrocks.py --deselect tests/integration/adapters/test_trino.py --deselect tests/integration/adapters/test_greenplum.py --deselect tests/integration/adapters/test_hive.py --deselect tests/integration/adapters/test_spark.py --tb=short --verbose --timeout=300 --reruns 1 --reruns-delay 5 --dist=loadscope -n auto | |
| # Group 4: Superset-backed nightly tests | |
| run_compose_suite "Superset Nightly Tests" "$SUPERSET_COMPOSE" "postgres:300" "superset:1200" -- uv run pytest -m nightly tests/integration/agent/test_gen_dashboard_agentic.py tests/integration/tools/test_bi_dashboard.py --tb=short --verbose --timeout=600 --reruns 1 --reruns-delay 5 | |
| # Group 5: Airflow-backed nightly tests | |
| run_compose_suite "Airflow Nightly Tests" "$AIRFLOW_COMPOSE" "airflow:900" -- uv run pytest -m nightly tests/integration/agent/test_scheduler_agentic.py --tb=short --verbose --timeout=600 --reruns 1 --reruns-delay 5 | |
| # Group 6+: Adapter contract tests, one compose environment at a time. | |
| run_compose_suite "PostgreSQL Adapter Tests" "$POSTGRES_COMPOSE" "postgres:300" -- uv run pytest -m nightly tests/integration/adapters/test_postgresql.py --tb=short --verbose --timeout=300 | |
| run_compose_suite "MySQL Adapter Tests" "$MYSQL_COMPOSE" "mysql:300" -- uv run pytest -m nightly tests/integration/adapters/test_mysql.py --tb=short --verbose --timeout=300 | |
| run_compose_suite "ClickHouse Adapter Tests" "$CLICKHOUSE_COMPOSE" "clickhouse:300" -- uv run pytest -m nightly tests/integration/adapters/test_clickhouse.py --tb=short --verbose --timeout=300 | |
| run_compose_suite "StarRocks Adapter Tests" "$STARROCKS_COMPOSE" "starrocks:600" -- uv run pytest -m nightly tests/integration/adapters/test_starrocks.py --tb=short --verbose --timeout=300 | |
| run_compose_suite "Trino Adapter Tests" "$TRINO_COMPOSE" "trino:300" -- uv run pytest -m nightly tests/integration/adapters/test_trino.py --tb=short --verbose --timeout=300 | |
| run_compose_suite "Greenplum Adapter Tests" "$GREENPLUM_COMPOSE" "greenplum:600" -- uv run pytest -m nightly tests/integration/adapters/test_greenplum.py --tb=short --verbose --timeout=300 | |
| run_compose_suite "Hive Adapter Tests" "$HIVE_COMPOSE" "hive-metastore:600" "hive-server:900" -- uv run pytest -m nightly tests/integration/adapters/test_hive.py --tb=short --verbose --timeout=300 | |
| run_compose_suite "Spark Adapter Tests" "$SPARK_COMPOSE" "spark-thrift:900" -- uv run pytest -m nightly tests/integration/adapters/test_spark.py --tb=short --verbose --timeout=300 | |
| echo "log_file=$LOG_FILE" >> "$GITHUB_OUTPUT" | |
| echo "test_exit_code=$test_exit_code" >> "$GITHUB_OUTPUT" | |
| exit "$test_exit_code" | |
| - name: Stop nightly services | |
| if: always() | |
| run: | | |
| set +e | |
| EXTERNAL_REPOS_ROOT="${GITHUB_WORKSPACE}/external" | |
| COMPOSE_FILES=( | |
| "${EXTERNAL_REPOS_ROOT}/datus-db-adapters/datus-postgresql/docker-compose.yml" | |
| "${EXTERNAL_REPOS_ROOT}/datus-db-adapters/datus-mysql/docker-compose.yml" | |
| "${EXTERNAL_REPOS_ROOT}/datus-db-adapters/datus-clickhouse/docker-compose.yml" | |
| "${EXTERNAL_REPOS_ROOT}/datus-db-adapters/datus-starrocks/docker-compose.yml" | |
| "${EXTERNAL_REPOS_ROOT}/datus-db-adapters/datus-trino/docker-compose.yml" | |
| "${EXTERNAL_REPOS_ROOT}/datus-db-adapters/datus-greenplum/docker-compose.yml" | |
| "${EXTERNAL_REPOS_ROOT}/datus-db-adapters/datus-hive/docker-compose.yml" | |
| "${EXTERNAL_REPOS_ROOT}/datus-db-adapters/datus-spark/docker-compose.yml" | |
| "${EXTERNAL_REPOS_ROOT}/datus-bi-adapters/datus-bi-superset/tests/integration/docker-compose.yml" | |
| "${EXTERNAL_REPOS_ROOT}/datus-scheduler-adapters/datus-scheduler-airflow/docker-compose.yml" | |
| ) | |
| for compose_file in "${COMPOSE_FILES[@]}"; do | |
| if [ -f "$compose_file" ]; then | |
| echo "Stopping services from $compose_file" | |
| docker compose -f "$compose_file" down -v --remove-orphans || true | |
| fi | |
| done | |
| - name: Upload test results | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: nightly-test-results-${{ github.run_number }} | |
| path: test_output_nightly_*.log | |
| retention-days: 7 | |
| - name: Send Feishu notification | |
| if: always() | |
| uses: actions/github-script@v7 | |
| env: | |
| FEISHU_WEBHOOK_URL: ${{ secrets.FEISHU_NOTIFY_CI_URL }} | |
| with: | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| script: | | |
| const fs = require('fs'); | |
| let benchmarkReport = ''; | |
| try { | |
| const logFiles = fs.readdirSync('.').filter(file => file.startsWith('test_output_nightly_') && file.endsWith('.log')); | |
| const latestLogFile = logFiles.sort().pop(); | |
| const { execSync } = require('child_process'); | |
| let logContent = ''; | |
| try { | |
| logContent = execSync(`tail -200 "${latestLogFile || 'test_output_nightly.log'}"`, { encoding: 'utf8' }); | |
| } catch (error) { | |
| logContent = fs.readFileSync(latestLogFile || 'test_output_nightly.log', 'utf8'); | |
| } | |
| let separator = '────────────────────────────────────────────────────────────────────────────────' | |
| const benchmarkStart = logContent.indexOf(separator); | |
| let benchmarkEnd = logContent.lastIndexOf(separator); | |
| if (benchmarkStart !== -1) { | |
| benchmarkEnd = benchmarkEnd === -1 ? logContent.length : benchmarkEnd + separator.length; | |
| benchmarkReport = logContent.substring(benchmarkStart, benchmarkEnd); | |
| } else { | |
| benchmarkReport = logContent; | |
| } | |
| } catch (error) { | |
| console.log('Could not read test report from log'); | |
| } | |
| const testStatus = '${{ steps.nightly_tests.outputs.test_exit_code }}' === '0' ? 'PASSED' : 'FAILED'; | |
| const today = new Date().toISOString().split('T')[0]; | |
| const messageBody = `## Daily Nightly Test Report - ${today} | |
| **Status:** ${testStatus} | |
| **Runs Details:** [${{ github.run_number }}](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) | |
| ### Test Results Summary | |
| \`\`\` | |
| ${benchmarkReport || 'Test completed - detailed report in artifacts'} | |
| \`\`\` | |
| --- | |
| *This is an automated report generated by the nightly test workflow.* | |
| `; | |
| // Send to Feishu | |
| const https = require('https'); | |
| const feishuWebhookUrl = process.env.FEISHU_WEBHOOK_URL; | |
| if (feishuWebhookUrl) { | |
| console.log('FEISHU_WEBHOOK_URL is set in JS environment.'); | |
| } else { | |
| console.error('FEISHU_WEBHOOK_URL is not set in JS environment.'); | |
| } | |
| if (!feishuWebhookUrl) { | |
| console.error('FEISHU_WEBHOOK_URL environment variable is not set'); | |
| return; | |
| } | |
| const url = new URL(feishuWebhookUrl); | |
| const data = JSON.stringify({ | |
| "msg_type": "interactive", | |
| "card": { | |
| "type": "template", | |
| "data": { | |
| "template_id": "AAqz7JC34dHlY", | |
| "template_version_name": "1.0.0", | |
| "template_variable": { | |
| "report_result": messageBody | |
| } | |
| } | |
| } | |
| }); | |
| const options = { | |
| hostname: url.hostname, | |
| port: url.port || 443, | |
| path: url.pathname + url.search, | |
| method: 'POST', | |
| headers: { | |
| 'Content-Type': 'application/json', | |
| 'Content-Length': Buffer.byteLength(data) | |
| } | |
| }; | |
| const req = https.request(options, (res) => { | |
| console.log('Feishu notification sent:', res.statusCode); | |
| }); | |
| req.on('error', (error) => { | |
| console.error('Error sending to Feishu:', error); | |
| }); | |
| req.write(data); | |
| req.end(); |