🔍 File Processing Matrix Verification Agent #177
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| name: 🔍 File Processing Matrix Verification Agent | |
| "on": | |
| push: | |
| branches: [main, develop] | |
| paths: | |
| - '**/*.md' | |
| - '**/*.py' | |
| - '**/*.js' | |
| - '**/*.ts' | |
| - '**/*.json' | |
| - '**/*.yml' | |
| - '**/*.yaml' | |
| pull_request: | |
| branches: [main] | |
| types: [opened, synchronize, reopened] | |
| schedule: | |
| # Run every 6 hours to verify matrix compliance | |
| - cron: '0 */6 * * *' | |
| workflow_dispatch: | |
| inputs: | |
| verification_scope: | |
| description: 'Scope of verification to perform' | |
| required: true | |
| default: 'full_matrix' | |
| type: choice | |
| options: | |
| - full_matrix | |
| - workflow_compliance | |
| - documentation_sync | |
| - performance_metrics | |
| generate_report: | |
| description: 'Generate detailed compliance report' | |
| required: false | |
| default: true | |
| type: boolean | |
| env: | |
| MATRIX_COMPLIANCE_VERSION: "1.0" | |
| VERIFICATION_ENABLED: true | |
| jobs: | |
| verify-file-processing-matrix: | |
| name: 🔍 Verify File Processing Matrix Compliance | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: write | |
| pull-requests: write | |
| issues: write | |
| steps: | |
| - name: 🔍 Checkout Repository | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: 🐍 Setup Python Environment | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.11' | |
| cache: 'pip' | |
| - name: 📦 Install Dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install pyyaml requests python-frontmatter markdownify beautifulsoup4 | |
| - name: 📊 Verify File Processing Matrix Implementation | |
| id: matrix_verification | |
| run: | | |
| python3 << 'EOF' | |
| import os | |
| import yaml | |
| import json | |
| from pathlib import Path | |
| from datetime import datetime | |
| print("🔍 Starting File Processing Matrix Verification...") | |
| # Load workflows and documentation | |
| workflows_dir = Path('.github/workflows') | |
| workflows_docs_dir = Path('.workflows') | |
| verification_results = { | |
| "verification_timestamp": datetime.now().isoformat(), | |
| "matrix_compliance": {}, | |
| "workflow_coverage": {}, | |
| "missing_implementations": [], | |
| "performance_metrics": {}, | |
| "recommendations": [] | |
| } | |
| # Expected file types from matrix | |
| expected_file_types = ['.md', '.py', '.js', '.ts', '.json', '.yml', '.yaml'] | |
| # Check each workflow for file type coverage | |
| total_workflows = 0 | |
| compliant_workflows = 0 | |
| for workflow_file in workflows_dir.glob('*.yml'): | |
| total_workflows += 1 | |
| with open(workflow_file, 'r') as f: | |
| workflow_content = f.read() | |
| try: | |
| workflow_data = yaml.safe_load(workflow_content) | |
| except yaml.YAMLError as e: | |
| print(f"❌ Invalid YAML in {workflow_file.name}: {e}") | |
| continue | |
| workflow_name = workflow_file.stem | |
| print(f"📋 Analyzing workflow: {workflow_name}") | |
| # Check if workflow has file type triggers | |
| file_triggers = [] | |
| has_file_processing = False | |
| if 'on' in workflow_data and workflow_data['on']: | |
| on_config = workflow_data['on'] | |
| if isinstance(on_config, dict): | |
| # Check push triggers | |
| if 'push' in on_config and isinstance(on_config['push'], dict): | |
| if 'paths' in on_config['push']: | |
| file_triggers.extend(on_config['push']['paths']) | |
| # Check pull request triggers | |
| if 'pull_request' in on_config and isinstance(on_config['pull_request'], dict): | |
| if 'paths' in on_config['pull_request']: | |
| file_triggers.extend(on_config['pull_request']['paths']) | |
| # Check if covers expected file types | |
| if file_triggers: | |
| covered_types = [] | |
| for ext in expected_file_types: | |
| if any(ext in str(trigger) for trigger in file_triggers): | |
| covered_types.append(ext) | |
| has_file_processing = len(covered_types) >= 5 # At least 5 of 7 file types | |
| if has_file_processing: | |
| compliant_workflows += 1 | |
| verification_results["workflow_coverage"][workflow_name] = { | |
| "triggers": file_triggers, | |
| "covers_matrix_files": has_file_processing, | |
| "file_types_covered": len([ext for ext in expected_file_types if any(ext in str(trigger) for trigger in file_triggers)]) | |
| } | |
| status = "✅ COMPLIANT" if has_file_processing else "❌ NON-COMPLIANT" | |
| print(f" {status} - {len(file_triggers)} triggers, covers {verification_results['workflow_coverage'][workflow_name]['file_types_covered']}/7 file types") | |
| # Calculate matrix compliance score | |
| compliance_score = compliant_workflows / total_workflows if total_workflows > 0 else 0 | |
| verification_results["matrix_compliance"] = { | |
| "score": compliance_score, | |
| "compliant_workflows": compliant_workflows, | |
| "total_workflows": total_workflows, | |
| "compliance_percentage": f"{compliance_score:.1%}" | |
| } | |
| print(f"\n📊 Matrix Compliance Summary:") | |
| print(f" Compliant Workflows: {compliant_workflows}/{total_workflows}") | |
| print(f" Compliance Score: {compliance_score:.1%}") | |
| # Check for missing documented workflows | |
| documented_workflows = { | |
| "integrated-documentation-validation.yml": "Security & validation workflow", | |
| "auto-documentation-update.yml": "Automatic documentation updates", | |
| "ai-orchestration-setup.yml": "AI coordination and setup", | |
| "copilot-documentation-agent.yml": "Copilot-powered documentation agent", | |
| "file-processing-matrix-verification.yml": "Matrix compliance verification", | |
| "github-models-documentation-enhancer.yml": "GitHub Models AI enhancement" | |
| } | |
| missing_count = 0 | |
| for workflow_name, description in documented_workflows.items(): | |
| workflow_path = workflows_dir / workflow_name | |
| if workflow_path.exists(): | |
| print(f"✅ {description} - IMPLEMENTED") | |
| else: | |
| print(f"❌ {description} - MISSING") | |
| verification_results["missing_implementations"].append(f"{workflow_name}: {description}") | |
| missing_count += 1 | |
| # Performance metrics | |
| verification_results["performance_metrics"] = { | |
| "estimated_processing_times": { | |
| ".md": "15-30s per file", | |
| ".py": "5-10s per file", | |
| ".js/.ts": "10-15s per file", | |
| ".json": "1-2s per file", | |
| ".yml/.yaml": "2-5s per file" | |
| }, | |
| "optimization_status": "Parallel processing configured", | |
| "current_compliance": f"{compliance_score:.1%}", | |
| "target_compliance": "100%" | |
| } | |
| # Generate recommendations | |
| if compliance_score < 0.8: | |
| verification_results["recommendations"].append( | |
| f"Matrix compliance is {compliance_score:.1%} - improve workflow file type coverage" | |
| ) | |
| if missing_count > 0: | |
| verification_results["recommendations"].append( | |
| f"Implement {missing_count} missing workflows for full documentation compliance" | |
| ) | |
| if compliance_score >= 0.8: | |
| verification_results["recommendations"].append( | |
| "Excellent compliance! Consider optimizing performance and adding advanced features" | |
| ) | |
| # Save verification report | |
| with open('matrix_verification_report.json', 'w') as f: | |
| json.dump(verification_results, f, indent=2) | |
| print(f"\n✅ File Processing Matrix Verification completed!") | |
| print(f"📊 Final Compliance Score: {compliance_score:.1%}") | |
| print(f"🔧 Recommendations: {len(verification_results['recommendations'])}") | |
| # Set outputs using proper GitHub Actions syntax | |
| with open(os.environ['GITHUB_OUTPUT'], 'a') as f: | |
| f.write(f"compliance_score={compliance_score:.2f}\n") | |
| f.write(f"missing_count={missing_count}\n") | |
| f.write(f"compliant_workflows={compliant_workflows}\n") | |
| f.write(f"total_workflows={total_workflows}\n") | |
| EOF | |
| - name: 📋 Generate Compliance Report | |
| if: github.event.inputs.generate_report != 'false' | |
| run: | | |
| python3 << 'EOF' | |
| import json | |
| from datetime import datetime | |
| # Load verification results | |
| with open('matrix_verification_report.json', 'r') as f: | |
| results = json.load(f) | |
| # Generate markdown report | |
| report = f"""# 🔍 File Processing Matrix Compliance Report | |
| **Generated:** {results['verification_timestamp']} | |
| **Verification Scope:** ${{ github.event.inputs.verification_scope || 'full_matrix' }} | |
| ## 📊 Compliance Summary | |
| - **Matrix Compliance Score:** {results['matrix_compliance']['compliance_percentage']} | |
| - **Compliant Workflows:** {results['matrix_compliance']['compliant_workflows']}/{results['matrix_compliance']['total_workflows']} | |
| - **Status:** {'🟢 EXCELLENT' if results['matrix_compliance']['score'] >= 0.8 else '🟡 NEEDS IMPROVEMENT' if results['matrix_compliance']['score'] >= 0.6 else '🔴 CRITICAL'} | |
| ## 🔄 Workflow Coverage Analysis | |
| """ | |
| for workflow, data in results['workflow_coverage'].items(): | |
| status = "✅ COMPLIANT" if data['covers_matrix_files'] else "❌ NON-COMPLIANT" | |
| report += f"### {workflow} - {status}\n" | |
| report += f"- **File Type Coverage:** {data['file_types_covered']}/7 file types\n" | |
| report += f"- **File Triggers:** {len(data['triggers'])} configured\n\n" | |
| if results['missing_implementations']: | |
| report += "## ❌ Missing Implementations\n\n" | |
| for missing in results['missing_implementations']: | |
| report += f"- {missing}\n" | |
| report += "\n" | |
| if results['recommendations']: | |
| report += "## 💡 Recommendations\n\n" | |
| for rec in results['recommendations']: | |
| report += f"- {rec}\n" | |
| report += "\n" | |
| report += f"""## ⚡ Performance Metrics | |
| **Estimated Processing Times:** | |
| """ | |
| for file_type, time in results['performance_metrics']['estimated_processing_times'].items(): | |
| report += f"- **{file_type}:** {time}\n" | |
| report += f""" | |
| **Current Status:** {results['performance_metrics']['optimization_status']} | |
| **Compliance Target:** {results['performance_metrics']['target_compliance']} | |
| --- | |
| *This report is automatically generated every 6 hours by the File Processing Matrix Verification Agent.* | |
| """ | |
| # Save report | |
| with open('MATRIX_COMPLIANCE_REPORT.md', 'w') as f: | |
| f.write(report) | |
| print("📋 Compliance report generated: MATRIX_COMPLIANCE_REPORT.md") | |
| EOF | |
| - name: 🚨 Check Compliance Threshold | |
| id: compliance_check | |
| run: | | |
| COMPLIANCE_SCORE=$(cat matrix_verification_report.json | python3 -c " | |
| import sys, json | |
| data = json.load(sys.stdin) | |
| print(data['matrix_compliance']['score']) | |
| ") | |
| MISSING_COUNT=$(cat matrix_verification_report.json | python3 -c " | |
| import sys, json | |
| data = json.load(sys.stdin) | |
| print(len(data['missing_implementations'])) | |
| ") | |
| echo "Compliance score: $COMPLIANCE_SCORE" | |
| echo "Missing implementations: $MISSING_COUNT" | |
| # Set thresholds | |
| MIN_COMPLIANCE=0.7 | |
| MAX_MISSING=2 | |
| if (( $(echo "$COMPLIANCE_SCORE < $MIN_COMPLIANCE" | bc -l) )); then | |
| echo "::warning::Matrix compliance below threshold ($COMPLIANCE_SCORE < $MIN_COMPLIANCE)" | |
| echo "compliance_status=warning" >> $GITHUB_OUTPUT | |
| elif [ "$MISSING_COUNT" -gt "$MAX_MISSING" ]; then | |
| echo "::warning::Too many missing implementations ($MISSING_COUNT > $MAX_MISSING)" | |
| echo "compliance_status=warning" >> $GITHUB_OUTPUT | |
| else | |
| echo "::notice::Matrix compliance acceptable ($COMPLIANCE_SCORE >= $MIN_COMPLIANCE)" | |
| echo "compliance_status=success" >> $GITHUB_OUTPUT | |
| fi | |
| - name: 📤 Upload Verification Artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: matrix-verification-report-${{ github.run_number }} | |
| path: | | |
| matrix_verification_report.json | |
| MATRIX_COMPLIANCE_REPORT.md | |
| retention-days: 30 | |
| - name: 💬 Create Issue for Non-Compliance (if needed) | |
| if: steps.compliance_check.outputs.compliance_status == 'warning' | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const fs = require('fs'); | |
| const reportContent = fs.readFileSync('MATRIX_COMPLIANCE_REPORT.md', 'utf8'); | |
| const issue = await github.rest.issues.create({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| title: '🚨 File Processing Matrix Compliance Warning', | |
| body: `## File Processing Matrix Compliance Issue Detected | |
| The automated verification agent has detected compliance issues with the file processing matrix implementation. | |
| **Action Required:** Review and address the issues identified in the compliance report. | |
| ## Verification Report | |
| ${reportContent} | |
| --- | |
| **Auto-generated by:** File Processing Matrix Verification Agent | |
| **Run ID:** ${{ github.run_number }} | |
| **Timestamp:** ${new Date().toISOString()}`, | |
| labels: ['automation', 'compliance', 'matrix-verification'] | |
| }); | |
| console.log(`Created issue #${issue.data.number}`); | |
| - name: ✅ Verification Complete | |
| run: | | |
| echo "🎉 File Processing Matrix Verification completed successfully!" | |
| echo "📊 Compliance status: ${{ steps.compliance_check.outputs.compliance_status || 'success' }}" | |
| echo "📋 Report available in artifacts and repository" | |
| echo "🔍 Monitoring: Automated verification runs every 6 hours" | |
| if [ "${{ steps.compliance_check.outputs.compliance_status }}" = "warning" ]; then | |
| echo "⚠️ Compliance issues detected - check the generated issue for details" | |
| else | |
| echo "✅ All systems compliant with file processing matrix specifications" | |
| fi |