AI Detector #24
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: AI Detector | |
| # ══════════════════════════════════════════════════════════════════════════════ | |
| # JOB 1 — AI-Generated PR Detector | |
| # Runs on every PR open / update / edit. | |
| # Scores the PR body across 10 deep heuristic signals. | |
| # Labels: | |
| # • ai-detected (confidence ≥ HIGH — 4+ signals) | |
| # • ai-suspected (confidence MEDIUM — 2-3 signals) | |
| # Removes stale AI labels when a PR is later edited and passes checks. | |
| # Posts a detailed comment explaining every triggered signal. | |
| # ══════════════════════════════════════════════════════════════════════════════ | |
| on: | |
| pull_request_target: | |
| types: [opened, synchronize, reopened, edited, ready_for_review] | |
| schedule: | |
| # Every 6 hours — catches issues closed 3+ days ago promptly | |
| - cron: '0 */6 * * *' | |
| concurrency: | |
| group: ai-detector-pr-${{ github.event.pull_request.number || 'schedule' }} | |
| cancel-in-progress: true | |
| permissions: | |
| issues: write | |
| pull-requests: write | |
| # ────────────────────────────────────────────────────────────────────────────── | |
| jobs: | |
| # ════════════════════════════════════════════════════════════════════════════ | |
| # JOB 1 — AI-Generated PR Detector | |
| # ════════════════════════════════════════════════════════════════════════════ | |
| ai-pr-detector: | |
| name: AI-Generated PR Detector | |
| runs-on: ubuntu-latest | |
| if: github.event_name == 'pull_request_target' | |
| steps: | |
| - name: Score PR for AI-generated content | |
| uses: actions/github-script@v9 | |
| with: | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| script: | | |
| const { owner, repo } = context.repo; | |
| const pr = context.payload.pull_request; | |
| const prNum = pr.number; | |
| const prTitle = (pr.title || ''); | |
| const prBody = (pr.body || ''); | |
| const titleLC = prTitle.toLowerCase(); | |
| const bodyLC = prBody.toLowerCase(); | |
| const author = pr.user?.login || ''; | |
| // ── Trusted users — never flag ──────────────────────────────── | |
| const TRUSTED = ['Ayushh-Sharmaa', 'SandeepVashishtha', 'RhythmPahwa14', | |
| 'github-actions[bot]', 'dependabot[bot]']; | |
| if (TRUSTED.includes(author)) { | |
| console.log(`⏭️ Trusted user @${author} — skipping AI check`); | |
| return; | |
| } | |
| // ── Label catalogue ─────────────────────────────────────────── | |
| const AI_LABELS = ['ai-detected', 'ai-suspected']; | |
| const LABEL_META = { | |
| 'ai-detected': { color: 'b60205', description: 'High confidence: PR body appears AI-generated' }, | |
| 'ai-suspected': { color: 'e99695', description: 'Moderate confidence: PR body shows AI signals' }, | |
| }; | |
| async function ensureLabels() { | |
| for (const [name, { color, description }] of Object.entries(LABEL_META)) { | |
| try { await github.rest.issues.getLabel({ owner, repo, name }); } | |
| catch (e) { | |
| if (e.status === 404) | |
| await github.rest.issues.createLabel({ owner, repo, name, color, description }); | |
| } | |
| } | |
| } | |
| async function getCurrentLabels() { | |
| const { data } = await github.rest.issues.get({ owner, repo, issue_number: prNum }); | |
| return data.labels.map(l => (typeof l === 'string' ? l : l.name)); | |
| } | |
| // ════════════════════════════════════════════════════════════ | |
| // SIGNAL DEFINITIONS (each returns a string reason if fired, | |
| // or null if not) | |
| // ════════════════════════════════════════════════════════════ | |
| const signals = []; | |
| // ── S1: Classic AI boilerplate phrases ─────────────────────── | |
| const AI_PHRASES = [ | |
| "as an ai language model", "i am an ai", "as a large language model", | |
| "i cannot assist with", "i'm sorry, but i", "certainly! here", | |
| "of course! here", "sure! here is", "please note that i", | |
| "it's important to note that", "as requested, here", "feel free to ask", | |
| "this is a placeholder", "lorem ipsum", "todo: add description", | |
| "generated by chatgpt", "generated by claude", "generated by gemini", | |
| "as your ai assistant", "i hope this helps!", "happy coding!", | |
| "let me know if you need anything else", "happy to help!", | |
| "certainly, here is a", "here is an example of", "below is a detailed", | |
| "in conclusion,", "to summarize,", "in summary,", | |
| "note: i am an ai", "this pr aims to", "this pull request aims to", | |
| "the following changes have been made", "the changes include", | |
| "i have implemented", "i have added", "i have updated", | |
| "the implementation includes", "as per the requirements", | |
| "as per the task", "as mentioned in the issue", | |
| "hereby", "thus ensuring", "thereby", "furthermore,", | |
| "additionally,", "in addition to the above", | |
| "the aforementioned", "the above changes", | |
| ]; | |
| for (const phrase of AI_PHRASES) { | |
| if (bodyLC.includes(phrase)) { | |
| signals.push(`🤖 **AI boilerplate phrase** — \`"${phrase}"\` found in PR body`); | |
| break; | |
| } | |
| } | |
| // ── S2: Passive-voice overuse ──────────────────────────────── | |
| const passiveMatches = (bodyLC.match(/\b(has been|have been|was|were|is being|are being|will be|shall be)\s+\w+ed\b/g) || []); | |
| if (passiveMatches.length >= 4) { | |
| signals.push(`📝 **Excessive passive voice** — ${passiveMatches.length} passive constructions detected (e.g. "${passiveMatches[0]}")`); | |
| } | |
| // ── S3: Unusually perfect bullet-point structure ────────────── | |
| // AI tends to produce bulleted lists where every item is roughly | |
| // the same length and starts with a capital letter. | |
| const bullets = prBody.split('\n') | |
| .filter(l => /^[\-\*\+]\s+[A-Z]/.test(l.trim())) | |
| .map(l => l.trim().length); | |
| if (bullets.length >= 5) { | |
| const avg = bullets.reduce((a, b) => a + b, 0) / bullets.length; | |
| const variance = bullets.reduce((a, b) => a + Math.abs(b - avg), 0) / bullets.length; | |
| if (variance < 15 && avg > 40) { | |
| signals.push(`📋 **Uniform bullet-point structure** — ${bullets.length} bullets with suspiciously consistent length (avg ${Math.round(avg)} chars, variance ${Math.round(variance)})`); | |
| } | |
| } | |
| // ── S4: Academic / AI vocabulary density ───────────────────── | |
| const ACADEMIC_WORDS = [ | |
| 'utilize', 'utilizes', 'utilized', 'utilization', | |
| 'implement', 'implements', 'implementation', 'implemented', | |
| 'facilitate', 'facilitates', 'facilitated', 'functionality', | |
| 'ensure', 'ensures', 'ensuring', 'ensured', | |
| 'leverage', 'leverages', 'leveraged', 'leveraging', | |
| 'streamline', 'streamlines', 'streamlined', | |
| 'robust', 'seamlessly', 'efficient', 'efficiently', | |
| 'comprehensive', 'straightforward', 'optimal', 'optimally', | |
| 'refactored', 'modular', 'scalable', 'maintainable', | |
| 'intuitive', 'enhance', 'enhances', 'enhancement', | |
| 'incorporate', 'incorporates', 'incorporated', | |
| 'address', 'addresses', 'addressed', 'resolve', 'resolves', | |
| ]; | |
| const bodyWords = bodyLC.split(/\s+/); | |
| const academicCount = bodyWords.filter(w => ACADEMIC_WORDS.includes(w.replace(/[.,!?;:]/g, ''))).length; | |
| const academicDensity = bodyWords.length > 0 ? academicCount / bodyWords.length : 0; | |
| if (academicDensity > 0.07 && academicCount >= 6) { | |
| signals.push(`🎓 **High academic/AI vocabulary density** — ${academicCount} AI-style words in ${bodyWords.length} total words (${(academicDensity * 100).toFixed(1)}% density)`); | |
| } | |
| // ── S5: Suspiciously comprehensive for a simple diff ───────── | |
| // If the body is very long (>800 chars) but the PR has very few | |
| // changed files, this is a red flag — humans rarely write essays | |
| // for small PRs. | |
| const bodyLen = prBody.replace(/\s+/g, '').length; | |
| try { | |
| const files = await github.paginate(github.rest.pulls.listFiles, { | |
| owner, repo, pull_number: prNum, per_page: 100, | |
| }); | |
| const totalLines = files.reduce((s, f) => s + f.additions + f.deletions, 0); | |
| if (bodyLen > 800 && files.length <= 3 && totalLines <= 50) { | |
| signals.push(`📏 **Over-explained small diff** — ${bodyLen} body chars for only ${files.length} file(s) and ${totalLines} lines changed. Humans rarely write essays for tiny PRs.`); | |
| } | |
| } catch (_) { /* ignore file-fetch errors */ } | |
| // ── S6: Meta-commentary (talking about the PR, not the code) ── | |
| const META_PATTERNS = [ | |
| /this pr (introduces|adds|updates|fixes|addresses|implements|refactors|improves)/i, | |
| /this pull request (introduces|adds|updates|fixes|addresses|implements)/i, | |
| /the (purpose|goal|aim|objective) of this (pr|pull request)/i, | |
| /in this (pr|pull request|commit|change)/i, | |
| /this change (introduces|adds|ensures|provides|enables)/i, | |
| /the changes? (in this pr|made in this|introduced in)/i, | |
| ]; | |
| const metaHits = META_PATTERNS.filter(p => p.test(prBody)).length; | |
| if (metaHits >= 2) { | |
| signals.push(`🗣️ **Meta-commentary about the PR** — ${metaHits} sentences describe the PR rather than the problem it solves (classic AI self-narration pattern)`); | |
| } | |
| // ── S7: Suspiciously structured "sections" on every header ──── | |
| // AI almost always fills in every template section with something. | |
| // Check if ALL markdown headers in the body have non-trivial content. | |
| const headerMatches = [...prBody.matchAll(/^#{1,3}\s+(.+)$/gm)]; | |
| if (headerMatches.length >= 4) { | |
| const allFilled = headerMatches.every(m => { | |
| const idx = prBody.indexOf(m[0]); | |
| const nextHeader = prBody.slice(idx + m[0].length).search(/^#{1,3}\s+/m); | |
| const section = nextHeader === -1 | |
| ? prBody.slice(idx + m[0].length) | |
| : prBody.slice(idx + m[0].length, idx + m[0].length + nextHeader); | |
| return section.replace(/\s+/g, '').length > 30; | |
| }); | |
| if (allFilled) { | |
| signals.push(`📑 **All ${headerMatches.length} template sections filled** — AI characteristically fills every section; humans often leave irrelevant ones blank`); | |
| } | |
| } | |
| // ── S8: Repeated sentence starters ─────────────────────────── | |
| const sentences = prBody.split(/[.!?]\s+/).map(s => s.trim().toLowerCase()); | |
| const starters = sentences.map(s => s.split(/\s+/).slice(0, 2).join(' ')).filter(s => s.length > 3); | |
| const starterFreq = {}; | |
| for (const s of starters) starterFreq[s] = (starterFreq[s] || 0) + 1; | |
| const repeatedStarters = Object.entries(starterFreq).filter(([, c]) => c >= 3); | |
| if (repeatedStarters.length > 0) { | |
| const ex = repeatedStarters[0]; | |
| signals.push(`🔁 **Repeated sentence starters** — "${ex[0]}" begins ${ex[1]} sentences (AI often reuses sentence templates)`); | |
| } | |
| // ── S9: Zero personal pronouns in a long body ───────────────── | |
| // Real contributors say "I", "we", "my". AI avoids first person. | |
| if (bodyLen > 300) { | |
| const personalPronouns = (bodyLC.match(/\b(i |i'|my |we |our |i've|i'll|i'd)\b/g) || []).length; | |
| if (personalPronouns === 0) { | |
| signals.push(`👤 **No personal pronouns** — a body of ${bodyLen} chars with zero "I/my/we/our" is unusual for a human contributor`); | |
| } | |
| } | |
| // ── S10: Changelog-style "Added / Fixed / Changed" dump ─────── | |
| const changelogLines = (prBody.match(/^[-*]\s+(Added|Fixed|Changed|Updated|Removed|Improved|Refactored)\s+/gim) || []); | |
| if (changelogLines.length >= 4) { | |
| signals.push(`📜 **Auto-generated changelog format** — ${changelogLines.length} lines follow "Added/Fixed/Changed X" pattern (typical of AI output)`); | |
| } | |
| // ════════════════════════════════════════════════════════════ | |
| // DECISION | |
| // ════════════════════════════════════════════════════════════ | |
| const score = signals.length; | |
| console.log(`\n🤖 AI Detector — PR #${prNum} by @${author}`); | |
| console.log(` Score: ${score}/10 signals fired`); | |
| signals.forEach(s => console.log(' 🔴', s.replace(/\*\*/g, ''))); | |
| await ensureLabels(); | |
| const current = await getCurrentLabels(); | |
| // Determine verdict | |
| let targetLabel = null; | |
| if (score >= 4) targetLabel = 'ai-detected'; | |
| else if (score >= 2) targetLabel = 'ai-suspected'; | |
| // Stale label cleanup (idempotent) | |
| const stale = current.filter(l => AI_LABELS.includes(l) && l !== targetLabel); | |
| for (const name of stale) { | |
| await github.rest.issues.removeLabel({ owner, repo, issue_number: prNum, name }); | |
| console.log(`🗑 Removed stale AI label: ${name}`); | |
| } | |
| if (targetLabel && !current.includes(targetLabel)) { | |
| await github.rest.issues.addLabels({ owner, repo, issue_number: prNum, labels: [targetLabel] }); | |
| console.log(`🏷️ Applied: ${targetLabel}`); | |
| } | |
| // ── Post / update bot comment ───────────────────────────────── | |
| const BOT_MARKER = '<!-- ai-detector-bot -->'; | |
| if (score >= 2) { | |
| const verdict = score >= 4 | |
| ? '🔴 **HIGH confidence — AI-generated content detected**' | |
| : '🟡 **MODERATE confidence — AI signals present**'; | |
| const commentBody = [ | |
| BOT_MARKER, | |
| `## 🤖 AI Detector Report — PR #${prNum}`, | |
| ``, | |
| `${verdict}`, | |
| ``, | |
| `**Score: ${score} / 10 signals fired**`, | |
| ``, | |
| `| # | Signal |`, | |
| `|---|--------|`, | |
| signals.map((s, i) => `| ${i+1} | ${s} |`).join('\n'), | |
| ``, | |
| `---`, | |
| `### ❓ Why does this matter?`, | |
| ``, | |
| `AI-generated PR descriptions often lack genuine understanding of the codebase. GSSoC contributions are evaluated on **authentic learning and effort** — not on how polished a description looks.`, | |
| ``, | |
| `### ✅ What to do`, | |
| ``, | |
| score >= 4 | |
| ? `The \`ai-detected\` label has been applied. **Please rewrite your PR description in your own words.** Explain what problem you solved, what you changed, and why — as if describing it to a friend. Remove AI-generated text and push an update.` | |
| : `The \`ai-suspected\` label has been applied. If your description is human-written, no action is needed. If you used AI to draft it, please revise it to reflect your genuine understanding.`, | |
| ``, | |
| `📖 Read the [Contributing Guide](https://github.com/Ayushh-Sharmaa/Eventra/blob/main/CONTRIBUTING.md) for expectations on PR quality.`, | |
| ``, | |
| `---`, | |
| `_GSSoC'26 AI Detector · Mentor: [@Ayushh-Sharmaa](https://github.com/Ayushh-Sharmaa)_`, | |
| ].join('\n'); | |
| const { data: comments } = await github.rest.issues.listComments({ owner, repo, issue_number: prNum }); | |
| const existing = comments.find(c => c.body.includes(BOT_MARKER)); | |
| if (existing) { | |
| await github.rest.issues.updateComment({ owner, repo, comment_id: existing.id, body: commentBody }); | |
| console.log('📝 Updated existing AI detector comment'); | |
| } else { | |
| await github.rest.issues.createComment({ owner, repo, issue_number: prNum, body: commentBody }); | |
| console.log('📝 Posted AI detector comment'); | |
| } | |
| } else { | |
| // Score < 2 — clean. Remove any stale bot comment. | |
| const { data: comments } = await github.rest.issues.listComments({ owner, repo, issue_number: prNum }); | |
| const existing = comments.find(c => c.body.includes(BOT_MARKER)); | |
| if (existing) { | |
| await github.rest.issues.deleteComment({ owner, repo, comment_id: existing.id }); | |
| console.log('🗑 Removed stale AI detector comment — PR now looks clean'); | |
| } | |
| console.log(`✅ PR #${prNum} passed AI detection (score ${score}/10) — no action needed`); | |
| } | |
| # ══════════════════════════════════════════════════════════════════════ | |
| mentor-label-closed-items: | |
| name: Mentor Label — Closed PRs & Issues (3-day delay) | |
| runs-on: ubuntu-latest | |
| if: github.event_name == 'schedule' | |
| steps: | |
| - name: Apply mentor label to PRs and issues closed 3+ days ago | |
| uses: actions/github-script@v9 | |
| with: | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| script: | | |
| const { owner, repo } = context.repo; | |
| const MENTOR_USERNAME = 'Ayushh-Sharmaa'; | |
| const MENTOR_LABEL = `mentor:${MENTOR_USERNAME}`; | |
| const LABEL_COLOR = '6f42c1'; | |
| const THREE_DAYS_MS = 3 * 24 * 60 * 60 * 1000; | |
| const now = Date.now(); | |
| // ── Ensure the mentor label exists ──────────────────────────── | |
| try { | |
| await github.rest.issues.getLabel({ owner, repo, name: MENTOR_LABEL }); | |
| } catch (e) { | |
| if (e.status === 404) { | |
| await github.rest.issues.createLabel({ | |
| owner, repo, | |
| name: MENTOR_LABEL, | |
| color: LABEL_COLOR, | |
| description: `GSSoC: Mentor — @${MENTOR_USERNAME}`, | |
| }); | |
| console.log(`✅ Created label: ${MENTOR_LABEL}`); | |
| } | |
| } | |
| // ── Helper: process a batch of closed items ─────────────────── | |
| async function processItems(items, typeName) { | |
| let applied = 0; | |
| let skipped = 0; | |
| for (const item of items) { | |
| const closedAt = new Date(item.closed_at).getTime(); | |
| if ((now - closedAt) < THREE_DAYS_MS) { skipped++; continue; } | |
| const currentLabels = item.labels.map(l => | |
| typeof l === 'string' ? l : l.name | |
| ); | |
| // Skip if any mentor:* label already present | |
| if (currentLabels.some(n => n.startsWith('mentor:'))) { | |
| skipped++; | |
| continue; | |
| } | |
| // Apply mentor label — no comment posted | |
| await github.rest.issues.addLabels({ | |
| owner, repo, | |
| issue_number: item.number, | |
| labels: [MENTOR_LABEL], | |
| }); | |
| const days = Math.floor((now - closedAt) / (24 * 60 * 60 * 1000)); | |
| console.log(`🏷️ Applied ${MENTOR_LABEL} to ${typeName} #${item.number} (closed ${days}d ago)`); | |
| applied++; | |
| } | |
| return { applied, skipped }; | |
| } | |
| // ── 1. Closed ISSUES (items without pull_request field) ─────── | |
| const allClosed = await github.paginate(github.rest.issues.listForRepo, { | |
| owner, repo, state: 'closed', per_page: 100, | |
| }); | |
| const closedIssues = allClosed.filter(i => !i.pull_request); | |
| const closedPRItems = allClosed.filter(i => !!i.pull_request); | |
| console.log(`\n📋 Closed items — Issues: ${closedIssues.length} | PRs (via issues API): ${closedPRItems.length}`); | |
| const issueResult = await processItems(closedIssues, 'issue'); | |
| // ── 2. Closed PULL REQUESTS (via pulls API for accuracy) ────── | |
| const allPRs = await github.paginate(github.rest.pulls.list, { | |
| owner, repo, state: 'closed', per_page: 100, | |
| }); | |
| console.log(` PRs (via pulls API): ${allPRs.length}`); | |
| // Map PRs to issue-shaped objects so processItems can handle them | |
| const prItems = allPRs.map(pr => ({ | |
| number: pr.number, | |
| closed_at: pr.closed_at || pr.merged_at, | |
| labels: pr.labels || [], | |
| })).filter(pr => pr.closed_at); // only PRs that are actually closed | |
| const prResult = await processItems(prItems, 'PR'); | |
| console.log(`\n✅ Issues — Applied: ${issueResult.applied} | Already labelled / too recent: ${issueResult.skipped}`); | |
| console.log(`✅ PRs — Applied: ${prResult.applied} | Already labelled / too recent: ${prResult.skipped}`); |