From 8f230c68fee678cc2a071fa54ad586c13c16b3ab Mon Sep 17 00:00:00 2001 From: Travis Prescott Date: Wed, 6 May 2026 11:03:34 -0700 Subject: [PATCH 1/3] Add `--include-implict` to the command. --- .../.github/skills/audit-filters/SKILL.md | 12 +- .../.github/skills/report-feedback/SKILL.md | 38 ++++- .../python-packages/apiview-copilot/cli.py | 15 +- .../apiview-copilot/src/_apiview.py | 143 +++++++++++++++++- 4 files changed, 195 insertions(+), 13 deletions(-) diff --git a/packages/python-packages/apiview-copilot/.github/skills/audit-filters/SKILL.md b/packages/python-packages/apiview-copilot/.github/skills/audit-filters/SKILL.md index ad60e737857..ac961c4049b 100644 --- a/packages/python-packages/apiview-copilot/.github/skills/audit-filters/SKILL.md +++ b/packages/python-packages/apiview-copilot/.github/skills/audit-filters/SKILL.md @@ -106,8 +106,16 @@ Based on the analysis, propose specific new lines to add to `metadata/{lang}/fil 1. Follow the existing format: ` N. DO NOT ` 2. Be numbered sequentially after the last existing rule 3. **Not duplicate an existing rule** — Before proposing a rule, compare it against every existing rule in the current `filter.yaml`. If an existing rule already covers the same behavior (even with different wording), do NOT propose it again. Explain in the analysis that the theme was already covered and cite the existing rule number. -4. Be supported by at least 2 feedback items or 1 memory with `is_exception: true` -5. Be phrased as a clear, actionable instruction the LLM can follow +4. Be phrased as a clear, actionable instruction the LLM can follow + +### Signal Strength + +When presenting recommendations, clearly label each with its signal strength: + +- **Strong signal**: 2+ explicit feedback items (downvotes with reasons) or 1 memory with `is_exception: true` +- **Low signal**: Only 1 explicit feedback item, or only implicit bad comments (no explicit downvote/reason) + +Do NOT automatically exclude low-signal items. Present ALL actionable patterns to the user with their signal strength clearly marked, and let the user (or reviewer) decide whether to include them in the PR. Present the recommendations in a numbered list, each with: - The proposed rule text diff --git a/packages/python-packages/apiview-copilot/.github/skills/report-feedback/SKILL.md b/packages/python-packages/apiview-copilot/.github/skills/report-feedback/SKILL.md index b3b525b869b..62392ae6742 100644 --- a/packages/python-packages/apiview-copilot/.github/skills/report-feedback/SKILL.md +++ b/packages/python-packages/apiview-copilot/.github/skills/report-feedback/SKILL.md @@ -23,8 +23,26 @@ Unless the user says otherwise, always apply these defaults: - **Environment**: `production` - **Language**: All languages (do not pass `--language` unless user specifies one) - **Exclude**: Do not pass `--exclude` unless user asks to filter out certain feedback types +- **Include implicit**: Always pass `--include-implicit` by default. Only omit it if the user explicitly asks to exclude implicit bad comments. - **Format**: JSON (do not pass `--format`) +## Implicit Bad Comments + +The `--include-implicit` flag also returns **implicit bad** comments: AI comments on approved revisions that were never upvoted, downvoted, or resolved. The inference is that the reviewer ignored them and approved anyway, suggesting they were unhelpful. + +Implicit bad is included **by default**. It has a weaker signal than explicit feedback because there is no reason or confirmation — just silence. Only omit `--include-implicit` if the user explicitly asks to exclude them (e.g., "only explicit feedback", "exclude implicit bad"). + +The output will contain items with `"FeedbackTypes": ["implicit_bad"]`. + +### Summarizing Implicit Bad + +When presenting results, **break out implicit bad themes separately** from explicit feedback: + +1. **Explicit feedback** — Summarize count, breakdown by reason, and themes for items that have explicit feedback types (e.g., `bad`, `delete`). +2. **Implicit bad** — Summarize separately: count, common comment topics/patterns, and any notable themes. Note that these lack a reason — group them by the comment content or guideline referenced instead. + +This separation helps the user understand the strength of signal behind each theme. + ## Date Resolution The user will typically specify a calendar month by name (e.g. "March", "January 2025"). Resolve to the full month date range: @@ -45,7 +63,7 @@ Show the resolved command and run it immediately in a **foreground terminal** wi **Full terminal command** (cleanup + run): ```powershell -New-Item -ItemType Directory -Path output -Force | Out-Null; if (Test-Path output/feedback_output.json) { Remove-Item output/feedback_output.json }; python cli.py report feedback -s -e | Out-File -Encoding UTF8 output/feedback_output.json +New-Item -ItemType Directory -Path output -Force | Out-Null; if (Test-Path output/feedback_output.json) { Remove-Item output/feedback_output.json }; python cli.py report feedback -s -e --include-implicit | Out-File -Encoding UTF8 output/feedback_output.json ``` After the command completes, **read the output file** with `read_file` to get the JSON results. Summarize the findings for the user (total count, breakdown by feedback reason, common themes, etc.). @@ -57,20 +75,23 @@ For follow-up questions about the same data (filtering, counting, searching), ** ### Examples ```powershell -# All feedback for March 2025 -python cli.py report feedback -s 2025-03-01 -e 2025-03-31 +# All feedback for March 2025 (implicit bad included by default) +python cli.py report feedback -s 2025-03-01 -e 2025-03-31 --include-implicit # Python feedback only -python cli.py report feedback -s 2025-03-01 -e 2025-03-31 -l python +python cli.py report feedback -s 2025-03-01 -e 2025-03-31 -l python --include-implicit + +# Exclude implicit bad (only explicit feedback) +python cli.py report feedback -s 2025-03-01 -e 2025-03-31 # Exclude good feedback (show only bad and deleted) -python cli.py report feedback -s 2025-03-01 -e 2025-03-31 --exclude good +python cli.py report feedback -s 2025-03-01 -e 2025-03-31 --include-implicit --exclude good # YAML output -python cli.py report feedback -s 2025-03-01 -e 2025-03-31 --format yaml +python cli.py report feedback -s 2025-03-01 -e 2025-03-31 --include-implicit --format yaml # Staging environment -python cli.py report feedback -s 2025-03-01 -e 2025-03-31 --environment staging +python cli.py report feedback -s 2025-03-01 -e 2025-03-31 --include-implicit --environment staging ``` ## Available Flags @@ -81,7 +102,8 @@ python cli.py report feedback -s 2025-03-01 -e 2025-03-31 --environment staging | `--end-date` / `-e` | string | required | End date (`YYYY-MM-DD`) | | `--language` / `-l` | string | all | Language to filter by (e.g., `python`, `Go`, `C#`) | | `--environment` | string | `production` | `production` or `staging` | -| `--exclude` | list | none | Feedback types to exclude: `good`, `bad`, `delete` | +| `--exclude` | list | none | Feedback types to exclude: `good`, `bad`, `delete`, `implicit_bad` | +| `--include-implicit` | flag | off | Include implicit bad comments (unresolved, unvoted on approved revisions) | | `--format` / `-f` | string | `json` | Output format: `json` or `yaml` | ## Gotchas diff --git a/packages/python-packages/apiview-copilot/cli.py b/packages/python-packages/apiview-copilot/cli.py index ceb487ff812..dfc4643c276 100644 --- a/packages/python-packages/apiview-copilot/cli.py +++ b/packages/python-packages/apiview-copilot/cli.py @@ -2543,10 +2543,13 @@ def get_feedback( exclude: Optional[list[str]] = None, environment: str = "production", output_format: str = "json", + include_implicit: bool = False, ): """ Retrieve AI comment feedback from APIView between start_date and end_date. If --language is omitted, returns feedback for all languages. + Use --include-implicit to also return implicit bad comments (unresolved, unvoted + AI comments on approved revisions). """ results = _get_ai_comment_feedback( language=language, @@ -2554,6 +2557,7 @@ def get_feedback( end_date=end_date, exclude=exclude, environment=environment, + include_implicit=include_implicit, ) if output_format == "yaml": print(yaml.dump(results, default_flow_style=False, allow_unicode=True, sort_keys=False)) @@ -3248,9 +3252,9 @@ def load_arguments(self, command): "exclude", type=str, nargs="*", - help="Feedback types to exclude. Can be 'good', 'bad', or 'delete'.", + help="Feedback types to exclude. Can be 'good', 'bad', 'delete', or 'implicit_bad'.", options_list=["--exclude"], - choices=["good", "bad", "delete"], + choices=["good", "bad", "delete", "implicit_bad"], ) ac.argument( "output_format", @@ -3260,6 +3264,13 @@ def load_arguments(self, command): default="json", choices=["json", "yaml"], ) + ac.argument( + "include_implicit", + action="store_true", + help="Include implicit bad comments (unresolved, unvoted AI comments on approved revisions).", + options_list=["--include-implicit"], + default=False, + ) with ArgumentsContext(self, "report memory") as ac: ac.argument( "language", diff --git a/packages/python-packages/apiview-copilot/src/_apiview.py b/packages/python-packages/apiview-copilot/src/_apiview.py index 1fe7799dc24..ede1481fec1 100644 --- a/packages/python-packages/apiview-copilot/src/_apiview.py +++ b/packages/python-packages/apiview-copilot/src/_apiview.py @@ -609,6 +609,7 @@ def get_ai_comment_feedback( end_date: str, exclude: Optional[list[str]] = None, environment: str = "production", + include_implicit: bool = False, ) -> list[dict]: """ Retrieves AI-generated comments that received feedback within the specified date range. @@ -617,6 +618,11 @@ def get_ai_comment_feedback( - For detailed feedback: checks Feedback[].SubmittedOn - For deletions: checks ChangeHistory[].ChangedOn where ChangeAction='Deleted' + When include_implicit is True, also returns "implicit bad" comments: AI comments created + in the date range that are on approved revisions but have no votes, no resolution, and + aren't deleted. These are inferred as unhelpful because the reviewer approved without + interacting with the comment. + Note: Upvotes/Downvotes lists don't have timestamps, so comments with only upvotes/downvotes (and no Feedback entries or deletion events in the date range) will not be returned. @@ -625,8 +631,10 @@ def get_ai_comment_feedback( language: Language to filter by (e.g., 'python', 'java'). If None, returns all languages. start_date: Start date in YYYY-MM-DD format (filters by feedback submission time) end_date: End date in YYYY-MM-DD format (filters by feedback submission time) - exclude: List of feedback types to exclude. Can include 'good', 'bad', 'delete'. + exclude: List of feedback types to exclude. Can include 'good', 'bad', 'delete', 'implicit_bad'. environment: The APIView environment ('production' or 'staging') + include_implicit: If True, also include implicit bad comments (unresolved, unvoted AI comments + on approved revisions created in the date range). Returns: List of dicts containing comment info and feedback, preserving database field names @@ -735,6 +743,139 @@ def get_ai_comment_feedback( comment["FeedbackTypes"] = feedback_types result.append(comment) + # If include_implicit is requested, also fetch implicit bad comments + if include_implicit and "implicit_bad" not in exclude: + implicit_comments = _get_implicit_bad_comments( + start_date=start_date, + end_date=end_date, + language=language, + environment=environment, + review_lang_map=review_lang_map, + ) + result.extend(implicit_comments) + + return result + + +def _get_implicit_bad_comments( + start_date: str, + end_date: str, + language: Optional[str], + environment: str, + review_lang_map: dict, +) -> list[dict]: + """ + Retrieves implicit bad comments: AI-generated comments created in the date range + that are on approved revisions but have no votes, no resolution, and aren't deleted. + + These are inferred as unhelpful because the reviewer approved the revision without + interacting with the comment. + """ + start_iso = to_iso8601(start_date) + end_iso = to_iso8601(end_date, end_of_day=True) + + # Query for AI comments created in the date range that have no interaction + comments_client = get_apiview_cosmos_client(container_name="Comments", environment=environment) + query = """ + SELECT c.id, c.ReviewId, c.APIRevisionId, c.ElementId, c.ThreadId, + c.CommentText, c.CorrelationId, c.ChangeHistory, c.IsResolved, + c.Upvotes, c.Downvotes, c.TaggedUsers, c.CommentType, c.Severity, + c.CommentSource, c.ResolutionLocked, c.CreatedBy, c.CreatedOn, + c.IsDeleted, c.IsGeneric, c.GuidelineIds, c.MemoryIds, + c.ConfidenceScore, c.Feedback + FROM c + WHERE c.CommentSource = 'AIGenerated' + AND c.CreatedOn >= @start_date AND c.CreatedOn <= @end_date + AND (NOT IS_DEFINED(c.IsDeleted) OR c.IsDeleted = false) + AND (NOT IS_DEFINED(c.IsResolved) OR c.IsResolved = false) + AND (NOT IS_DEFINED(c.Upvotes) OR ARRAY_LENGTH(c.Upvotes) = 0) + AND (NOT IS_DEFINED(c.Downvotes) OR ARRAY_LENGTH(c.Downvotes) = 0) + """ + + comments = list( + comments_client.query_items( + query=query, + parameters=[ + {"name": "@start_date", "value": start_iso}, + {"name": "@end_date", "value": end_iso}, + ], + enable_cross_partition_query=True, + ) + ) + + if not comments: + return [] + + # Collect revision IDs to check approval status + revision_ids = set(c.get("APIRevisionId") for c in comments if c.get("APIRevisionId")) + review_ids = set(c.get("ReviewId") for c in comments if c.get("ReviewId")) + + # Fetch revision metadata to determine approval status + revisions_container = get_apiview_cosmos_client(container_name="APIRevisions", environment=environment) + approved_revision_ids = set() + + if revision_ids: + rev_params = [] + rev_clauses = [] + for i, rev_id in enumerate(revision_ids): + param_name = f"@rev_{i}" + rev_clauses.append(f"c.id = {param_name}") + rev_params.append({"name": param_name, "value": rev_id}) + + rev_query = f"SELECT c.id, c.ChangeHistory FROM c WHERE ({' OR '.join(rev_clauses)})" + rev_results = list( + revisions_container.query_items( + query=rev_query, parameters=rev_params, enable_cross_partition_query=True + ) + ) + + for rev in rev_results: + change_history = rev.get("ChangeHistory", []) + if change_history and isinstance(change_history, list): + for change in change_history: + if change.get("ChangeAction") == "Approved": + approved_revision_ids.add(rev["id"]) + break + + if not approved_revision_ids: + return [] + + # Fetch language info for any new review IDs not already in review_lang_map + new_review_ids = review_ids - set(review_lang_map.keys()) + if new_review_ids: + reviews_container = get_apiview_cosmos_client(container_name="Reviews", environment=environment) + params = [] + clauses = [] + for i, rid in enumerate(new_review_ids): + param_name = f"@id_{i}" + clauses.append(f"c.id = {param_name}") + params.append({"name": param_name, "value": rid}) + + review_query = f"SELECT c.id, c.Language FROM c WHERE ({' OR '.join(clauses)})" + review_results = list( + reviews_container.query_items(query=review_query, parameters=params, enable_cross_partition_query=True) + ) + for r in review_results: + review_lang_map[r["id"]] = get_language_pretty_name(r.get("Language", "")) + + target_language = get_language_pretty_name(language).lower() if language else None + + # Filter to comments on approved revisions, matching language + result = [] + for comment in comments: + rev_id = comment.get("APIRevisionId") + if rev_id not in approved_revision_ids: + continue + + review_id = comment.get("ReviewId", "") + comment_language = review_lang_map.get(review_id, "").lower() + if target_language and comment_language != target_language: + continue + + comment["Language"] = review_lang_map.get(review_id, "") + comment["FeedbackTypes"] = ["implicit_bad"] + result.append(comment) + return result From a72e48ee0c4c56a7cb718bcac069925d0f02d191 Mon Sep 17 00:00:00 2001 From: Travis Prescott Date: Wed, 6 May 2026 15:16:24 -0700 Subject: [PATCH 2/3] Code review feedback. --- .../.github/skills/report-feedback/SKILL.md | 2 +- .../apiview-copilot/docs/metrics.md | 2 +- .../apiview-copilot/src/_apiview.py | 1 + .../apiview-copilot/tests/apiview_test.py | 120 ++++++++++++++++++ 4 files changed, 123 insertions(+), 2 deletions(-) diff --git a/packages/python-packages/apiview-copilot/.github/skills/report-feedback/SKILL.md b/packages/python-packages/apiview-copilot/.github/skills/report-feedback/SKILL.md index 62392ae6742..233a8da2e48 100644 --- a/packages/python-packages/apiview-copilot/.github/skills/report-feedback/SKILL.md +++ b/packages/python-packages/apiview-copilot/.github/skills/report-feedback/SKILL.md @@ -30,7 +30,7 @@ Unless the user says otherwise, always apply these defaults: The `--include-implicit` flag also returns **implicit bad** comments: AI comments on approved revisions that were never upvoted, downvoted, or resolved. The inference is that the reviewer ignored them and approved anyway, suggesting they were unhelpful. -Implicit bad is included **by default**. It has a weaker signal than explicit feedback because there is no reason or confirmation — just silence. Only omit `--include-implicit` if the user explicitly asks to exclude them (e.g., "only explicit feedback", "exclude implicit bad"). +This skill always passes `--include-implicit` (the CLI flag defaults to off, but the skill includes it for completeness). It has a weaker signal than explicit feedback because there is no reason or confirmation — just silence. Only omit `--include-implicit` if the user explicitly asks to exclude them (e.g., "only explicit feedback", "exclude implicit bad"). The output will contain items with `"FeedbackTypes": ["implicit_bad"]`. diff --git a/packages/python-packages/apiview-copilot/docs/metrics.md b/packages/python-packages/apiview-copilot/docs/metrics.md index 68db0a5e625..ae1b15a4383 100644 --- a/packages/python-packages/apiview-copilot/docs/metrics.md +++ b/packages/python-packages/apiview-copilot/docs/metrics.md @@ -39,7 +39,7 @@ Every AI-generated comment is assigned to exactly one mutually exclusive quality | `downvoted` (`bad`) | Has ≥1 downvote (trumps upvotes) | Reviewer explicitly disagreed | | `upvoted` (`good`) | Has ≥1 upvote and no downvotes | Reviewer explicitly agreed | | `implicit_good` | `IsResolved = true`, no votes | Comment resolved without explicit feedback — likely acted on | -| `implicit_bad` | In an **approved** revision, not resolved, no votes | Comment was ignored after approval — likely not useful | +| `implicit_bad` | In an **approved** revision, not resolved, no votes, no feedback | Comment was ignored after approval — likely not useful | | `neutral` | In an **unapproved** revision, not resolved, no votes | No signal yet (review still in progress) | The sum of all six buckets equals `total_ai_comment_count`. diff --git a/packages/python-packages/apiview-copilot/src/_apiview.py b/packages/python-packages/apiview-copilot/src/_apiview.py index ede1481fec1..c7cc2cc4ae4 100644 --- a/packages/python-packages/apiview-copilot/src/_apiview.py +++ b/packages/python-packages/apiview-copilot/src/_apiview.py @@ -790,6 +790,7 @@ def _get_implicit_bad_comments( AND (NOT IS_DEFINED(c.IsResolved) OR c.IsResolved = false) AND (NOT IS_DEFINED(c.Upvotes) OR ARRAY_LENGTH(c.Upvotes) = 0) AND (NOT IS_DEFINED(c.Downvotes) OR ARRAY_LENGTH(c.Downvotes) = 0) + AND (NOT IS_DEFINED(c.Feedback) OR ARRAY_LENGTH(c.Feedback) = 0) """ comments = list( diff --git a/packages/python-packages/apiview-copilot/tests/apiview_test.py b/packages/python-packages/apiview-copilot/tests/apiview_test.py index 6031e2dc337..6a1f1875b3f 100644 --- a/packages/python-packages/apiview-copilot/tests/apiview_test.py +++ b/packages/python-packages/apiview-copilot/tests/apiview_test.py @@ -672,6 +672,126 @@ def test_language_empty_for_unknown_review(self, mock_comments_data): assert all(c["Language"] == "" for c in results) + def test_include_implicit_returns_implicit_bad(self): + """Test that include_implicit=True returns implicit bad comments from approved revisions.""" + implicit_comments = [ + { + "id": "implicit-1", + "ReviewId": "review-1", + "APIRevisionId": "rev-1", + "CommentText": "No interaction comment", + "CommentSource": "AIGenerated", + "CreatedOn": "2026-01-10T12:00:00Z", + "Upvotes": [], + "Downvotes": [], + "IsDeleted": False, + "IsResolved": False, + "Feedback": [], + }, + ] + revisions_data = [ + {"id": "rev-1", "ChangeHistory": [{"ChangeAction": "Approved"}]}, + ] + reviews_data = [{"id": "review-1", "Language": "Python"}] + + with patch("src._apiview.get_apiview_cosmos_client") as mock_client: + comments_container = MockContainerClient(implicit_comments) + revisions_container = MockContainerClient(revisions_data) + reviews_container = MockContainerClient(reviews_data) + + call_count = [0] + + def get_container(container_name, environment, db_name=None): + if container_name == "APIRevisions": + return revisions_container + if container_name == "Reviews": + return reviews_container + # First Comments call is for explicit feedback (returns nothing), + # second is for implicit bad comments + call_count[0] += 1 + if call_count[0] == 1: + return MockContainerClient([]) + return comments_container + + mock_client.side_effect = get_container + results = get_ai_comment_feedback(None, "2026-01-01", "2026-01-31", include_implicit=True) + + implicit_results = [c for c in results if "implicit_bad" in c.get("FeedbackTypes", [])] + assert len(implicit_results) == 1 + assert implicit_results[0]["id"] == "implicit-1" + + def test_include_implicit_excluded_when_in_exclude_list(self): + """Test that implicit_bad in exclude list suppresses implicit comments.""" + with patch("src._apiview.get_apiview_cosmos_client") as mock_client: + # Even with include_implicit=True, exclude=['implicit_bad'] should skip them + self._setup_mocks(mock_client, [], []) + results = get_ai_comment_feedback( + None, "2026-01-01", "2026-01-31", include_implicit=True, exclude=["implicit_bad"] + ) + assert results == [] + + def test_include_implicit_language_filter(self): + """Test that language filtering applies to implicit bad comments.""" + implicit_comments = [ + { + "id": "implicit-py", + "ReviewId": "review-py", + "APIRevisionId": "rev-1", + "CommentText": "Python implicit", + "CommentSource": "AIGenerated", + "CreatedOn": "2026-01-10T12:00:00Z", + "Upvotes": [], + "Downvotes": [], + "IsDeleted": False, + "IsResolved": False, + "Feedback": [], + }, + { + "id": "implicit-java", + "ReviewId": "review-java", + "APIRevisionId": "rev-1", + "CommentText": "Java implicit", + "CommentSource": "AIGenerated", + "CreatedOn": "2026-01-10T12:00:00Z", + "Upvotes": [], + "Downvotes": [], + "IsDeleted": False, + "IsResolved": False, + "Feedback": [], + }, + ] + revisions_data = [ + {"id": "rev-1", "ChangeHistory": [{"ChangeAction": "Approved"}]}, + ] + reviews_data = [ + {"id": "review-py", "Language": "Python"}, + {"id": "review-java", "Language": "Java"}, + ] + + with patch("src._apiview.get_apiview_cosmos_client") as mock_client: + comments_container = MockContainerClient(implicit_comments) + revisions_container = MockContainerClient(revisions_data) + reviews_container = MockContainerClient(reviews_data) + + call_count = [0] + + def get_container(container_name, environment, db_name=None): + if container_name == "APIRevisions": + return revisions_container + if container_name == "Reviews": + return reviews_container + call_count[0] += 1 + if call_count[0] == 1: + return MockContainerClient([]) + return comments_container + + mock_client.side_effect = get_container + results = get_ai_comment_feedback("python", "2026-01-01", "2026-01-31", include_implicit=True) + + assert len(results) == 1 + assert results[0]["id"] == "implicit-py" + assert results[0]["Language"] == "Python" + class TestGetCommentsInDateRange: """Tests for get_comments_in_date_range query construction.""" From bb9ccdd419728f8b9a75337115c5c6b49e0af8ff Mon Sep 17 00:00:00 2001 From: Travis Prescott Date: Thu, 7 May 2026 08:57:40 -0700 Subject: [PATCH 3/3] Code review feedback. --- .../.github/skills/report-feedback/SKILL.md | 6 ++++-- packages/python-packages/apiview-copilot/cli.py | 8 +++++--- packages/python-packages/apiview-copilot/src/_apiview.py | 9 +++++---- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/packages/python-packages/apiview-copilot/.github/skills/report-feedback/SKILL.md b/packages/python-packages/apiview-copilot/.github/skills/report-feedback/SKILL.md index 233a8da2e48..d75c3ab515c 100644 --- a/packages/python-packages/apiview-copilot/.github/skills/report-feedback/SKILL.md +++ b/packages/python-packages/apiview-copilot/.github/skills/report-feedback/SKILL.md @@ -28,7 +28,9 @@ Unless the user says otherwise, always apply these defaults: ## Implicit Bad Comments -The `--include-implicit` flag also returns **implicit bad** comments: AI comments on approved revisions that were never upvoted, downvoted, or resolved. The inference is that the reviewer ignored them and approved anyway, suggesting they were unhelpful. +The `--include-implicit` flag also returns **implicit bad** comments: AI comments on approved revisions that were never upvoted, downvoted, resolved, and have no Feedback entries. The inference is that the reviewer ignored them and approved anyway, suggesting they were unhelpful. + +> **Date semantics differ**: Explicit feedback is filtered by feedback submission time (`Feedback[].SubmittedOn` / `ChangeHistory[].ChangedOn`), but implicit bad is filtered by comment creation time (`CreatedOn`). A comment created in January with no interaction will appear in January's implicit bad results, not March's. This skill always passes `--include-implicit` (the CLI flag defaults to off, but the skill includes it for completeness). It has a weaker signal than explicit feedback because there is no reason or confirmation — just silence. Only omit `--include-implicit` if the user explicitly asks to exclude them (e.g., "only explicit feedback", "exclude implicit bad"). @@ -109,7 +111,7 @@ python cli.py report feedback -s 2025-03-01 -e 2025-03-31 --include-implicit --e ## Gotchas - **Output can be large**: Redirect to file and use `read_file` rather than relying on terminal output. -- **Date range filters by feedback submission time**: Not by when the comment was created. A comment created in January but downvoted in March will appear in March's feedback report. +- **Date range semantics are mixed**: Explicit feedback filters by feedback submission time (a comment created in January but downvoted in March appears in March). Implicit bad filters by comment creation time (a comment created in January with no interaction appears in January). - **Use `python cli.py` not `.\avc`**: The `avc.bat` script may resolve to system Python. - **Do NOT use `2>&1`**: Merges stderr into stdout, corrupting JSON. Only redirect stdout. - **Do NOT use `>`**: Produces UTF-16 in PowerShell 5.1. Use `| Out-File -Encoding UTF8`. diff --git a/packages/python-packages/apiview-copilot/cli.py b/packages/python-packages/apiview-copilot/cli.py index dfc4643c276..5f9e7afc858 100644 --- a/packages/python-packages/apiview-copilot/cli.py +++ b/packages/python-packages/apiview-copilot/cli.py @@ -2548,8 +2548,10 @@ def get_feedback( """ Retrieve AI comment feedback from APIView between start_date and end_date. If --language is omitted, returns feedback for all languages. - Use --include-implicit to also return implicit bad comments (unresolved, unvoted - AI comments on approved revisions). + Use --include-implicit to also return implicit bad comments: AI comments created + in the date range that are on approved revisions with no votes, no Feedback entries, + no resolution, and not deleted. Note that the date range filters by comment creation + time for implicit bad (vs. feedback submission time for explicit feedback). """ results = _get_ai_comment_feedback( language=language, @@ -3267,7 +3269,7 @@ def load_arguments(self, command): ac.argument( "include_implicit", action="store_true", - help="Include implicit bad comments (unresolved, unvoted AI comments on approved revisions).", + help="Include implicit bad comments (AI comments created in date range on approved revisions with no votes, no feedback, and no resolution).", options_list=["--include-implicit"], default=False, ) diff --git a/packages/python-packages/apiview-copilot/src/_apiview.py b/packages/python-packages/apiview-copilot/src/_apiview.py index c7cc2cc4ae4..d9c8741fb78 100644 --- a/packages/python-packages/apiview-copilot/src/_apiview.py +++ b/packages/python-packages/apiview-copilot/src/_apiview.py @@ -619,9 +619,9 @@ def get_ai_comment_feedback( - For deletions: checks ChangeHistory[].ChangedOn where ChangeAction='Deleted' When include_implicit is True, also returns "implicit bad" comments: AI comments created - in the date range that are on approved revisions but have no votes, no resolution, and - aren't deleted. These are inferred as unhelpful because the reviewer approved without - interacting with the comment. + in the date range that are on approved revisions but have no votes, no resolution, no + Feedback entries, and aren't deleted. These are inferred as unhelpful because the reviewer + approved without interacting with the comment. Note: Upvotes/Downvotes lists don't have timestamps, so comments with only upvotes/downvotes (and no Feedback entries or deletion events in the date range) @@ -766,7 +766,8 @@ def _get_implicit_bad_comments( ) -> list[dict]: """ Retrieves implicit bad comments: AI-generated comments created in the date range - that are on approved revisions but have no votes, no resolution, and aren't deleted. + that are on approved revisions but have no votes, no resolution, no Feedback entries, + and aren't deleted. These are inferred as unhelpful because the reviewer approved the revision without interacting with the comment.