apply Dachary feedback and update to support yaml and shared toctree functionality

Author: cbullinger

audit-cli/README.md

@@ -223,7 +223,9 @@ With `-v` flag, also shows:
 
 #### `analyze includes`
 
-Analyze `include` directive relationships in RST files to understand file dependencies.
+Analyze `include` directive and `toctree` relationships in RST files to understand file dependencies.
+
+This command recursively follows both `.. include::` directives and `.. toctree::` entries to show all files that are referenced from a starting file. This provides a complete picture of file dependencies including both content includes and table of contents structure.
 
 **Use Cases:**
 
@@ -232,6 +234,7 @@ This command helps writers:
 - Identify circular include dependencies (files included multiple times)
 - Document file relationships for maintenance
 - Plan refactoring of complex include structures
+- Understand table of contents structure and page hierarchies
 
 **Basic Usage:**
 
@@ -283,35 +286,35 @@ times (e.g., file A includes file C, and file B also includes file C), the file
 However, the tree view will show it in all locations where it appears, with subsequent occurrences marked as circular
 includes in verbose mode.
 
-#### `analyze references`
+#### `analyze file-references`
 
-Find all files that reference a target file through RST directives. This performs reverse dependency analysis, showing which files reference the target file through `include`, `literalinclude`, or `io-code-block` directives.
+Find all files that reference a target file through RST directives. This performs reverse dependency analysis, showing which files reference the target file through `include`, `literalinclude`, `io-code-block`, or `toctree` directives.
 
-The command searches all RST files (both `.rst` and `.txt` extensions) in the source directory tree.
+The command searches all RST files (`.rst` and `.txt` extensions) and YAML files (`.yaml` and `.yml` extensions) in the source directory tree. YAML files are included because extract and release files contain RST directives within their content blocks.
 
 **Use Cases:**
 
 This command helps writers:
 - Understand the impact of changes to a file (what pages will be affected)
 - Find all usages of an include file across the documentation
 - Track where code examples are referenced
-- Identify orphaned files (files with no references)
+- Identify orphaned files (files with no references, including toctree entries)
 - Plan refactoring by understanding file dependencies
 
 **Basic Usage:**
 
 ```bash
 # Find what references an include file
-./audit-cli analyze references path/to/includes/fact.rst
+./audit-cli analyze file-references path/to/includes/fact.rst
 
 # Find what references a code example
-./audit-cli analyze references path/to/code-examples/example.js
+./audit-cli analyze file-references path/to/code-examples/example.js
 
 # Get JSON output for automation
-./audit-cli analyze references path/to/file.rst --format json
+./audit-cli analyze file-references path/to/file.rst --format json
 
 # Show detailed information with line numbers
-./audit-cli analyze references path/to/file.rst --verbose
+./audit-cli analyze file-references path/to/file.rst --verbose
 ```
 
 **Flags:**
@@ -438,39 +441,39 @@ include             : 3 files, 4 references
 
 ```bash
 # Check if an include file is being used
-./audit-cli analyze references ~/docs/source/includes/fact-atlas.rst
+./audit-cli analyze file-references ~/docs/source/includes/fact-atlas.rst
 
 # Find all pages that use a specific code example
-./audit-cli analyze references ~/docs/source/code-examples/connect.py
+./audit-cli analyze file-references ~/docs/source/code-examples/connect.py
 
 # Get machine-readable output for scripting
-./audit-cli analyze references ~/docs/source/includes/fact.rst --format json | jq '.total_references'
+./audit-cli analyze file-references ~/docs/source/includes/fact.rst --format json | jq '.total_references'
 
 # See exactly where a file is referenced (with line numbers)
-./audit-cli analyze references ~/docs/source/includes/intro.rst --verbose
+./audit-cli analyze file-references ~/docs/source/includes/intro.rst --verbose
 
 # Quick check: just show the count
-./audit-cli analyze references ~/docs/source/includes/fact.rst --count-only
+./audit-cli analyze file-references ~/docs/source/includes/fact.rst --count-only
 # Output: 5
 
 # Get list of files for piping to other commands
-./audit-cli analyze references ~/docs/source/includes/fact.rst --paths-only
+./audit-cli analyze file-references ~/docs/source/includes/fact.rst --paths-only
 # Output:
 # page1.rst
 # page2.rst
 # page3.rst
 
 # Filter to only show include directives (not literalinclude or io-code-block)
-./audit-cli analyze references ~/docs/source/includes/fact.rst --directive-type include
+./audit-cli analyze file-references ~/docs/source/includes/fact.rst --directive-type include
 
 # Filter to only show literalinclude references
-./audit-cli analyze references ~/docs/source/code-examples/example.py --directive-type literalinclude
+./audit-cli analyze file-references ~/docs/source/code-examples/example.py --directive-type literalinclude
 
 # Combine filters: count only literalinclude references
-./audit-cli analyze references ~/docs/source/code-examples/example.py -t literalinclude -c
+./audit-cli analyze file-references ~/docs/source/code-examples/example.py -t literalinclude -c
 
 # Combine filters: list files that use this as an io-code-block
-./audit-cli analyze references ~/docs/source/code-examples/query.js -t io-code-block --paths-only
+./audit-cli analyze file-references ~/docs/source/code-examples/query.js -t io-code-block --paths-only
 ```
 
 ### Compare Commands

audit-cli/commands/analyze/analyze.go

@@ -3,14 +3,14 @@
 // This package serves as the parent command for various analysis operations.
 // Currently supports:
 //   - includes: Analyze include directive relationships in RST files
-//   - references: Find all files that reference a target file
+//   - file-references: Find all files that reference a target file
 //
 // Future subcommands could include analyzing cross-references, broken links, or content metrics.
 package analyze
 
 import (
 	"github.com/mongodb/code-example-tooling/audit-cli/commands/analyze/includes"
-	"github.com/mongodb/code-example-tooling/audit-cli/commands/analyze/references"
+	filereferences "github.com/mongodb/code-example-tooling/audit-cli/commands/analyze/file-references"
 	"github.com/spf13/cobra"
 )
 
@@ -26,14 +26,14 @@ func NewAnalyzeCommand() *cobra.Command {
 
 Currently supports:
   - includes: Analyze include directive relationships (forward dependencies)
-  - references: Find all files that reference a target file (reverse dependencies)
+  - file-references: Find all files that reference a target file (reverse dependencies)
 
 Future subcommands may support analyzing cross-references, broken links, or content metrics.`,
 	}
 
 	// Add subcommands
 	cmd.AddCommand(includes.NewIncludesCommand())
-	cmd.AddCommand(references.NewReferencesCommand())
+	cmd.AddCommand(filereferences.NewFileReferencesCommand())
 
 	return cmd
 }

audit-cli/commands/analyze/references/analyzer.go renamed to audit-cli/commands/analyze/file-references/analyzer.go

@@ -1,4 +1,4 @@
-package references
+package filereferences
 
 import (
 	"bufio"
@@ -10,6 +10,7 @@ import (
 	"strings"
 
 	"github.com/mongodb/code-example-tooling/audit-cli/internal/pathresolver"
+	"github.com/mongodb/code-example-tooling/audit-cli/internal/rst"
 )
 
 // Regular expressions for matching directives
@@ -32,9 +33,10 @@ var (
 
 // AnalyzeReferences finds all files that reference the target file.
 //
-// This function searches through all RST files in the source directory to find
-// files that reference the target file using include, literalinclude, or
-// io-code-block directives.
+// This function searches through all RST files (.rst, .txt) and YAML files (.yaml, .yml)
+// in the source directory to find files that reference the target file using include,
+// literalinclude, or io-code-block directives. YAML files are included because extract
+// and release files contain RST directives within their content blocks.
 //
 // Parameters:
 //   - targetFile: Absolute path to the file to analyze
@@ -62,7 +64,7 @@ func AnalyzeReferences(targetFile string) (*ReferenceAnalysis, error) {
 		ReferencingFiles: []FileReference{},
 	}
 
-	// Walk through all RST files in the source directory
+	// Walk through all RST and YAML files in the source directory
 	err = filepath.Walk(sourceDir, func(path string, info os.FileInfo, err error) error {
 		if err != nil {
 			return err
@@ -73,9 +75,10 @@ func AnalyzeReferences(targetFile string) (*ReferenceAnalysis, error) {
 			return nil
 		}
 
-		// Only process RST files (.rst and .txt extensions)
+		// Only process RST files (.rst, .txt) and YAML files (.yaml, .yml)
+		// YAML files may contain RST directives in extract/release content blocks
 		ext := filepath.Ext(path)
-		if ext != ".rst" && ext != ".txt" {
+		if ext != ".rst" && ext != ".txt" && ext != ".yaml" && ext != ".yml" {
 			return nil
 		}
 
@@ -107,7 +110,7 @@ func AnalyzeReferences(targetFile string) (*ReferenceAnalysis, error) {
 // findReferencesInFile searches a single file for references to the target file.
 //
 // This function scans through the file line by line looking for include,
-// literalinclude, and io-code-block directives that reference the target file.
+// literalinclude, io-code-block, and toctree directives that reference the target file.
 //
 // Parameters:
 //   - filePath: Path to the file to search
@@ -129,19 +132,33 @@ func findReferencesInFile(filePath, targetFile, sourceDir string) ([]FileReferen
 	lineNum := 0
 	inIOCodeBlock := false
 	ioCodeBlockStartLine := 0
+	inToctree := false
+	toctreeStartLine := 0
 
 	for scanner.Scan() {
 		lineNum++
 		line := scanner.Text()
 		trimmedLine := strings.TrimSpace(line)
 
+		// Check for toctree start (use shared regex from rst package)
+		if rst.ToctreeDirectiveRegex.MatchString(trimmedLine) {
+			inToctree = true
+			toctreeStartLine = lineNum
+			continue
+		}
+
 		// Check for io-code-block start
 		if ioCodeBlockRegex.MatchString(trimmedLine) {
 			inIOCodeBlock = true
 			ioCodeBlockStartLine = lineNum
 			continue
 		}
 
+		// Check if we're exiting toctree (unindented line that's not empty and not an option)
+		if inToctree && len(line) > 0 && line[0] != ' ' && line[0] != '\t' {
+			inToctree = false
+		}
+
 		// Check if we're exiting io-code-block (unindented line that's not empty)
 		if inIOCodeBlock && len(line) > 0 && line[0] != ' ' && line[0] != '\t' {
 			inIOCodeBlock = false
@@ -205,6 +222,26 @@ func findReferencesInFile(filePath, targetFile, sourceDir string) ([]FileReferen
 				continue
 			}
 		}
+
+		// Check for toctree entries (indented document names)
+		if inToctree {
+			// Skip empty lines and option lines (starting with :)
+			if trimmedLine == "" || strings.HasPrefix(trimmedLine, ":") {
+				continue
+			}
+
+			// This is a document name in the toctree
+			// Document names can be relative or absolute (starting with /)
+			docName := trimmedLine
+			if referencesToctreeTarget(docName, targetFile, sourceDir, filePath) {
+				references = append(references, FileReference{
+					FilePath:      filePath,
+					DirectiveType: "toctree",
+					ReferencePath: docName,
+					LineNumber:    toctreeStartLine,
+				})
+			}
+		}
 	}
 
 	if err := scanner.Err(); err != nil {
@@ -250,6 +287,31 @@ func referencesTarget(refPath, targetFile, sourceDir, currentFile string) bool {
 	return absResolvedPath == targetFile
 }
 
+// referencesToctreeTarget checks if a toctree document name points to the target file.
+//
+// This function uses the shared rst.ResolveToctreePath to resolve the document name
+// and then compares it to the target file.
+//
+// Parameters:
+//   - docName: The document name from the toctree (e.g., "intro" or "/includes/intro")
+//   - targetFile: Absolute path to the target file
+//   - sourceDir: Source directory (for resolving relative paths)
+//   - currentFile: Path to the file containing the toctree
+//
+// Returns:
+//   - bool: true if the document name points to the target file
+func referencesToctreeTarget(docName, targetFile, sourceDir, currentFile string) bool {
+	// Use the shared toctree path resolution from rst package
+	resolvedPath, err := rst.ResolveToctreePath(currentFile, docName)
+	if err != nil {
+		// If we can't resolve it, it doesn't match
+		return false
+	}
+
+	// Compare with target file
+	return resolvedPath == targetFile
+}
+
 // FilterByDirectiveType filters the analysis results to only include references
 // of the specified directive type.
 //