refactor(toctree matching): create new directive_regex.go and optional toctree flag

Author: cbullinger

audit-cli/README.md

@@ -294,22 +294,29 @@ The command searches all RST files (`.rst` and `.txt` extensions) and YAML files
 
 **Use Cases:**
 
+By default, this command searches for content inclusion directives (include, literalinclude,
+io-code-block) that transclude content into pages. Use `--include-toctree` to also search
+for toctree entries, which are navigation links rather than content transclusion.
+
 This command helps writers:
 - Understand the impact of changes to a file (what pages will be affected)
 - Find all usages of an include file across the documentation
 - Track where code examples are referenced
-- Identify orphaned files (files with no references from include, literalinclude, io-code-block, or toctree directives)
+- Identify orphaned files (files with no references from content inclusion directives)
 - Plan refactoring by understanding file dependencies
 
 **Basic Usage:**
 
 ```bash
-# Find what references an include file
+# Find what references an include file (content inclusion only)
 ./audit-cli analyze file-references path/to/includes/fact.rst
 
 # Find what references a code example
 ./audit-cli analyze file-references path/to/code-examples/example.js
 
+# Include toctree references (navigation links)
+./audit-cli analyze file-references path/to/file.rst --include-toctree
+
 # Get JSON output for automation
 ./audit-cli analyze file-references path/to/file.rst --format json
 
@@ -323,7 +330,8 @@ This command helps writers:
 - `-v, --verbose` - Show detailed information including line numbers and reference paths
 - `-c, --count-only` - Only show the count of references (useful for quick checks and scripting)
 - `--paths-only` - Only show the file paths, one per line (useful for piping to other commands)
-- `-t, --directive-type <type>` - Filter by directive type: `include`, `literalinclude`, or `io-code-block`
+- `-t, --directive-type <type>` - Filter by directive type: `include`, `literalinclude`, `io-code-block`, or `toctree`
+- `--include-toctree` - Include toctree entries (navigation links) in addition to content inclusion directives
 
 **Understanding the Counts:**
 
@@ -339,20 +347,20 @@ This helps identify both the impact scope (how many files) and duplicate include
 
 **Supported Directive Types:**
 
-The command tracks three types of RST directives:
+By default, the command tracks content inclusion directives:
 
-1. **`.. include::`** - RST content includes
+1. **`.. include::`** - RST content includes (transcluded)
    ```rst
    .. include:: /includes/intro.rst
    ```
 
-2. **`.. literalinclude::`** - Code file references
+2. **`.. literalinclude::`** - Code file references (transcluded)
    ```rst
    .. literalinclude:: /code-examples/example.py
       :language: python
    ```
 
-3. **`.. io-code-block::`** - Input/output examples with file arguments
+3. **`.. io-code-block::`** - Input/output examples with file arguments (transcluded)
    ```rst
    .. io-code-block::
 
@@ -363,6 +371,17 @@ The command tracks three types of RST directives:
          :language: json
    ```
 
+With `--include-toctree`, also tracks:
+
+4. **`.. toctree::`** - Table of contents entries (navigation links, not transcluded)
+   ```rst
+   .. toctree::
+      :maxdepth: 2
+
+      intro
+      getting-started
+   ```
+
 **Note:** Only file-based references are tracked. Inline content (e.g., `.. input::` with `:language:` but no file path) is not tracked.
 
 **Output Formats:**

audit-cli/commands/analyze/file-references/analyzer.go

@@ -5,46 +5,31 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
-	"regexp"
 	"sort"
 	"strings"
 
 	"github.com/mongodb/code-example-tooling/audit-cli/internal/pathresolver"
 	"github.com/mongodb/code-example-tooling/audit-cli/internal/rst"
 )
 
-// Regular expressions for matching directives
-var (
-	// includeRegex matches: .. include:: /path/to/file.rst
-	includeRegex = regexp.MustCompile(`^\.\.\s+include::\s+(.+)$`)
-
-	// literalIncludeRegex matches: .. literalinclude:: /path/to/file.ext
-	literalIncludeRegex = regexp.MustCompile(`^\.\.\s+literalinclude::\s+(.+)$`)
-
-	// ioCodeBlockRegex matches: .. io-code-block::
-	ioCodeBlockRegex = regexp.MustCompile(`^\.\.\s+io-code-block::`)
-
-	// inputRegex matches: .. input:: /path/to/file.ext (within io-code-block)
-	inputRegex = regexp.MustCompile(`^\.\.\s+input::\s+(.+)$`)
-
-	// outputRegex matches: .. output:: /path/to/file.ext (within io-code-block)
-	outputRegex = regexp.MustCompile(`^\.\.\s+output::\s+(.+)$`)
-)
-
 // AnalyzeReferences finds all files that reference the target file.
 //
 // This function searches through all RST files (.rst, .txt) and YAML files (.yaml, .yml)
 // in the source directory to find files that reference the target file using include,
 // literalinclude, or io-code-block directives. YAML files are included because extract
 // and release files contain RST directives within their content blocks.
 //
+// By default, only content inclusion directives are searched. Set includeToctree to true
+// to also search for toctree entries (navigation links).
+//
 // Parameters:
 //   - targetFile: Absolute path to the file to analyze
+//   - includeToctree: If true, include toctree entries in the search
 //
 // Returns:
 //   - *ReferenceAnalysis: The analysis results
 //   - error: Any error encountered during analysis
-func AnalyzeReferences(targetFile string) (*ReferenceAnalysis, error) {
+func AnalyzeReferences(targetFile string, includeToctree bool) (*ReferenceAnalysis, error) {
 	// Get absolute path
 	absTargetFile, err := filepath.Abs(targetFile)
 	if err != nil {
@@ -83,7 +68,7 @@ func AnalyzeReferences(targetFile string) (*ReferenceAnalysis, error) {
 		}
 
 		// Search for references in this file
-		refs, err := findReferencesInFile(path, absTargetFile, sourceDir)
+		refs, err := findReferencesInFile(path, absTargetFile, sourceDir, includeToctree)
 		if err != nil {
 			// Log error but continue processing other files
 			fmt.Fprintf(os.Stderr, "Warning: failed to process %s: %v\n", path, err)
@@ -110,17 +95,19 @@ func AnalyzeReferences(targetFile string) (*ReferenceAnalysis, error) {
 // findReferencesInFile searches a single file for references to the target file.
 //
 // This function scans through the file line by line looking for include,
-// literalinclude, io-code-block, and toctree directives that reference the target file.
+// literalinclude, and io-code-block directives that reference the target file.
+// If includeToctree is true, also searches for toctree entries.
 //
 // Parameters:
 //   - filePath: Path to the file to search
 //   - targetFile: Absolute path to the target file
 //   - sourceDir: Source directory (for resolving relative paths)
+//   - includeToctree: If true, include toctree entries in the search
 //
 // Returns:
 //   - []FileReference: List of references found in this file
 //   - error: Any error encountered during processing
-func findReferencesInFile(filePath, targetFile, sourceDir string) ([]FileReference, error) {
+func findReferencesInFile(filePath, targetFile, sourceDir string, includeToctree bool) ([]FileReference, error) {
 	file, err := os.Open(filePath)
 	if err != nil {
 		return nil, err
@@ -140,15 +127,15 @@ func findReferencesInFile(filePath, targetFile, sourceDir string) ([]FileReferen
 		line := scanner.Text()
 		trimmedLine := strings.TrimSpace(line)
 
-		// Check for toctree start (use shared regex from rst package)
-		if rst.ToctreeDirectiveRegex.MatchString(trimmedLine) {
+		// Check for toctree start (only if includeToctree is enabled)
+		if includeToctree && rst.ToctreeDirectiveRegex.MatchString(trimmedLine) {
 			inToctree = true
 			toctreeStartLine = lineNum
 			continue
 		}
 
 		// Check for io-code-block start
-		if ioCodeBlockRegex.MatchString(trimmedLine) {
+		if rst.IOCodeBlockDirectiveRegex.MatchString(trimmedLine) {
 			inIOCodeBlock = true
 			ioCodeBlockStartLine = lineNum
 			continue
@@ -165,7 +152,7 @@ func findReferencesInFile(filePath, targetFile, sourceDir string) ([]FileReferen
 		}
 
 		// Check for include directive
-		if matches := includeRegex.FindStringSubmatch(trimmedLine); matches != nil {
+		if matches := rst.IncludeDirectiveRegex.FindStringSubmatch(trimmedLine); matches != nil {
 			refPath := strings.TrimSpace(matches[1])
 			if referencesTarget(refPath, targetFile, sourceDir, filePath) {
 				references = append(references, FileReference{
@@ -179,7 +166,7 @@ func findReferencesInFile(filePath, targetFile, sourceDir string) ([]FileReferen
 		}
 
 		// Check for literalinclude directive
-		if matches := literalIncludeRegex.FindStringSubmatch(trimmedLine); matches != nil {
+		if matches := rst.LiteralIncludeDirectiveRegex.FindStringSubmatch(trimmedLine); matches != nil {
 			refPath := strings.TrimSpace(matches[1])
 			if referencesTarget(refPath, targetFile, sourceDir, filePath) {
 				references = append(references, FileReference{
@@ -195,7 +182,7 @@ func findReferencesInFile(filePath, targetFile, sourceDir string) ([]FileReferen
 		// Check for input/output directives within io-code-block
 		if inIOCodeBlock {
 			// Check for input directive
-			if matches := inputRegex.FindStringSubmatch(trimmedLine); matches != nil {
+			if matches := rst.InputDirectiveRegex.FindStringSubmatch(trimmedLine); matches != nil {
 				refPath := strings.TrimSpace(matches[1])
 				if referencesTarget(refPath, targetFile, sourceDir, filePath) {
 					references = append(references, FileReference{
@@ -209,7 +196,7 @@ func findReferencesInFile(filePath, targetFile, sourceDir string) ([]FileReferen
 			}
 
 			// Check for output directive
-			if matches := outputRegex.FindStringSubmatch(trimmedLine); matches != nil {
+			if matches := rst.OutputDirectiveRegex.FindStringSubmatch(trimmedLine); matches != nil {
 				refPath := strings.TrimSpace(matches[1])
 				if referencesTarget(refPath, targetFile, sourceDir, filePath) {
 					references = append(references, FileReference{

audit-cli/commands/analyze/file-references/file_references.go

@@ -37,11 +37,12 @@ import (
 //   - -t, --directive-type: Filter by directive type (include, literalinclude, io-code-block, toctree)
 func NewFileReferencesCommand() *cobra.Command {
 	var (
-		format        string
-		verbose       bool
-		countOnly     bool
-		pathsOnly     bool
-		directiveType string
+		format         string
+		verbose        bool
+		countOnly      bool
+		pathsOnly      bool
+		directiveType  string
+		includeToctree bool
 	)
 
 	cmd := &cobra.Command{
@@ -50,13 +51,15 @@ func NewFileReferencesCommand() *cobra.Command {
 		Long: `Find all files that reference a target file through RST directives.
 
 This command performs reverse dependency analysis, showing which files reference
-the target file through include, literalinclude, io-code-block, or toctree directives.
+the target file through content inclusion directives (include, literalinclude,
+io-code-block). Use --include-toctree to also search for toctree entries, which
+are navigation links rather than content transclusion.
 
 Supported directive types:
-  - .. include::         RST content includes
-  - .. literalinclude::  Code file references
-  - .. io-code-block::   Input/output examples with file arguments
-  - .. toctree::         Table of contents entries
+  - .. include::         RST content includes (transcluded)
+  - .. literalinclude::  Code file references (transcluded)
+  - .. io-code-block::   Input/output examples with file arguments (transcluded)
+  - .. toctree::         Table of contents entries (navigation links, requires --include-toctree)
 
 The command searches all RST files (.rst, .txt) and YAML files (.yaml, .yml) in
 the source directory tree. YAML files are included because extract and release
@@ -66,7 +69,7 @@ This is useful for:
   - Understanding the impact of changes to a file
   - Finding all usages of an include file
   - Tracking code example references
-  - Identifying orphaned files (files with no references, including toctree entries)
+  - Identifying orphaned files (files with no references from content inclusion directives)
 
 Examples:
   # Find what references an include file
@@ -75,6 +78,9 @@ Examples:
   # Find what references a code example
   analyze file-references /path/to/code-examples/example.js
 
+  # Include toctree references (navigation links)
+  analyze file-references /path/to/file.rst --include-toctree
+
   # Get JSON output
   analyze file-references /path/to/file.rst --format json
 
@@ -91,7 +97,7 @@ Examples:
   analyze file-references /path/to/file.rst --directive-type include`,
 		Args: cobra.ExactArgs(1),
 		RunE: func(cmd *cobra.Command, args []string) error {
-			return runReferences(args[0], format, verbose, countOnly, pathsOnly, directiveType)
+			return runReferences(args[0], format, verbose, countOnly, pathsOnly, directiveType, includeToctree)
 		},
 	}
 
@@ -100,6 +106,7 @@ Examples:
 	cmd.Flags().BoolVarP(&countOnly, "count-only", "c", false, "Only show the count of references")
 	cmd.Flags().BoolVar(&pathsOnly, "paths-only", false, "Only show the file paths (one per line)")
 	cmd.Flags().StringVarP(&directiveType, "directive-type", "t", "", "Filter by directive type (include, literalinclude, io-code-block, toctree)")
+	cmd.Flags().BoolVar(&includeToctree, "include-toctree", false, "Include toctree entries (navigation links) in addition to content inclusion directives")
 
 	return cmd
 }
@@ -115,10 +122,11 @@ Examples:
 //   - countOnly: If true, only show the count
 //   - pathsOnly: If true, only show the file paths
 //   - directiveType: Filter by directive type (empty string means all types)
+//   - includeToctree: If true, include toctree entries in the search
 //
 // Returns:
 //   - error: Any error encountered during analysis
-func runReferences(targetFile, format string, verbose, countOnly, pathsOnly bool, directiveType string) error {
+func runReferences(targetFile, format string, verbose, countOnly, pathsOnly bool, directiveType string, includeToctree bool) error {
 	// Validate directive type if specified
 	if directiveType != "" {
 		validTypes := map[string]bool{
@@ -147,7 +155,7 @@ func runReferences(targetFile, format string, verbose, countOnly, pathsOnly bool
 	}
 
 	// Perform analysis
-	analysis, err := AnalyzeReferences(targetFile)
+	analysis, err := AnalyzeReferences(targetFile, includeToctree)
 	if err != nil {
 		return fmt.Errorf("failed to analyze references: %w", err)
 	}