fix: smart exclusions -- expand SKIP_DIRS, .saarignore support (OPE-128)

Author: DevanshuNEU

saar/extractor.py

@@ -50,8 +50,21 @@ class DNAExtractor:
     """
 
     SKIP_DIRS = {
-        "node_modules", ".git", "__pycache__", "venv", "env", "dist",
-        "build", ".next", "coverage", ".venv", "site-packages",
+        # version control
+        ".git",
+        # python
+        "__pycache__", "venv", "env", ".venv", "site-packages",
+        "*.egg-info", ".eggs",
+        # js/ts
+        "node_modules", ".next", ".nuxt", ".svelte-kit",
+        # build outputs
+        "dist", "build", "out", "target",
+        # test/coverage artifacts
+        "coverage", ".pytest_cache", "htmlcov", ".nyc_output",
+        # common data / cloned repo dirs that aren't project code
+        "repos", "data", "datasets", "tmp", "temp", "cache",
+        # ide
+        ".idea", ".vscode",
     }
     MAX_FILE_SIZE = 1024 * 1024  # 1MB
     MAX_FILES = 5000
@@ -168,33 +181,35 @@ def _detect_language(self, file_path: str) -> str:
             ".ts": "typescript", ".tsx": "typescript",
         }.get(ext, "unknown")
 
-    def _read_gitignore_dirs(self, repo_path: Path) -> set:
-        """Parse .gitignore for directory patterns to skip.
+    def _read_ignore_dirs(self, repo_path: Path) -> set:
+        """Parse .gitignore and .saarignore for directory patterns to skip.
 
-        Only extracts simple directory names (like 'repos/' or 'data/').
-        Does not handle globs or negation -- those need a full gitignore parser.
+        Reads both files and merges results. Only handles simple directory
+        names and trailing-slash patterns -- no glob negation. A full
+        gitignore-spec parser is overkill for our use case.
         """
         dirs: set = set()
-        gitignore = repo_path / ".gitignore"
-        if not gitignore.exists():
-            return dirs
-        try:
-            for line in gitignore.read_text(encoding="utf-8").splitlines():
-                line = line.strip()
-                if not line or line.startswith("#"):
-                    continue
-                # lines ending with / are directories
-                if line.endswith("/"):
-                    dirs.add(line.rstrip("/"))
-                # bare names that exist as dirs
-                elif "/" not in line and "*" not in line and "!" not in line:
-                    candidate = repo_path / line
-                    if candidate.is_dir():
-                        dirs.add(line)
-        except Exception as e:
-            logger.debug("Error reading .gitignore: %s", e)
+        # check both ignore files -- .saarignore takes same syntax as .gitignore
+        for ignore_file in [repo_path / ".gitignore", repo_path / ".saarignore"]:
+            if not ignore_file.exists():
+                continue
+            try:
+                for line in ignore_file.read_text(encoding="utf-8").splitlines():
+                    line = line.strip()
+                    if not line or line.startswith("#"):
+                        continue
+                    # lines ending with / are explicitly directories
+                    if line.endswith("/"):
+                        dirs.add(line.rstrip("/"))
+                    # bare names without glob chars that exist as dirs in the repo
+                    elif "/" not in line and "*" not in line and "!" not in line:
+                        candidate = repo_path / line
+                        if candidate.is_dir():
+                            dirs.add(line)
+            except Exception as e:
+                logger.debug("Error reading %s: %s", ignore_file.name, e)
         if dirs:
-            logger.info("Gitignore dirs to skip: %s", dirs)
+            logger.debug("Ignore file dirs to skip: %s", dirs)
         return dirs
 
     # -- team rules -------------------------------------------------------
@@ -652,9 +667,14 @@ def extract(
         skip = set(self.SKIP_DIRS)
         if exclude_dirs:
             skip.update(exclude_dirs)
-        skip.update(self._read_gitignore_dirs(path))
+        skip.update(self._read_ignore_dirs(path))
         self._active_skip_dirs = skip
 
+        # show user-added skips at info level so --verbose surfaces them
+        extra_skips = skip - self.SKIP_DIRS
+        if extra_skips:
+            logger.info("Extra dirs excluded: %s", sorted(extra_skips))
+
         self._reset_cache()
         repo_name = path.name
         logger.info("Extracting DNA from %s", repo_name)

tests/test_extractor.py

@@ -195,3 +195,60 @@ def test_respects_max_file_size(self, tmp_path: Path):
         extractor = DNAExtractor()
         dna = extractor.extract(str(tmp_path))
         assert dna is not None
+
+    def test_skips_repos_dir(self, tmp_path: Path):
+        """repos/ should be skipped -- it contains user-cloned repos, not project code."""
+        repos_dir = tmp_path / "repos" / "some-user-repo"
+        repos_dir.mkdir(parents=True)
+        # put a python file inside repos/ -- should NOT be counted
+        (repos_dir / "main.py").write_text("def user_code(): pass\n" * 50)
+        # real project code
+        (tmp_path / "app.py").write_text("def real_function(): pass\n")
+
+        extractor = DNAExtractor()
+        dna = extractor.extract(str(tmp_path))
+        assert dna is not None
+        # only app.py should be counted, not repos/
+        assert dna.language_distribution.get("python", 0) == 1
+
+    def test_skips_dist_and_build(self, tmp_path: Path):
+        """dist/ and build/ are generated artifacts, not source code."""
+        for junk_dir in ["dist", "build", "out"]:
+            d = tmp_path / junk_dir
+            d.mkdir()
+            (d / "bundle.py").write_text("# compiled output\n" * 100)
+        (tmp_path / "src.py").write_text("def real(): pass\n")
+
+        extractor = DNAExtractor()
+        dna = extractor.extract(str(tmp_path))
+        assert dna is not None
+        assert dna.language_distribution.get("python", 0) == 1
+
+    def test_respects_saarignore(self, tmp_path: Path):
+        """.saarignore uses same syntax as .gitignore and is merged into skip dirs."""
+        custom_dir = tmp_path / "vendor"
+        custom_dir.mkdir()
+        (custom_dir / "lib.py").write_text("def vendor_code(): pass\n" * 20)
+        (tmp_path / "app.py").write_text("def real(): pass\n")
+        # tell saar to skip vendor/ via .saarignore
+        (tmp_path / ".saarignore").write_text("vendor/\n")
+
+        extractor = DNAExtractor()
+        dna = extractor.extract(str(tmp_path))
+        assert dna is not None
+        assert dna.language_distribution.get("python", 0) == 1
+
+    def test_saarignore_stacks_with_gitignore(self, tmp_path: Path):
+        """Both .gitignore and .saarignore dirs are skipped -- they merge, not replace."""
+        (tmp_path / "gitignored").mkdir()
+        (tmp_path / "gitignored" / "a.py").write_text("x = 1\n" * 10)
+        (tmp_path / "saarignored").mkdir()
+        (tmp_path / "saarignored" / "b.py").write_text("y = 2\n" * 10)
+        (tmp_path / "real.py").write_text("def real(): pass\n")
+        (tmp_path / ".gitignore").write_text("gitignored/\n")
+        (tmp_path / ".saarignore").write_text("saarignored/\n")
+
+        extractor = DNAExtractor()
+        dna = extractor.extract(str(tmp_path))
+        assert dna is not None
+        assert dna.language_distribution.get("python", 0) == 1