Merge pull request #51 from BapRx/feat/exclude-paths-based-on-regex

chore(repo/find): Exlude paths based on regex

Author: hakoerber

Justfile

@@ -13,6 +13,7 @@ clean:
 
 fmt:
     cargo fmt
+    git ls-files | grep '\.py$' | xargs isort
     git ls-files | grep '\.py$' | xargs black
     git ls-files | grep '\.sh$' | xargs -L 1 shfmt --indent 4 --write
 
@@ -23,6 +24,7 @@ fmt-check:
 
 lint:
     cargo clippy --no-deps -- -Dwarnings
+    git ls-files | grep '\.py$' | xargs ruff --ignore E501
     git ls-files | grep '\.sh$' | xargs -L 1 shellcheck --norc
 
 lint-fix:

depcheck/update-cargo-dependencies.py

@@ -1,9 +1,8 @@
 #!/usr/bin/env python3
 
-import subprocess
-import os
 import json
-import sys
+import os
+import subprocess
 
 import semver
 import tomlkit

docs/src/developing.md

@@ -34,8 +34,8 @@ You will need the following tools:
   [here](https://github.com/casey/just#installation) for installation
   instructions (it's most likely just a simple `cargo install just`).
 * Docker & docker-compose for the e2e tests
-* `black` and `shfmt` for formatting.
-* `shellcheck` for shell script linting
+* `isort`, `black` and `shfmt` for formatting.
+* `ruff` and `shellcheck` for linting.
 * `mdbook` for the documentation
 
 Here are the tools:

docs/src/local_configuration.md

@@ -11,7 +11,7 @@ Let's try it out:
 ## Get the example configuration
 
 ```bash
-$ curl --proto '=https' --tlsv1.2 -sSfO https://raw.githubusercontent.com/hakoerber/git-repo-manager/master/example.config.toml
+curl --proto '=https' --tlsv1.2 -sSfO https://raw.githubusercontent.com/hakoerber/git-repo-manager/master/example.config.toml
 ```
 
 Then, you're ready to run the first sync. This will clone all configured
@@ -30,7 +30,7 @@ $ grm repos sync config --config example.config.toml
 
 If you run it again, it will report no changes:
 
-```
+```bash
 $ grm repos sync config -c example.config.toml
 [✔] git-repo-manager: OK
 [✔] dotfiles: OK
@@ -43,11 +43,18 @@ write a configuration from scratch. Luckily, GRM has a way to generate a
 configuration from an existing file tree:
 
 ```bash
-$ grm repos find local ~/your/project/root > config.toml
+grm repos find local ~/your/project/root > config.toml
 ```
 
 This will detect all repositories and remotes and write them to `config.toml`.
 
+You can exclude repositories from the generated configuration by providing
+a regex that will be test against the path of each discovered repository:
+
+```bash
+grm repos find local ~/your/project/root --exclude "^.*/subdir/match-(foo|bar)/.*$" > config.toml
+```
+
 ### Show the state of your projects
 
 ```bash
@@ -65,7 +72,7 @@ $ grm repos status --config example.config.toml
 You can also use `status` without `--config` to check the repository you're
 currently in:
 
-```
+```bash
 $ cd ~/example-projects/dotfiles
 $ grm repos status
 ╭──────────┬──────────┬────────┬──────────┬───────┬─────────╮
@@ -79,5 +86,5 @@ $ grm repos status
 
 By default, the repo configuration uses TOML. If you prefer YAML, just give it a
 YAML file instead (file ending does not matter, `grm` will figure out the
-format).  For generating a configuration, pass `--format yaml` to `grm repo
+format). For generating a configuration, pass `--format yaml` to `grm repo
 find` which generates a YAML configuration instead of a TOML configuration.

docs/src/testing.md

@@ -49,7 +49,7 @@ Note: You will most likely not need to read this.
 Each test parameter will exponentially increase the number of tests that will be
 run. As a general rule, comprehensiveness is more important than test suite
 runtime (so if in doubt, better to add another parameter to catch every edge
-case). But try to keep the total runtime sane. Currently, the whole `just e2e`
+case). But try to keep the total runtime sane. Currently, the whole `just test-e2e`
 target runs ~8'000 tests and takes around 5 minutes on my machine, exlucding
 binary and docker build time. I'd say that keeping it under 10 minutes is a good
 idea.

e2e_tests/conftest.py

@@ -1,7 +1,5 @@
 import os
 
-from helpers import *
-
 
 def pytest_configure(config):
     os.environ["GIT_AUTHOR_NAME"] = "Example user"

e2e_tests/docker-rest/flask/app.py

@@ -3,5 +3,5 @@
 app = Flask(__name__)
 app.url_map.strict_slashes = False
 
-import github
-import gitlab
+import github  # noqa: E402,F401
+import gitlab  # noqa: E402,F401

e2e_tests/docker-rest/flask/github.py

@@ -1,10 +1,8 @@
 import os.path
 
-from app import app
-
-from flask import Flask, request, abort, jsonify, make_response
-
 import jinja2
+from app import app
+from flask import abort, jsonify, make_response, request
 
 
 def check_headers():
@@ -48,7 +46,7 @@ def args(page):
 
 def read_project_files(namespaces=[]):
     last_page = 4
-    page = username = int(request.args.get("page", "1"))
+    page = int(request.args.get("page", "1"))
     response_file = f"./github_api_page_{page}.json.j2"
     if not os.path.exists(response_file):
         return jsonify([])

e2e_tests/docker-rest/flask/gitlab.py

@@ -1,10 +1,8 @@
 import os.path
 
-from app import app
-
-from flask import Flask, request, abort, jsonify, make_response
-
 import jinja2
+from app import app
+from flask import abort, jsonify, make_response, request
 
 
 def check_headers():
@@ -48,7 +46,7 @@ def args(page):
 
 def read_project_files(namespaces=[]):
     last_page = 4
-    page = username = int(request.args.get("page", "1"))
+    page = int(request.args.get("page", "1"))
     response_file = f"./gitlab_api_page_{page}.json"
     if not os.path.exists(response_file):
         return jsonify([])

e2e_tests/helpers.py

@@ -1,12 +1,12 @@
 #!/usr/bin/env python3
 
+import hashlib
+import inspect
 import os
 import os.path
+import shutil
 import subprocess
 import tempfile
-import hashlib
-import shutil
-import inspect
 
 import git
 
@@ -176,7 +176,6 @@ def get(cls, cachekey=None, initfunc=None):
             newobj.remoteid = remoteid
             return newobj, remoteid
         else:
-            refresh = False
             if cachekey not in cls.obj:
                 tmpdir = get_temporary_directory()
                 shell(

e2e_tests/test_basic.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 
-from helpers import *
+from helpers import grm
 
 
 def test_invalid_command():

e2e_tests/test_repos_find.py

@@ -1,12 +1,13 @@
 #!/usr/bin/env python3
 
+import os
+import re
 import tempfile
 
-import toml
 import pytest
+import toml
 import yaml
-
-from helpers import *
+from helpers import NonExistentPath, TempGitRepository, grm, shell
 
 
 def test_repos_find_nonexistent():
@@ -63,9 +64,10 @@ def test_repos_find_non_git_repos():
         assert len(cmd.stderr) != 0
 
 
-@pytest.mark.parametrize("default", [True, False])
+@pytest.mark.parametrize("default_format", [True, False])
 @pytest.mark.parametrize("configtype", ["toml", "yaml"])
-def test_repos_find(configtype, default):
+@pytest.mark.parametrize("exclude", [None, "^.*/repo2$", "^not_matching$"])
+def test_repos_find(configtype, exclude, default_format):
     with tempfile.TemporaryDirectory() as tmpdir:
         shell(
             f"""
@@ -99,13 +101,19 @@ def test_repos_find(configtype, default):
         )
 
         args = ["repos", "find", "local", tmpdir]
-        if not default:
+        if not default_format:
             args += ["--format", configtype]
+        if exclude:
+            args += ["--exclude", exclude]
         cmd = grm(args)
         assert cmd.returncode == 0
-        assert len(cmd.stderr) == 0
+        if exclude == "^.*/repo2$":
+            assert re.match(r"^.*\[skipped\] .*\/repo2$", cmd.stderr.lower())
+            assert "repo2" in cmd.stderr.lower()
+        else:
+            assert len(cmd.stderr) == 0
 
-        if default or configtype == "toml":
+        if default_format or configtype == "toml":
             output = toml.loads(cmd.stdout)
         elif configtype == "yaml":
             output = yaml.safe_load(cmd.stdout)
@@ -120,7 +128,7 @@ def test_repos_find(configtype, default):
             assert set(tree.keys()) == {"root", "repos"}
             assert tree["root"] == tmpdir
             assert isinstance(tree["repos"], list)
-            assert len(tree["repos"]) == 2
+            assert len(tree["repos"]) == (1 if exclude == "^.*/repo2$" else 2)
 
             repo1 = [r for r in tree["repos"] if r["name"] == "repo1"][0]
             assert repo1["worktree_setup"] is False
@@ -137,30 +145,33 @@ def test_repos_find(configtype, default):
             assert someremote["type"] == "ssh"
             assert someremote["url"] == "ssh://example.com/repo2.git"
 
-            repo2 = [r for r in tree["repos"] if r["name"] == "repo2"][0]
-            assert repo2["worktree_setup"] is False
-            assert isinstance(repo1["remotes"], list)
-            assert len(repo2["remotes"]) == 1
+            if exclude == "^.*/repo2$":
+                assert [r for r in tree["repos"] if r["name"] == "repo2"] == []
+            else:
+                repo2 = [r for r in tree["repos"] if r["name"] == "repo2"][0]
+                assert repo2["worktree_setup"] is False
+                assert isinstance(repo1["remotes"], list)
+                assert len(repo2["remotes"]) == 1
 
-            origin = [r for r in repo2["remotes"] if r["name"] == "origin"][0]
-            assert set(origin.keys()) == {"name", "type", "url"}
-            assert origin["type"] == "https"
-            assert origin["url"] == "https://example.com/repo2.git"
+                origin = [r for r in repo2["remotes"] if r["name"] == "origin"][0]
+                assert set(origin.keys()) == {"name", "type", "url"}
+                assert origin["type"] == "https"
+                assert origin["url"] == "https://example.com/repo2.git"
 
 
-@pytest.mark.parametrize("default", [True, False])
+@pytest.mark.parametrize("default_format", [True, False])
 @pytest.mark.parametrize("configtype", ["toml", "yaml"])
-def test_repos_find_in_root(configtype, default):
+def test_repos_find_in_root(configtype, default_format):
     with TempGitRepository() as repo_dir:
 
         args = ["repos", "find", "local", repo_dir]
-        if not default:
+        if not default_format:
             args += ["--format", configtype]
         cmd = grm(args)
         assert cmd.returncode == 0
         assert len(cmd.stderr) == 0
 
-        if default or configtype == "toml":
+        if default_format or configtype == "toml":
             output = toml.loads(cmd.stdout)
         elif configtype == "yaml":
             output = yaml.safe_load(cmd.stdout)
@@ -194,8 +205,8 @@ def test_repos_find_in_root(configtype, default):
 
 
 @pytest.mark.parametrize("configtype", ["toml", "yaml"])
-@pytest.mark.parametrize("default", [True, False])
-def test_repos_find_with_invalid_repo(configtype, default):
+@pytest.mark.parametrize("default_format", [True, False])
+def test_repos_find_with_invalid_repo(configtype, default_format):
     with tempfile.TemporaryDirectory() as tmpdir:
         shell(
             f"""
@@ -229,13 +240,13 @@ def test_repos_find_with_invalid_repo(configtype, default):
         )
 
         args = ["repos", "find", "local", tmpdir]
-        if not default:
+        if not default_format:
             args += ["--format", configtype]
         cmd = grm(args)
         assert cmd.returncode == 0
         assert "broken" in cmd.stderr
 
-        if default or configtype == "toml":
+        if default_format or configtype == "toml":
             output = toml.loads(cmd.stdout)
         elif configtype == "yaml":
             output = yaml.safe_load(cmd.stdout)