feat: allow the concurrency to be configured through CLI flags

antur/__main__.py

@@ -15,14 +15,22 @@ def parse_args() -> argparse.Namespace:
     parser.add_argument("url", nargs="?", help="The URL of a sitemap to start with.")
 
     parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
+
+    parser.add_argument(
+        "-c",
+        "--concurrent",
+        type=int,
+        help="The number of concurrent requests to make.",
+        default=40,
+    )
     return parser.parse_args()
 
 
 def main() -> None:
     """Run the Antur application."""
     args = parse_args()
 
-    app = AnturApp(args.url)
+    app = AnturApp(args.url, max_concurrent_requests=args.concurrent)
     app.run()
 
 

antur/app.py

@@ -37,19 +37,20 @@ class AnturApp(App):
 
     show_markdown = reactive(False)
 
-    def __init__(self: "AnturApp", url: str | None = None) -> None:
+    def __init__(self: "AnturApp", url: str | None = None, *, max_concurrent_requests: int) -> None:
         """Initialize the Antur application."""
         super().__init__()
 
         self.url = url
+        self.max_concurrent_requests = max_concurrent_requests
 
     def compose(self: "AnturApp") -> ComposeResult:
         """Compose the layout of the app."""
         yield Header(False)
         yield SearchBar(self.url)
         with Vertical(id="contents"):
             with Vertical(id="main"):
-                yield SitemapTree(self.url)
+                yield SitemapTree(self.url, max_concurrent_requests=self.max_concurrent_requests)
                 yield NodeInfo()
             yield Footer()
 

antur/utils/sitemap_parser.py

@@ -34,13 +34,11 @@ class Error:
 
 IGNORE_TAGS = ["lastmod", "changefreq", "priority", "loc"]
 
-MAX_CONCURRENT_REQUESTS = 40
-
 
 class SitemapParser:
     """Parse a sitemap file and return a dict of URLs and their metadata."""
 
-    def __init__(self: "SitemapParser", url: str) -> None:
+    def __init__(self: "SitemapParser", url: str, max_concurrent_requests: int) -> None:
         """Initialize the parser."""
         self.url = url
         self.data = {}
@@ -49,7 +47,7 @@ def __init__(self: "SitemapParser", url: str) -> None:
         self.http_errors = 0
         self.xml_errors = 0
 
-        self.semaphore = asyncio.Semaphore(MAX_CONCURRENT_REQUESTS)
+        self.semaphore = asyncio.Semaphore(max_concurrent_requests)
 
     async def get_data(self: "SitemapParser", url: str) -> bytes:
         """Fetch the data from the URL."""

antur/widgets/sitemap_tree.py

@@ -59,11 +59,18 @@ class SitemapTree(Widget):
 
     BINDINGS: ClassVar = [("o", "open", "Open the selected item in the browser.")]
 
-    def __init__(self: "SitemapTree", url: str | None = None, *args: tuple, **kwargs: dict) -> None:
+    def __init__(
+        self: "SitemapTree",
+        url: str | None = None,
+        max_concurrent_requests: int = 40,
+        *args: tuple,
+        **kwargs: dict,
+    ) -> None:
         """Initialize the sitemap tree."""
         super().__init__(*args, **kwargs)
 
         self.target = url
+        self.max_concurrent_requests = max_concurrent_requests
 
     def compose(self: "SitemapTree") -> ComposeResult:
         """Compose the sitemap tree."""
@@ -86,7 +93,7 @@ async def watch_target(self: "SitemapTree", value: str) -> None:
         if value:
             self.loading = True
 
-            parser = SitemapParser(value)
+            parser = SitemapParser(value, self.max_concurrent_requests)
             self.tree_data = await parser.parse()
 
             self.loading = False