refactor: Update Firecrawl API parameters and default settings (#13082)

langgenius · Jan 29, 2025 · d0a2108 · d0a2108
1 parent d44882c
commit d0a2108
Show file tree

Hide file tree

Showing 4 changed files with 10 additions and 12 deletions.
diff --git a/api/core/rag/extractor/firecrawl/firecrawl_web_extractor.py b/api/core/rag/extractor/firecrawl/firecrawl_web_extractor.py
@@ -13,9 +13,10 @@ class FirecrawlWebExtractor(BaseExtractor):
         api_key: The API key for Firecrawl.
         base_url: The base URL for the Firecrawl API. Defaults to 'https://api.firecrawl.dev'.
         mode: The mode of operation. Defaults to 'scrape'. Options are 'crawl', 'scrape' and 'crawl_return_urls'.
+        only_main_content: Only return the main content of the page excluding headers, navs, footers, etc.
     """
 
-    def __init__(self, url: str, job_id: str, tenant_id: str, mode: str = "crawl", only_main_content: bool = False):
+    def __init__(self, url: str, job_id: str, tenant_id: str, mode: str = "crawl", only_main_content: bool = True):
         """Initialize with url, api_key, base_url and mode."""
         self._url = url
         self.job_id = job_id

diff --git a/api/services/auth/firecrawl/firecrawl.py b/api/services/auth/firecrawl/firecrawl.py
@@ -21,8 +21,8 @@ def validate_credentials(self):
         headers = self._prepare_headers()
         options = {
             "url": "https://example.com",
-            "excludes": [],
-            "includes": [],
+            "includePaths": [],
+            "excludePaths": [],
             "limit": 1,
             "scrapeOptions": {"onlyMainContent": True},
         }

diff --git a/api/services/website_service.py b/api/services/website_service.py
@@ -38,19 +38,17 @@ def crawl_url(cls, args: dict) -> dict:
             only_main_content = options.get("only_main_content", False)
             if not crawl_sub_pages:
                 params = {
-                    "includes": [],
-                    "excludes": [],
-                    "generateImgAltText": True,
+                    "includePaths": [],
+                    "excludePaths": [],
                     "limit": 1,
                     "scrapeOptions": {"onlyMainContent": only_main_content},
                 }
             else:
                 includes = options.get("includes").split(",") if options.get("includes") else []
                 excludes = options.get("excludes").split(",") if options.get("excludes") else []
                 params = {
-                    "includes": includes,
-                    "excludes": excludes,
-                    "generateImgAltText": True,
+                    "includePaths": includes,
+                    "excludePaths": excludes,
                     "limit": options.get("limit", 1),
                     "scrapeOptions": {"onlyMainContent": only_main_content},
                 }

diff --git a/api/tests/unit_tests/core/rag/extractor/firecrawl/test_firecrawl.py b/api/tests/unit_tests/core/rag/extractor/firecrawl/test_firecrawl.py
@@ -10,9 +10,8 @@ def test_firecrawl_web_extractor_crawl_mode(mocker):
     base_url = "https://api.firecrawl.dev"
     firecrawl_app = FirecrawlApp(api_key=api_key, base_url=base_url)
     params = {
-        "includes": [],
-        "excludes": [],
-        "generateImgAltText": True,
+        "includePaths": [],
+        "excludePaths": [],
         "maxDepth": 1,
         "limit": 1,
     }