add example and tests

Author: VinciGit00

examples/local_scraper_example.py

@@ -0,0 +1,61 @@
+from bs4 import BeautifulSoup
+import os
+
+def scrape_local_html(file_path):
+    """
+    Scrape content from a local HTML file.
+    
+    Args:
+        file_path (str): Path to the local HTML file
+        
+    Returns:
+        dict: Extracted data from the HTML file
+    """
+    # Check if file exists
+    if not os.path.exists(file_path):
+        raise FileNotFoundError(f"HTML file not found at: {file_path}")
+    
+    # Read the HTML file
+    with open(file_path, 'r', encoding='utf-8') as file:
+        html_content = file.read()
+    
+    # Parse HTML with BeautifulSoup
+    soup = BeautifulSoup(html_content, 'html.parser')
+    
+    # Example extraction - modify based on your HTML structure
+    data = {
+        'title': soup.title.string if soup.title else None,
+        'paragraphs': [p.text for p in soup.find_all('p')],
+        'links': [{'text': a.text, 'href': a.get('href')} for a in soup.find_all('a')],
+        'headers': [h.text for h in soup.find_all(['h1', 'h2', 'h3'])]
+    }
+    
+    return data
+
+def main():
+    # Example usage
+    try:
+        # Assuming you have a sample.html file in the same directory
+        result = scrape_local_html('sample.html')
+        
+        # Print extracted data
+        print("Title:", result['title'])
+        print("\nParagraphs:")
+        for p in result['paragraphs']:
+            print(f"- {p}")
+            
+        print("\nLinks:")
+        for link in result['links']:
+            print(f"- {link['text']}: {link['href']}")
+            
+        print("\nHeaders:")
+        for header in result['headers']:
+            print(f"- {header}")
+            
+    except FileNotFoundError as e:
+        print(f"Error: {e}")
+    except Exception as e:
+        print(f"An error occurred: {e}")
+
+if __name__ == "__main__":
+    main()

tests/local_scraper_test.py

@@ -0,0 +1,79 @@
+import unittest
+from unittest.mock import patch
+from scrapegraph_py.local_scraper import scrape_text
+from pydantic import BaseModel, Field
+import requests
+
+class TestSchema(BaseModel):
+    title: str = Field(description="The title")
+    content: str = Field(description="The content")
+
+class TestLocalScraper(unittest.TestCase):
+    
+    @patch('scrapegraph_py.local_scraper.requests.post')
+    def test_scrape_text_success(self, mock_post):
+        # Setup mock response
+        mock_post.return_value.status_code = 200
+        mock_post.return_value.text = '{"title": "Test", "content": "Content"}'
+        
+        # Test basic scraping without schema
+        response = scrape_text(
+            "test_api_key",
+            "Sample website text",
+            "Extract information"
+        )
+        self.assertEqual(response, '{"title": "Test", "content": "Content"}')
+
+    @patch('scrapegraph_py.local_scraper.requests.post')
+    def test_scrape_text_with_schema(self, mock_post):
+        # Setup mock response
+        mock_post.return_value.status_code = 200
+        mock_post.return_value.text = '{"title": "Test", "content": "Content"}'
+        
+        # Test scraping with schema
+        response = scrape_text(
+            "test_api_key",
+            "Sample website text",
+            "Extract information",
+            schema=TestSchema
+        )
+        self.assertEqual(response, '{"title": "Test", "content": "Content"}')
+
+    @patch('scrapegraph_py.local_scraper.requests.post')
+    def test_scrape_text_http_error(self, mock_post):
+        # Test HTTP error handling
+        mock_post.side_effect = requests.exceptions.HTTPError("404 Client Error")
+        response = scrape_text(
+            "test_api_key",
+            "Sample website text",
+            "Extract information"
+        )
+        self.assertIn("HTTP error occurred", response)
+
+    @patch('scrapegraph_py.local_scraper.requests.post')
+    def test_scrape_text_forbidden(self, mock_post):
+        # Test 403 forbidden error
+        mock_response = mock_post.return_value
+        mock_response.status_code = 403
+        mock_post.side_effect = requests.exceptions.HTTPError("403 Forbidden")
+        
+        response = scrape_text(
+            "test_api_key",
+            "Sample website text",
+            "Extract information"
+        )
+        self.assertIn("Access forbidden (403)", response)
+
+    @patch('scrapegraph_py.local_scraper.requests.post')
+    def test_scrape_text_general_error(self, mock_post):
+        # Test general request exception handling
+        mock_post.side_effect = requests.exceptions.RequestException("Connection error")
+        response = scrape_text(
+            "test_api_key",
+            "Sample website text",
+            "Extract information"
+        )
+        self.assertIn("An error occurred", response)
+
+if __name__ == '__main__':
+    unittest.main()