added new changes which make it work with python, proper

Author: justrach

pathik/__init__.py

@@ -20,16 +20,77 @@
     from typing import List, Dict, Optional
 
     def crawl(urls: List[str], output_dir: Optional[str] = None, parallel: bool = True) -> Dict[str, Dict[str, str]]:
-        """Fallback implementation of crawl function"""
-        print("Using fallback crawl implementation")
-        # Basic implementation here
-        # ...
+        """Crawl function that uses the binary to crawl the given URLs"""
+        from .crawler import get_binary_path
+        
+        result = {}
+        temp_dir = None
+        
+        if not output_dir:
+            # Create temporary directory if no output_dir specified
+            temp_dir = tempfile.mkdtemp(prefix="pathik_")
+            output_dir = temp_dir
+            print(f"Created temporary directory: {temp_dir}")
+        
+        try:
+            binary_path = get_binary_path()
+            
+            # Prepare command
+            cmd = [binary_path, "-crawl", f"-outdir={output_dir}"]
+            if not parallel:
+                cmd.append("-parallel=false")
+            cmd.extend(urls)
+            
+            print(f"crawl() called with urls={urls}, output_dir={output_dir}, parallel={parallel}")
+            
+            # Run command
+            subprocess.run(cmd, check=True)
+            
+            # Collect results
+            for url in urls:
+                result[url] = {}
+                # TODO: Add result collection logic
+        except Exception as e:
+            print(f"Error in crawl function: {e}")
+            for url in urls:
+                if url not in result:
+                    result[url] = {"error": str(e)}
+        
+        return result
 
     def crawl_to_r2(urls: List[str], uuid_str: Optional[str] = None, parallel: bool = True) -> Dict[str, Dict[str, str]]:
-        """Fallback implementation of crawl_to_r2 function"""
-        print("Using fallback crawl_to_r2 implementation")
-        # Basic implementation here
-        # ...
+        """Upload to R2 function that uses the binary to upload the given URLs"""
+        from .crawler import get_binary_path
+        
+        result = {}
+        if not uuid_str:
+            uuid_str = str(uuid.uuid4())
+        
+        try:
+            binary_path = get_binary_path()
+            
+            # Prepare command
+            cmd = [binary_path, "-r2", f"-uuid={uuid_str}"]
+            if not parallel:
+                cmd.append("-parallel=false")
+            cmd.extend(urls)
+            
+            # Run command
+            subprocess.run(cmd, check=True)
+            
+            # Collect results
+            for url in urls:
+                result[url] = {
+                    "uuid": uuid_str,
+                    # TODO: Add more result details
+                }
+        except Exception as e:
+            print(f"Error in crawl_to_r2 function: {e}")
+            for url in urls:
+                if url not in result:
+                    result[url] = {"error": str(e)}
+        
+        return result
 
 # Export the functions
 __all__ = ["crawl", "crawl_to_r2", "__version__"] 

pathik/__pycache__/cli.cpython-312.pyc

@@ -7,12 +7,27 @@
 import os
 import json
 import subprocess
-from . import crawl, crawl_to_r2
-from .crawler import get_binary_path
+
+# Fix the import to use direct import instead of relative
+import pathik
+from pathik.crawler import get_binary_path
 
 def main():
     """Main entry point for the CLI"""
-    parser = argparse.ArgumentParser(description="Pathik - A fast web crawler with Python integration")
+    parser = argparse.ArgumentParser(
+        description="Pathik - A fast web crawler with Python integration",
+        epilog="""
+Note: This Python CLI uses subcommands (crawl, r2, kafka, version) rather than flags.
+For example:
+  pathik kafka https://example.com
+  pathik crawl -o ./output https://example.com
+
+If you prefer flag-style syntax, use the Go binary directly:
+  ./pathik -kafka https://example.com
+  ./pathik -crawl -outdir ./output https://example.com
+""",
+        formatter_class=argparse.RawDescriptionHelpFormatter
+    )
     subparsers = parser.add_subparsers(dest="command", help="Command to run")
 
     # Crawl command
@@ -35,19 +50,33 @@ def main():
     kafka_parser.add_argument("-t", "--topic", help="Kafka topic to stream to")
     kafka_parser.add_argument("-c", "--content", choices=["html", "markdown", "both"], default="both",
                             help="Content type to stream (html, markdown, or both)")
+    kafka_parser.add_argument("--session", help="Session ID to include with messages (for multi-user environments)")
 
     # Version command
     version_parser = subparsers.add_parser("version", help="Print version information")
 
-    args = parser.parse_args()
+    try:
+        args = parser.parse_args()
+    except SystemExit as e:
+        # Check if user might be using Go binary syntax with dashes
+        for i, arg in enumerate(sys.argv[1:]):
+            if arg.startswith('-') and not arg.startswith('--') and arg not in ['-o', '-s', '-u', '-b', '-t', '-c']:
+                print("\nError: It seems you're using Go binary syntax with the Python CLI.")
+                print("The Python CLI uses subcommands instead of flags:")
+                print("  ✅ Correct: pathik kafka https://example.com")
+                print("  ❌ Incorrect: pathik -kafka https://example.com")
+                print("\nAvailable subcommands: crawl, r2, kafka, version")
+                return 1
+        # If not caught by our check, let the original error propagate
+        return e.code
 
     if not args.command:
         parser.print_help()
         return 1
 
     try:
         if args.command == "crawl":
-            result = crawl(
+            result = pathik.crawl(
                 urls=args.urls, 
                 output_dir=args.outdir, 
                 parallel=not args.sequential
@@ -72,7 +101,7 @@ def main():
                 print(f"\nResults saved to: {results_file}")
 
         elif args.command == "r2":
-            result = crawl_to_r2(
+            result = pathik.crawl_to_r2(
                 urls=args.urls,
                 uuid_str=args.uuid,
                 parallel=not args.sequential
@@ -101,11 +130,17 @@ def main():
                 if args.content and args.content != "both":
                     cmd.extend(["-content", args.content])
 
+                # Add topic if specified
+                if args.topic:
+                    cmd.extend(["-topic", args.topic])
+                    
+                # Add session ID if provided
+                if args.session:
+                    cmd.extend(["-session", args.session])
+                
                 # Add Kafka-specific options if provided
                 if args.brokers:
                     os.environ["KAFKA_BROKERS"] = args.brokers
-                if args.topic:
-                    os.environ["KAFKA_TOPIC"] = args.topic
 
                 # Add URLs
                 cmd.extend(args.urls)
@@ -126,8 +161,7 @@ def main():
                 return 1
 
         elif args.command == "version":
-            from . import __version__  # Importing here to avoid circular imports
-            print(f"Pathik v{__version__}")
+            print(f"Pathik v{pathik.__version__}")
             return 0
 
         return 0