From 300cb3b210a09a2b68ba86deb61ee6128ef93af3 Mon Sep 17 00:00:00 2001 From: AnshSinghal Date: Wed, 23 Apr 2025 16:50:06 +0000 Subject: [PATCH 01/12] phunter --- .../0156_analyzer_config_phunter.py | 125 ++++++++++++++++++ .../observable_analyzers/phunter.py | 78 +++++++++++ integrations/phunter/Dockerfile | 24 ++++ integrations/phunter/app.py | 115 ++++++++++++++++ integrations/phunter/compose-tests.yml | 6 + integrations/phunter/compose.yml | 11 ++ integrations/phunter/requirements.txt | 2 + requirements/project-requirements.txt | 1 + start | 7 +- 9 files changed, 368 insertions(+), 1 deletion(-) create mode 100644 api_app/analyzers_manager/migrations/0156_analyzer_config_phunter.py create mode 100644 api_app/analyzers_manager/observable_analyzers/phunter.py create mode 100644 integrations/phunter/Dockerfile create mode 100644 integrations/phunter/app.py create mode 100644 integrations/phunter/compose-tests.yml create mode 100644 integrations/phunter/compose.yml create mode 100644 integrations/phunter/requirements.txt diff --git a/api_app/analyzers_manager/migrations/0156_analyzer_config_phunter.py b/api_app/analyzers_manager/migrations/0156_analyzer_config_phunter.py new file mode 100644 index 0000000000..65703989ff --- /dev/null +++ b/api_app/analyzers_manager/migrations/0156_analyzer_config_phunter.py @@ -0,0 +1,125 @@ +from django.db import migrations +from django.db.models.fields.related_descriptors import ( + ForwardManyToOneDescriptor, + ForwardOneToOneDescriptor, + ManyToManyDescriptor, + ReverseManyToOneDescriptor, + ReverseOneToOneDescriptor, +) + +plugin = { + "python_module": { + "health_check_schedule": None, + "update_schedule": None, + "module": "phunter.PhunterAnalyzer", + "base_path": "api_app.analyzers_manager.observable_analyzers", + }, + "name": "Phunter", + "description": "[Phunter Analyzer](https://github.com/N0rz3/Phunter) is an OSINT tool for finding information about a phone number.", + "disabled": False, + "soft_time_limit": 60, + "routing_key": "default", + "health_check_status": True, + "type": "observable", + "docker_based": True, + "maximum_tlp": "RED", + "observable_supported": ["generic"], + "supported_filetypes": [], + "run_hash": False, + "run_hash_type": "", + "not_supported_filetypes": [], + "mapping_data_model": {}, + "model": "analyzers_manager.AnalyzerConfig", +} + +params = [] + +values = [] + + +def _get_real_obj(Model, field, value): + def _get_obj(Model, other_model, value): + if isinstance(value, dict): + real_vals = {} + for key, real_val in value.items(): + real_vals[key] = _get_real_obj(other_model, key, real_val) + value = other_model.objects.get_or_create(**real_vals)[0] + # it is just the primary key serialized + else: + if isinstance(value, int): + if Model.__name__ == "PluginConfig": + value = other_model.objects.get(name=plugin["name"]) + else: + value = other_model.objects.get(pk=value) + else: + value = other_model.objects.get(name=value) + return value + + if ( + type(getattr(Model, field)) + in [ + ForwardManyToOneDescriptor, + ReverseManyToOneDescriptor, + ReverseOneToOneDescriptor, + ForwardOneToOneDescriptor, + ] + and value + ): + other_model = getattr(Model, field).get_queryset().model + value = _get_obj(Model, other_model, value) + elif type(getattr(Model, field)) in [ManyToManyDescriptor] and value: + other_model = getattr(Model, field).rel.model + value = [_get_obj(Model, other_model, val) for val in value] + return value + + +def _create_object(Model, data): + mtm, no_mtm = {}, {} + for field, value in data.items(): + value = _get_real_obj(Model, field, value) + if type(getattr(Model, field)) is ManyToManyDescriptor: + mtm[field] = value + else: + no_mtm[field] = value + try: + o = Model.objects.get(**no_mtm) + except Model.DoesNotExist: + o = Model(**no_mtm) + o.full_clean() + o.save() + for field, value in mtm.items(): + attribute = getattr(o, field) + if value is not None: + attribute.set(value) + return False + return True + + +def migrate(apps, schema_editor): + Parameter = apps.get_model("api_app", "Parameter") + PluginConfig = apps.get_model("api_app", "PluginConfig") + python_path = plugin.pop("model") + Model = apps.get_model(*python_path.split(".")) + if not Model.objects.filter(name=plugin["name"]).exists(): + exists = _create_object(Model, plugin) + if not exists: + for param in params: + _create_object(Parameter, param) + for value in values: + _create_object(PluginConfig, value) + + +def reverse_migrate(apps, schema_editor): + python_path = plugin.pop("model") + Model = apps.get_model(*python_path.split(".")) + Model.objects.get(name=plugin["name"]).delete() + + +class Migration(migrations.Migration): + atomic = False + dependencies = [ + ("api_app", "0071_delete_last_elastic_report"), + ("analyzers_manager", "0155_analyzer_config_debloat"), + ] + + operations = [migrations.RunPython(migrate, reverse_migrate)] diff --git a/api_app/analyzers_manager/observable_analyzers/phunter.py b/api_app/analyzers_manager/observable_analyzers/phunter.py new file mode 100644 index 0000000000..fa8323ea47 --- /dev/null +++ b/api_app/analyzers_manager/observable_analyzers/phunter.py @@ -0,0 +1,78 @@ +import logging + +import phonenumbers +import requests + +from api_app.analyzers_manager.classes import DockerBasedAnalyzer, ObservableAnalyzer +from api_app.analyzers_manager.exceptions import AnalyzerRunException +from tests.mock_utils import MockUpResponse + +logging.basicConfig(level=logging.DEBUG) +logger = logging.getLogger(__name__) + + +class PhunterAnalyzer(ObservableAnalyzer, DockerBasedAnalyzer): + """ + PhunterAnalyzer is a class that analyzes phone numbers using the Phunter API. + """ + + name: str = "Phunter" + url: str = "http://phunter:5000/analyze" + max_tries: int = 1 + poll_distance: int = 0 + + def run(self): + try: + parsed_number = phonenumbers.parse(self.observable_name) + if not phonenumbers.is_valid_number(parsed_number): + logger.error(f"Invalid phone number: {self.observable_name}") + raise AnalyzerRunException("Invalid phone number.") + except phonenumbers.phonenumberutil.NumberParseException: + logger.error(f"Phone number parsing failed for: {self.observable_name}") + raise AnalyzerRunException("Invalid phone number format") + + req_data = {"phone_number": self.observable_name} + logger.info(f"Sending {self.name} scan request: {req_data} to {self.url}") + try: + response = requests.post(self.url, json=req_data) + logger.debug( + f"[{self.name}] Raw response: {response.status_code} - {response.text}" + ) + response.raise_for_status() + try: + result = response.json() + logger.info(f"[{self.name}] Scan successful. Result: {result}") + return result + except ValueError: + logger.error( + f"[{self.name}] JSON parsing failed. Response content: {response.text}" + ) + raise AnalyzerRunException("Invalid JSON returned by Phunter API") + except requests.RequestException as e: + logger.error(f"[{self.name}] HTTP request failed: {e}", exc_info=True) + raise AnalyzerRunException("Failed to connect to Phunter API") + + @classmethod + def update(self): + pass + + @staticmethod + def mocked_docker_analyzer_post(*args, **kwargs): + mock_response = { + "success": True, + "report": { + "valid": "yes", + "views": "9", + "carrier": "Vodafone", + "location": "India", + "operator": "Vodafone", + "possible": "yes", + "line_type": "FIXED LINE OR MOBILE", + "local_time": "21:34:45", + "spam_status": "Not spammer", + "phone_number": "+918929554991", + "national_format": "089295 54991", + "international_format": "+91 89295 54991", + }, + } + return MockUpResponse(mock_response, 200) diff --git a/integrations/phunter/Dockerfile b/integrations/phunter/Dockerfile new file mode 100644 index 0000000000..1fefc96a30 --- /dev/null +++ b/integrations/phunter/Dockerfile @@ -0,0 +1,24 @@ +FROM python:3.12-slim + +# Install dependencies +RUN apt-get update && apt-get install -y git + +# Clone Phunter +RUN git clone https://github.com/N0rz3/Phunter.git /app/Phunter + +# Set working directory +WORKDIR /app + +# Copy requirements file and app.py to the working directory +COPY requirements.txt app.py ./ + +# Upgrade pip and install Python packages +RUN pip install --no-cache-dir --upgrade pip && \ + pip install --no-cache-dir -r requirements.txt && \ + pip install --no-cache-dir -r /app/Phunter/requirements.txt + +# Expose port +EXPOSE 5000 + +# Run the app +CMD ["python", "app.py"] \ No newline at end of file diff --git a/integrations/phunter/app.py b/integrations/phunter/app.py new file mode 100644 index 0000000000..21b014beed --- /dev/null +++ b/integrations/phunter/app.py @@ -0,0 +1,115 @@ +import logging +import re +import subprocess + +import phonenumbers +from flask import Flask, jsonify, request + +# Logging Configuration +logging.basicConfig(level=logging.DEBUG) +logger = logging.getLogger(__name__) + +app = Flask(__name__) + + +# Remove ANSI codes from script output +def strip_ansi_codes(text): + return re.sub(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])", "", text) + + +# Extract structured data from Phunter CLI output +def parse_phunter_output(output): + result = {} + lines = output.splitlines() + + for line in lines: + line = line.strip() + + if "Phone number:" in line: + result["phone_number"] = line.partition(":")[2].strip() + + elif "Possible:" in line: + result["possible"] = "yes" if "✔" in line else "no" + + elif "Valid:" in line: + result["valid"] = "yes" if "✔" in line else "no" + + elif "Operator:" in line: + result["operator"] = line.partition(":")[2].strip() + + elif "Possible location:" in line: + result["location"] = line.partition(":")[2].strip() + + elif "Carrier:" in line: + result["carrier"] = line.partition(":")[2].strip() + + elif "Line Type:" in line: + result["line_type"] = line.partition(":")[2].strip() + + elif "International:" in line: + result["international_format"] = line.partition(":")[2].strip() + + elif "National:" in line: + result["national_format"] = line.partition(":")[2].strip() + + elif "Local Time:" in line: + result["local_time"] = line.partition(":")[2].strip() + + elif "Views count:" in line: + result["views"] = line.partition(":")[2].strip() + + elif "Not spammer" in line: + result["spam_status"] = "Not spammer" + + return result + + +@app.route("/analyze", methods=["POST"]) +def analyze(): + data = request.get_json() + phone_number = data.get("phone_number") + + logger.info(f"Received request to analyze phone number: {phone_number}") + + if not phone_number: + logger.warning("Phone number missing from request") + return jsonify({"error": "No phone number provided"}), 400 + + try: + parsed_number = phonenumbers.parse(phone_number) + if not phonenumbers.is_valid_number(parsed_number): + logger.warning(f"Phone number is not valid: {phone_number}") + return jsonify({"error": "Invalid phone number"}), 400 + except phonenumbers.phonenumberutil.NumberParseException as e: + logger.warning(f"Number parsing failed: {e}") + return jsonify({"error": "Invalid phone number format"}), 400 + + try: + logger.info(f"Executing Phunter on: {phone_number}") + result = subprocess.run( + ["python3", "phunter.py", "-t", phone_number], + capture_output=True, + text=True, + check=True, + cwd="/app/Phunter", # Update if path is different + ) + raw_output = result.stdout + logger.debug(f"Raw Phunter output:\n{raw_output}") + + clean_output = strip_ansi_codes(raw_output) + parsed_output = parse_phunter_output(clean_output) + + logger.info("Phunter analysis completed successfully.") + return jsonify(parsed_output) + + except subprocess.CalledProcessError as e: + logger.error(f"Phunter execution failed: {e.stderr}") + return jsonify({"error": "Phunter execution failed", "details": e.stderr}), 500 + except Exception as e: + logger.exception("Unexpected error during Phunter analysis") + return jsonify({"error": "Internal server error", "details": str(e)}), 500 + + +if __name__ == "__main__": + logger.info("Starting Phunter Flask API...") + app.run(host="0.0.0.0", port=5000) diff --git a/integrations/phunter/compose-tests.yml b/integrations/phunter/compose-tests.yml new file mode 100644 index 0000000000..de900177dd --- /dev/null +++ b/integrations/phunter/compose-tests.yml @@ -0,0 +1,6 @@ +services: + phunter: + build: + context: ../integrations/phunter + dockerfile: Dockerfile + image: intelowlproject/intelowl_phunter:test \ No newline at end of file diff --git a/integrations/phunter/compose.yml b/integrations/phunter/compose.yml new file mode 100644 index 0000000000..2b9889b636 --- /dev/null +++ b/integrations/phunter/compose.yml @@ -0,0 +1,11 @@ +services: + phunter: + image: intelowlproject/phunter:${REACT_APP_INTELOWL_VERSION} + container_name: phunter + restart: unless-stopped + expose: + - "5000" + volumes: + - generic_logs:/var/log/intel_owl + depends_on: + - uwsgi \ No newline at end of file diff --git a/integrations/phunter/requirements.txt b/integrations/phunter/requirements.txt new file mode 100644 index 0000000000..ba426941c2 --- /dev/null +++ b/integrations/phunter/requirements.txt @@ -0,0 +1,2 @@ +flask==3.1.0 +phonenumbers==9.0.3 \ No newline at end of file diff --git a/requirements/project-requirements.txt b/requirements/project-requirements.txt index 222877bf59..a36322237e 100644 --- a/requirements/project-requirements.txt +++ b/requirements/project-requirements.txt @@ -88,6 +88,7 @@ docxpy==0.8.5 pylnk3==0.4.2 androguard==3.4.0a1 # version >=4.x of androguard raises a dependency conflict with quark-engine==25.1.1 wad==0.4.6 +phonenumbers==9.0.3 # httpx required for HTTP/2 support (Mullvad DNS rejects HTTP/1.1 with protocol errors) httpx[http2]==0.28.1 diff --git a/start b/start index f6fb3e1de1..186bedaf02 100755 --- a/start +++ b/start @@ -38,6 +38,7 @@ print_help () { echo " --pcap_analyzers Uses the integrations/pcap_analyzers/compose.yml" echo " file." echo " --bbot Uses the integrations/bbot/compose.yml" + echo " --phunter Uses the integrations/phunter/compose.yml" echo " --multi_queue Uses the multiqueue.override.yml compose file." echo " --nfs Uses the nfs.override.yml compose file." echo " --traefik_prod Uses the traefik.yml and traefik_prod.yml compose file." @@ -100,7 +101,7 @@ check_parameters "$@" && shift 2 current_version=${REACT_APP_INTELOWL_VERSION/"v"/""} -docker_analyzers=("pcap_analyzers" "tor_analyzers" "malware_tools_analyzers" "thug" "cyberchef" "phoneinfoga" "phishing_analyzers" "nuclei_analyzer" "bbot") +docker_analyzers=("pcap_analyzers" "tor_analyzers" "malware_tools_analyzers" "thug" "cyberchef" "phoneinfoga" "phishing_analyzers" "nuclei_analyzer" "bbot" "phunter") for value in "${docker_analyzers[@]}"; do @@ -182,6 +183,10 @@ while [[ $# -gt 0 ]]; do analyzers["bbot"]=true shift 1 ;; + --phunter) + analyzers["phunter"]=true + shift 1 + ;; --multi_queue) params["multi_queue"]=true shift 1 From d42f9508a14c09c9136620678061deec9bf56a06 Mon Sep 17 00:00:00 2001 From: AnshSinghal Date: Wed, 23 Apr 2025 17:11:13 +0000 Subject: [PATCH 02/12] fixed DeepSource errors --- integrations/phunter/Dockerfile | 2 +- integrations/phunter/app.py | 67 +++++++++++++++------------------ 2 files changed, 32 insertions(+), 37 deletions(-) diff --git a/integrations/phunter/Dockerfile b/integrations/phunter/Dockerfile index 1fefc96a30..81cc613ffc 100644 --- a/integrations/phunter/Dockerfile +++ b/integrations/phunter/Dockerfile @@ -1,7 +1,7 @@ FROM python:3.12-slim # Install dependencies -RUN apt-get update && apt-get install -y git +RUN apt-get update && apt-get install -y --no-install-recommends git # Clone Phunter RUN git clone https://github.com/N0rz3/Phunter.git /app/Phunter diff --git a/integrations/phunter/app.py b/integrations/phunter/app.py index 21b014beed..8551d13e12 100644 --- a/integrations/phunter/app.py +++ b/integrations/phunter/app.py @@ -20,46 +20,40 @@ def strip_ansi_codes(text): # Extract structured data from Phunter CLI output def parse_phunter_output(output): result = {} + key_mapping = { + "phone number:": "phone_number", + "possible:": "possible", + "valid:": "valid", + "operator:": "operator", + "possible location:": "location", + "location:": "location", + "carrier:": "carrier", + "line type:": "line_type", + "international:": "international_format", + "national:": "national_format", + "local time:": "local_time", + "views count:": "views", + } + lines = output.splitlines() for line in lines: - line = line.strip() - - if "Phone number:" in line: - result["phone_number"] = line.partition(":")[2].strip() - - elif "Possible:" in line: - result["possible"] = "yes" if "✔" in line else "no" - - elif "Valid:" in line: - result["valid"] = "yes" if "✔" in line else "no" - - elif "Operator:" in line: - result["operator"] = line.partition(":")[2].strip() - - elif "Possible location:" in line: - result["location"] = line.partition(":")[2].strip() - - elif "Carrier:" in line: - result["carrier"] = line.partition(":")[2].strip() - - elif "Line Type:" in line: - result["line_type"] = line.partition(":")[2].strip() - - elif "International:" in line: - result["international_format"] = line.partition(":")[2].strip() - - elif "National:" in line: - result["national_format"] = line.partition(":")[2].strip() - - elif "Local Time:" in line: - result["local_time"] = line.partition(":")[2].strip() - - elif "Views count:" in line: - result["views"] = line.partition(":")[2].strip() + line = line.strip().lower() - elif "Not spammer" in line: + # Special case: spam status + if "not spammer" in line: result["spam_status"] = "Not spammer" + continue + + for keyword, key_name in key_mapping.items(): + if keyword in line: + value = line.partition(":")[2].strip() + # Special handling for booleans + if key_name in ("possible", "valid"): + result[key_name] = "yes" if "✔" in value else "no" + else: + result[key_name] = value + break # No need to check other keywords once matched return result @@ -86,8 +80,9 @@ def analyze(): try: logger.info(f"Executing Phunter on: {phone_number}") + command = ["python3", "phunter.py", "-t", phone_number] result = subprocess.run( - ["python3", "phunter.py", "-t", phone_number], + command, capture_output=True, text=True, check=True, From 4f9a6015232f3865f9f17cca3b12c3615579d686 Mon Sep 17 00:00:00 2001 From: AnshSinghal Date: Thu, 24 Apr 2025 07:01:26 +0000 Subject: [PATCH 03/12] fixed errors related to docker run method --- api_app/analyzers_manager/classes.py | 4 +-- .../observable_analyzers/phunter.py | 32 ++++++------------- integrations/phunter/app.py | 16 +++++++++- 3 files changed, 26 insertions(+), 26 deletions(-) diff --git a/api_app/analyzers_manager/classes.py b/api_app/analyzers_manager/classes.py index c3d3e688ee..c4619ba4cc 100644 --- a/api_app/analyzers_manager/classes.py +++ b/api_app/analyzers_manager/classes.py @@ -433,8 +433,8 @@ def _docker_run( self._raise_container_not_running() # step #2: raise AnalyzerRunException in case of error - # Modified to support synchronous analyzer BBOT that return results directly in the initial response, avoiding unnecessary polling. - if analyzer_name == "BBOT_Analyzer": + # Modified to support synchronous analyzers that return results directly in the initial response, avoiding unnecessary polling. + if analyzer_name in ["BBOT_Analyzer", "Phunter"]: report = resp1.json().get("report", None) err = resp1.json().get("error", None) else: diff --git a/api_app/analyzers_manager/observable_analyzers/phunter.py b/api_app/analyzers_manager/observable_analyzers/phunter.py index fa8323ea47..15a24b140f 100644 --- a/api_app/analyzers_manager/observable_analyzers/phunter.py +++ b/api_app/analyzers_manager/observable_analyzers/phunter.py @@ -1,7 +1,6 @@ import logging import phonenumbers -import requests from api_app.analyzers_manager.classes import DockerBasedAnalyzer, ObservableAnalyzer from api_app.analyzers_manager.exceptions import AnalyzerRunException @@ -12,10 +11,6 @@ class PhunterAnalyzer(ObservableAnalyzer, DockerBasedAnalyzer): - """ - PhunterAnalyzer is a class that analyzes phone numbers using the Phunter API. - """ - name: str = "Phunter" url: str = "http://phunter:5000/analyze" max_tries: int = 1 @@ -26,7 +21,7 @@ def run(self): parsed_number = phonenumbers.parse(self.observable_name) if not phonenumbers.is_valid_number(parsed_number): logger.error(f"Invalid phone number: {self.observable_name}") - raise AnalyzerRunException("Invalid phone number.") + return {"success": False, "error": "Invalid phone number"} except phonenumbers.phonenumberutil.NumberParseException: logger.error(f"Phone number parsing failed for: {self.observable_name}") raise AnalyzerRunException("Invalid phone number format") @@ -34,23 +29,14 @@ def run(self): req_data = {"phone_number": self.observable_name} logger.info(f"Sending {self.name} scan request: {req_data} to {self.url}") try: - response = requests.post(self.url, json=req_data) - logger.debug( - f"[{self.name}] Raw response: {response.status_code} - {response.text}" - ) - response.raise_for_status() - try: - result = response.json() - logger.info(f"[{self.name}] Scan successful. Result: {result}") - return result - except ValueError: - logger.error( - f"[{self.name}] JSON parsing failed. Response content: {response.text}" - ) - raise AnalyzerRunException("Invalid JSON returned by Phunter API") - except requests.RequestException as e: - logger.error(f"[{self.name}] HTTP request failed: {e}", exc_info=True) - raise AnalyzerRunException("Failed to connect to Phunter API") + response = self._docker_run(req_data, analyzer_name=self.name) + logger.info(f"[{self.name}] Scan successful by Phunter. Result: {response}") + print(response) + return response + + except Exception as e: + logger.error(f"[{self.name}] Request failed: {str(e)}", exc_info=True) + raise AnalyzerRunException(f"Failed to connect to Phunter API: {str(e)}") @classmethod def update(self): diff --git a/integrations/phunter/app.py b/integrations/phunter/app.py index 8551d13e12..19ed25a57c 100644 --- a/integrations/phunter/app.py +++ b/integrations/phunter/app.py @@ -95,7 +95,21 @@ def analyze(): parsed_output = parse_phunter_output(clean_output) logger.info("Phunter analysis completed successfully.") - return jsonify(parsed_output) + logger.info( + { + "success": True, + "report": parsed_output, + } + ) + return ( + jsonify( + { + "success": True, + "report": parsed_output, + } + ), + 200, + ) except subprocess.CalledProcessError as e: logger.error(f"Phunter execution failed: {e.stderr}") From 373e875cc913a589137e2afe62a7b379f78801e8 Mon Sep 17 00:00:00 2001 From: AnshSinghal Date: Thu, 24 Apr 2025 07:21:17 +0000 Subject: [PATCH 04/12] fixed errors with sensitive data leaks --- .../observable_analyzers/phunter.py | 15 +++-- integrations/phunter/app.py | 56 +++++++++---------- 2 files changed, 36 insertions(+), 35 deletions(-) diff --git a/api_app/analyzers_manager/observable_analyzers/phunter.py b/api_app/analyzers_manager/observable_analyzers/phunter.py index 15a24b140f..764e4ac806 100644 --- a/api_app/analyzers_manager/observable_analyzers/phunter.py +++ b/api_app/analyzers_manager/observable_analyzers/phunter.py @@ -1,6 +1,7 @@ import logging import phonenumbers +import requests from api_app.analyzers_manager.classes import DockerBasedAnalyzer, ObservableAnalyzer from api_app.analyzers_manager.exceptions import AnalyzerRunException @@ -28,15 +29,21 @@ def run(self): req_data = {"phone_number": self.observable_name} logger.info(f"Sending {self.name} scan request: {req_data} to {self.url}") + try: response = self._docker_run(req_data, analyzer_name=self.name) logger.info(f"[{self.name}] Scan successful by Phunter. Result: {response}") - print(response) return response - except Exception as e: - logger.error(f"[{self.name}] Request failed: {str(e)}", exc_info=True) - raise AnalyzerRunException(f"Failed to connect to Phunter API: {str(e)}") + except requests.exceptions.RequestException as e: + logger.error( + f"[{self.name}] Request failed due to network issue: {e}", exc_info=True + ) + raise AnalyzerRunException(f"Request error to Phunter API: {e}") + + except ValueError as e: + logger.error(f"[{self.name}] Invalid response format: {e}", exc_info=True) + raise AnalyzerRunException(f"Invalid response format from Phunter API: {e}") @classmethod def update(self): diff --git a/integrations/phunter/app.py b/integrations/phunter/app.py index 19ed25a57c..90dfa9f613 100644 --- a/integrations/phunter/app.py +++ b/integrations/phunter/app.py @@ -12,13 +12,13 @@ app = Flask(__name__) -# Remove ANSI codes from script output def strip_ansi_codes(text): + """Remove ANSI escape codes from terminal output""" return re.sub(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])", "", text) -# Extract structured data from Phunter CLI output def parse_phunter_output(output): + """Parse output from Phunter CLI and convert to structured JSON""" result = {} key_mapping = { "phone number:": "phone_number", @@ -40,20 +40,18 @@ def parse_phunter_output(output): for line in lines: line = line.strip().lower() - # Special case: spam status if "not spammer" in line: result["spam_status"] = "Not spammer" continue - for keyword, key_name in key_mapping.items(): + for keyword, key in key_mapping.items(): if keyword in line: value = line.partition(":")[2].strip() - # Special handling for booleans - if key_name in ("possible", "valid"): - result[key_name] = "yes" if "✔" in value else "no" + if key in ("possible", "valid"): + result[key] = "yes" if "✔" in value else "no" else: - result[key_name] = value - break # No need to check other keywords once matched + result[key] = value + break return result @@ -63,44 +61,43 @@ def analyze(): data = request.get_json() phone_number = data.get("phone_number") - logger.info(f"Received request to analyze phone number: {phone_number}") + logger.info("Received analysis request") if not phone_number: - logger.warning("Phone number missing from request") + logger.warning("No phone number provided in request") return jsonify({"error": "No phone number provided"}), 400 try: parsed_number = phonenumbers.parse(phone_number) if not phonenumbers.is_valid_number(parsed_number): - logger.warning(f"Phone number is not valid: {phone_number}") + logger.warning("Invalid phone number") return jsonify({"error": "Invalid phone number"}), 400 - except phonenumbers.phonenumberutil.NumberParseException as e: - logger.warning(f"Number parsing failed: {e}") + + formatted_number = phonenumbers.format_number( + parsed_number, phonenumbers.PhoneNumberFormat.E164 + ) + + except phonenumbers.phonenumberutil.NumberParseException: + logger.warning("Phone number parsing failed") return jsonify({"error": "Invalid phone number format"}), 400 try: - logger.info(f"Executing Phunter on: {phone_number}") - command = ["python3", "phunter.py", "-t", phone_number] + logger.info("Executing Phunter CLI tool") + command = ["python3", "phunter.py", "-t", formatted_number] result = subprocess.run( command, capture_output=True, text=True, check=True, - cwd="/app/Phunter", # Update if path is different + cwd="/app/Phunter", ) - raw_output = result.stdout - logger.debug(f"Raw Phunter output:\n{raw_output}") + raw_output = result.stdout clean_output = strip_ansi_codes(raw_output) parsed_output = parse_phunter_output(clean_output) - logger.info("Phunter analysis completed successfully.") - logger.info( - { - "success": True, - "report": parsed_output, - } - ) + logger.info("Phunter analysis completed") + return ( jsonify( { @@ -112,11 +109,8 @@ def analyze(): ) except subprocess.CalledProcessError as e: - logger.error(f"Phunter execution failed: {e.stderr}") - return jsonify({"error": "Phunter execution failed", "details": e.stderr}), 500 - except Exception as e: - logger.exception("Unexpected error during Phunter analysis") - return jsonify({"error": "Internal server error", "details": str(e)}), 500 + logger.error(f"Phunter CLI failed: {e.stderr}") + return jsonify({"error": "Phunter execution failed"}), 500 if __name__ == "__main__": From 4aa4a0620e655c44c0e19f9698f937833d00b33f Mon Sep 17 00:00:00 2001 From: AnshSinghal Date: Thu, 24 Apr 2025 10:05:03 +0000 Subject: [PATCH 05/12] fixed wrong number issue --- api_app/analyzers_manager/observable_analyzers/phunter.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/api_app/analyzers_manager/observable_analyzers/phunter.py b/api_app/analyzers_manager/observable_analyzers/phunter.py index 764e4ac806..8233e0734c 100644 --- a/api_app/analyzers_manager/observable_analyzers/phunter.py +++ b/api_app/analyzers_manager/observable_analyzers/phunter.py @@ -19,13 +19,10 @@ class PhunterAnalyzer(ObservableAnalyzer, DockerBasedAnalyzer): def run(self): try: - parsed_number = phonenumbers.parse(self.observable_name) - if not phonenumbers.is_valid_number(parsed_number): - logger.error(f"Invalid phone number: {self.observable_name}") - return {"success": False, "error": "Invalid phone number"} + phonenumbers.parse(self.observable_name) except phonenumbers.phonenumberutil.NumberParseException: logger.error(f"Phone number parsing failed for: {self.observable_name}") - raise AnalyzerRunException("Invalid phone number format") + return {"success": False, "error": "Invalid phone number"} req_data = {"phone_number": self.observable_name} logger.info(f"Sending {self.name} scan request: {req_data} to {self.url}") From fb9084f5c471b6a59497c929a2f32daa207befda Mon Sep 17 00:00:00 2001 From: AnshSinghal Date: Tue, 29 Apr 2025 14:25:07 +0000 Subject: [PATCH 06/12] fixed some minor bugs in phunter --- api_app/analyzers_manager/classes.py | 8 ++++++-- .../observable_analyzers/bbot.py | 4 +++- .../observable_analyzers/phunter.py | 20 +++++++++---------- integrations/phunter/Dockerfile | 2 +- integrations/phunter/app.py | 5 ++--- integrations/phunter/compose.yml | 6 +++--- 6 files changed, 25 insertions(+), 20 deletions(-) diff --git a/api_app/analyzers_manager/classes.py b/api_app/analyzers_manager/classes.py index c4619ba4cc..d9a6dd7863 100644 --- a/api_app/analyzers_manager/classes.py +++ b/api_app/analyzers_manager/classes.py @@ -399,7 +399,11 @@ def _raise_container_not_running(self) -> None: ) def _docker_run( - self, req_data: dict, req_files: dict = None, analyzer_name: str = None + self, + req_data: dict, + req_files: dict = None, + analyzer_name: str = None, + avoid_polling: bool = False, ) -> dict: """ Helper function that takes of care of requesting new analysis, @@ -434,7 +438,7 @@ def _docker_run( # step #2: raise AnalyzerRunException in case of error # Modified to support synchronous analyzers that return results directly in the initial response, avoiding unnecessary polling. - if analyzer_name in ["BBOT_Analyzer", "Phunter"]: + if avoid_polling: report = resp1.json().get("report", None) err = resp1.json().get("error", None) else: diff --git a/api_app/analyzers_manager/observable_analyzers/bbot.py b/api_app/analyzers_manager/observable_analyzers/bbot.py index 779d502eb9..5ac0b5524c 100644 --- a/api_app/analyzers_manager/observable_analyzers/bbot.py +++ b/api_app/analyzers_manager/observable_analyzers/bbot.py @@ -54,7 +54,9 @@ def run(self): logger.info(f"Sending {self.name} scan request: {req_data} to {self.url}") try: - report = self._docker_run(req_data, analyzer_name=self.name) + report = self._docker_run( + req_data, analyzer_name=self.name, avoid_polling=True + ) logger.info(f"BBOT scan completed successfully with report: {report}") return report except requests.RequestException as e: diff --git a/api_app/analyzers_manager/observable_analyzers/phunter.py b/api_app/analyzers_manager/observable_analyzers/phunter.py index 8233e0734c..555254b85a 100644 --- a/api_app/analyzers_manager/observable_analyzers/phunter.py +++ b/api_app/analyzers_manager/observable_analyzers/phunter.py @@ -13,7 +13,7 @@ class PhunterAnalyzer(ObservableAnalyzer, DockerBasedAnalyzer): name: str = "Phunter" - url: str = "http://phunter:5000/analyze" + url: str = "http://phunter:5612/analyze" max_tries: int = 1 poll_distance: int = 0 @@ -28,19 +28,19 @@ def run(self): logger.info(f"Sending {self.name} scan request: {req_data} to {self.url}") try: - response = self._docker_run(req_data, analyzer_name=self.name) + response = self._docker_run( + req_data, analyzer_name=self.name, avoid_polling=True + ) logger.info(f"[{self.name}] Scan successful by Phunter. Result: {response}") return response except requests.exceptions.RequestException as e: - logger.error( - f"[{self.name}] Request failed due to network issue: {e}", exc_info=True + raise AnalyzerRunException( + f"[{self.name}] Request failed due to network issue: {e}" ) - raise AnalyzerRunException(f"Request error to Phunter API: {e}") except ValueError as e: - logger.error(f"[{self.name}] Invalid response format: {e}", exc_info=True) - raise AnalyzerRunException(f"Invalid response format from Phunter API: {e}") + raise AnalyzerRunException(f"[{self.name}] Invalid response format: {e}") @classmethod def update(self): @@ -60,9 +60,9 @@ def mocked_docker_analyzer_post(*args, **kwargs): "line_type": "FIXED LINE OR MOBILE", "local_time": "21:34:45", "spam_status": "Not spammer", - "phone_number": "+918929554991", - "national_format": "089295 54991", - "international_format": "+91 89295 54991", + "phone_number": "+911234567890", + "national_format": "01234567890", + "international_format": "+91 1234567890", }, } return MockUpResponse(mock_response, 200) diff --git a/integrations/phunter/Dockerfile b/integrations/phunter/Dockerfile index 81cc613ffc..4221db03bb 100644 --- a/integrations/phunter/Dockerfile +++ b/integrations/phunter/Dockerfile @@ -18,7 +18,7 @@ RUN pip install --no-cache-dir --upgrade pip && \ pip install --no-cache-dir -r /app/Phunter/requirements.txt # Expose port -EXPOSE 5000 +EXPOSE 5612 # Run the app CMD ["python", "app.py"] \ No newline at end of file diff --git a/integrations/phunter/app.py b/integrations/phunter/app.py index 90dfa9f613..86ede20123 100644 --- a/integrations/phunter/app.py +++ b/integrations/phunter/app.py @@ -109,10 +109,9 @@ def analyze(): ) except subprocess.CalledProcessError as e: - logger.error(f"Phunter CLI failed: {e.stderr}") - return jsonify({"error": "Phunter execution failed"}), 500 + return jsonify({"error": f"Phunter execution failed with error {e}"}), 500 if __name__ == "__main__": logger.info("Starting Phunter Flask API...") - app.run(host="0.0.0.0", port=5000) + app.run(host="0.0.0.0", port=5612) diff --git a/integrations/phunter/compose.yml b/integrations/phunter/compose.yml index 2b9889b636..b4bcba1667 100644 --- a/integrations/phunter/compose.yml +++ b/integrations/phunter/compose.yml @@ -1,10 +1,10 @@ services: phunter: - image: intelowlproject/phunter:${REACT_APP_INTELOWL_VERSION} - container_name: phunter + image: intelowlproject/intelowl_phunter:${REACT_APP_INTELOWL_VERSION} + container_name: intelowl_phunter restart: unless-stopped expose: - - "5000" + - "5612" volumes: - generic_logs:/var/log/intel_owl depends_on: From 4fb7ba3fc5464e58988fc799aaa6476e5345ce47 Mon Sep 17 00:00:00 2001 From: AnshSinghal Date: Fri, 2 May 2025 20:19:16 +0000 Subject: [PATCH 07/12] used shlex and removed repeated phonenumber check --- .../observable_analyzers/phunter.py | 13 +++++++++++-- integrations/phunter/app.py | 19 +++---------------- integrations/phunter/requirements.txt | 3 +-- 3 files changed, 15 insertions(+), 20 deletions(-) diff --git a/api_app/analyzers_manager/observable_analyzers/phunter.py b/api_app/analyzers_manager/observable_analyzers/phunter.py index 555254b85a..152da33f86 100644 --- a/api_app/analyzers_manager/observable_analyzers/phunter.py +++ b/api_app/analyzers_manager/observable_analyzers/phunter.py @@ -19,12 +19,16 @@ class PhunterAnalyzer(ObservableAnalyzer, DockerBasedAnalyzer): def run(self): try: - phonenumbers.parse(self.observable_name) + parsed_number = phonenumbers.parse(self.observable_name) + + formatted_number = phonenumbers.format_number( + parsed_number, phonenumbers.PhoneNumberFormat.E164 + ) except phonenumbers.phonenumberutil.NumberParseException: logger.error(f"Phone number parsing failed for: {self.observable_name}") return {"success": False, "error": "Invalid phone number"} - req_data = {"phone_number": self.observable_name} + req_data = {"phone_number": formatted_number} logger.info(f"Sending {self.name} scan request: {req_data} to {self.url}") try: @@ -42,6 +46,11 @@ def run(self): except ValueError as e: raise AnalyzerRunException(f"[{self.name}] Invalid response format: {e}") + except Exception as e: + raise AnalyzerRunException( + f"[{self.name}] An unexpected error occurred: {e}" + ) + @classmethod def update(self): pass diff --git a/integrations/phunter/app.py b/integrations/phunter/app.py index 86ede20123..56f28e0537 100644 --- a/integrations/phunter/app.py +++ b/integrations/phunter/app.py @@ -1,8 +1,8 @@ import logging import re +import shlex import subprocess -import phonenumbers from flask import Flask, jsonify, request # Logging Configuration @@ -67,23 +67,10 @@ def analyze(): logger.warning("No phone number provided in request") return jsonify({"error": "No phone number provided"}), 400 - try: - parsed_number = phonenumbers.parse(phone_number) - if not phonenumbers.is_valid_number(parsed_number): - logger.warning("Invalid phone number") - return jsonify({"error": "Invalid phone number"}), 400 - - formatted_number = phonenumbers.format_number( - parsed_number, phonenumbers.PhoneNumberFormat.E164 - ) - - except phonenumbers.phonenumberutil.NumberParseException: - logger.warning("Phone number parsing failed") - return jsonify({"error": "Invalid phone number format"}), 400 - try: logger.info("Executing Phunter CLI tool") - command = ["python3", "phunter.py", "-t", formatted_number] + command_str = f"python3 phunter.py -t {phone_number}" + command = shlex.split(command_str) result = subprocess.run( command, capture_output=True, diff --git a/integrations/phunter/requirements.txt b/integrations/phunter/requirements.txt index ba426941c2..174be8eeb2 100644 --- a/integrations/phunter/requirements.txt +++ b/integrations/phunter/requirements.txt @@ -1,2 +1 @@ -flask==3.1.0 -phonenumbers==9.0.3 \ No newline at end of file +flask==3.1.0 \ No newline at end of file From 6caac8684828093dd19a4d4718b7bd8c9b6a46cd Mon Sep 17 00:00:00 2001 From: AnshSinghal Date: Fri, 2 May 2025 20:34:32 +0000 Subject: [PATCH 08/12] fixed migration issue --- .../migrations/0156_analyzer_config_phunter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api_app/analyzers_manager/migrations/0156_analyzer_config_phunter.py b/api_app/analyzers_manager/migrations/0156_analyzer_config_phunter.py index 65703989ff..2e68e7b0fb 100644 --- a/api_app/analyzers_manager/migrations/0156_analyzer_config_phunter.py +++ b/api_app/analyzers_manager/migrations/0156_analyzer_config_phunter.py @@ -119,7 +119,7 @@ class Migration(migrations.Migration): atomic = False dependencies = [ ("api_app", "0071_delete_last_elastic_report"), - ("analyzers_manager", "0155_analyzer_config_debloat"), + ("analyzers_manager", "0156_alter_analyzer_config_required_api_key_abuse_ch"), ] operations = [migrations.RunPython(migrate, reverse_migrate)] From b32e62ff2af0e0f5676d0c912d853fec9185c7c2 Mon Sep 17 00:00:00 2001 From: AnshSinghal Date: Fri, 2 May 2025 20:56:08 +0000 Subject: [PATCH 09/12] cleaned --- api_app/analyzers_manager/observable_analyzers/phunter.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/api_app/analyzers_manager/observable_analyzers/phunter.py b/api_app/analyzers_manager/observable_analyzers/phunter.py index 152da33f86..8d63333b1d 100644 --- a/api_app/analyzers_manager/observable_analyzers/phunter.py +++ b/api_app/analyzers_manager/observable_analyzers/phunter.py @@ -47,9 +47,7 @@ def run(self): raise AnalyzerRunException(f"[{self.name}] Invalid response format: {e}") except Exception as e: - raise AnalyzerRunException( - f"[{self.name}] An unexpected error occurred: {e}" - ) + raise AnalyzerRunException(f"{self.name} An unexpected error occurred: {e}") @classmethod def update(self): From 0c91e86e0e1c48ce27b231647665104a2fd09f67 Mon Sep 17 00:00:00 2001 From: AnshSinghal Date: Sat, 3 May 2025 05:22:40 +0000 Subject: [PATCH 10/12] fixed migration file name --- ...analyzer_config_phunter.py => 0157_analyzer_config_phunter.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename api_app/analyzers_manager/migrations/{0156_analyzer_config_phunter.py => 0157_analyzer_config_phunter.py} (100%) diff --git a/api_app/analyzers_manager/migrations/0156_analyzer_config_phunter.py b/api_app/analyzers_manager/migrations/0157_analyzer_config_phunter.py similarity index 100% rename from api_app/analyzers_manager/migrations/0156_analyzer_config_phunter.py rename to api_app/analyzers_manager/migrations/0157_analyzer_config_phunter.py From a669283531c9d36be316ceb281951d7b166518d7 Mon Sep 17 00:00:00 2001 From: AnshSinghal Date: Wed, 21 May 2025 11:57:05 +0000 Subject: [PATCH 11/12] chore: trigger CI From 6afa95e2a514c8e3650d9e2980f175ac203ddec2 Mon Sep 17 00:00:00 2001 From: AnshSinghal Date: Wed, 21 May 2025 12:01:25 +0000 Subject: [PATCH 12/12] updated requirements file --- integrations/phunter/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integrations/phunter/requirements.txt b/integrations/phunter/requirements.txt index 174be8eeb2..97b3cabb88 100644 --- a/integrations/phunter/requirements.txt +++ b/integrations/phunter/requirements.txt @@ -1 +1 @@ -flask==3.1.0 \ No newline at end of file +flask==3.1.1 \ No newline at end of file