implement pdf conversion [WIP] (#271)

* implement pdf conversion #267

* fix test failure?

* implement processing-log #270 and attack-navigator #269

* add missing migration

Author: fqrious

.github/workflows/run-tests.yml

@@ -48,13 +48,16 @@ jobs:
             export DJANGO_WWW_PATH=$PWD/django_www_path/
             echo DJANGO_WWW_PATH="$DJANGO_WWW_PATH" >> tests/tests.env
             mkdir -p $DJANGO_WWW_PATH
+            
             pip install -r tests/requirements.txt -r requirements.txt
+            sudo bash ./install_deps.sh
+            
   
       - name: Unit tests
         id: unit_tests
         run: |
           set -a; source tests/tests.env; set +a
-          pytest --cov --cov-branch --cov-report=xml --junitxml=unittest.junit.xml -o junit_family=legacy tests/ --ignore=tests/st/
+          pytest --cov --cov-branch --cov-report=xml --cov-report=term --junitxml=unittest.junit.xml -o junit_family=legacy tests/ --ignore=tests/st/
 
       - name: Schema tests
         id: schema_tests

Dockerfile

@@ -1,6 +1,12 @@
 FROM python:3.12-slim
 ENV PYTHONUNBUFFERED=1
 WORKDIR /usr/src/app
+
+COPY install_deps.sh .
+RUN --mount=type=cache,target=/var/cache/apt/archives/ \
+    bash install_deps.sh
+
+sudo bash ./install_deps.sh
 COPY requirements.txt ./
 RUN --mount=type=cache,target=/root/.cache \
     pip install -r requirements.txt

Dockerfile.deploy

@@ -29,6 +29,10 @@ ENV R2_CUSTOM_DOMAIN=""
 ENV SRO_OBJECTS_ONLY_LATEST=False
 
 WORKDIR /usr/src/app
+COPY install_deps.sh .
+RUN --mount=type=cache,target=/var/cache/apt/archives/ \
+    bash install_deps.sh
+    
 COPY requirements.txt ./
 RUN --mount=type=cache,target=/root/.cache \
     pip install -r requirements.txt

install_deps.sh

@@ -0,0 +1,2 @@
+apt update
+apt install -y libreoffice

siemrules/siemrules/migrations/0021_file_pdf_file.py

@@ -0,0 +1,19 @@
+# Generated by Django 5.2.4 on 2025-10-09 14:33
+
+import siemrules.siemrules.models
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('siemrules', '0020_alter_job_type'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='file',
+            name='pdf_file',
+            field=models.FileField(max_length=1024, null=True, upload_to=siemrules.siemrules.models.upload_to_func),
+        ),
+    ]

siemrules/siemrules/migrations/0022_file_txt2detection_data.py

@@ -0,0 +1,18 @@
+# Generated by Django 5.2.4 on 2025-10-10 12:04
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('siemrules', '0021_file_pdf_file'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='file',
+            name='txt2detection_data',
+            field=models.JSONField(default=None, null=True),
+        ),
+    ]

siemrules/siemrules/models.py

@@ -59,6 +59,8 @@ class File(models.Model):
     extract_text_from_image = models.BooleanField(default=True)
     ai_provider = models.CharField(max_length=256, null=True)
     markdown_file = models.FileField(max_length=512, upload_to=upload_to_func, null=True)
+    pdf_file = models.FileField(max_length=1024, upload_to=upload_to_func, null=True)
+    txt2detection_data = models.JSONField(default=None, null=True)
 
     created = models.DateTimeField(default=timezone.now, null=True)
 
@@ -87,6 +89,13 @@ def clean(self) -> None:
 
     def __str__(self) -> str:
         return f"File(id={self.id})"
+    
+        
+    @property
+    def archived_pdf(self):
+        if self.mode == 'pdf':
+            return self.file
+        return self.pdf_file
 
 @receiver(post_delete, sender=File)
 def remove_reports_on_delete(sender, instance: File, **kwargs):

siemrules/siemrules/serializers.py

@@ -9,6 +9,7 @@
 from django.template.defaultfilters import slugify
 import stix2, json
 from txt2detection.models import TAG_PATTERN
+from dogesec_commons.utils.serializers import JSONSchemaSerializer
 
 
 def validate_model(model):
@@ -86,8 +87,8 @@ def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
     def to_internal_value(self, data):
-        if isinstance(data, (str, Mapping)) or not hasattr(data, '__iter__'):
-            self.fail('not_a_list', input_type=type(data).__name__)
+        if isinstance(data, (str, Mapping)) or not hasattr(data, "__iter__"):
+            self.fail("not_a_list", input_type=type(data).__name__)
         retval = []
         if hasattr(self.parent, "skip_csv"):
             retval = data
@@ -110,7 +111,7 @@ class FileSerializer(serializers.ModelSerializer):
         help_text="Full path to the file to be converted. The mimetype of the file uploaded must match that expected by the `mode` selected. This is a file2txt setting.",
     )
     mode = serializers.ChoiceField(
-        choices=list(f2t_core.BaseParser.PARSERS.keys()),
+        choices=list(["txt", "html", "html_article", "word", "pdf", "md"]),
         help_text="How the File should be processed. This is a file2txt setting.",
     )
     report_id = ReportIDField(
@@ -181,6 +182,7 @@ class Meta:
         exclude = ["markdown_file", "status", "level"]
         read_only_fields = ["id"]
 
+
 class FileDocumentSerializer(FileSerializer):
     type_label = "siemrules.file"
     ai_provider = serializers.CharField(
@@ -305,8 +307,6 @@ class RuleCloneSerializer(serializers.Serializer):
     description = serializers.CharField(required=False)
 
 
-
-
 class HealthCheckChoices(StrEnum):
     AUTHORIZED = auto()
     UNAUTHORIZED = auto()
@@ -315,19 +315,109 @@ class HealthCheckChoices(StrEnum):
     UNKNOWN = auto()
     OFFLINE = auto()
 
+
 class HealthCheckChoiceField(serializers.ChoiceField):
     def __init__(self, **kwargs):
         choices = [m.value for m in HealthCheckChoices]
         super().__init__(choices, **kwargs)
-        
+
+
 class HealthCheckLLMs(serializers.Serializer):
     openai = HealthCheckChoiceField()
     deepseek = HealthCheckChoiceField()
     anthropic = HealthCheckChoiceField()
     gemini = HealthCheckChoiceField()
     openrouter = HealthCheckChoiceField()
 
+
 class HealthCheckSerializer(serializers.Serializer):
     ctibutler = HealthCheckChoiceField()
     vulmatch = HealthCheckChoiceField()
-    llms = HealthCheckLLMs()
+    llms = HealthCheckLLMs()
+
+
+class AttackNavigatorDomainSerializer(JSONSchemaSerializer):
+    json_schema = {
+        "$schema": "http://json-schema.org/draft-07/schema#",
+        "title": "MITRE ATT&CK Navigator Layer v4.5",
+        "type": "object",
+        "required": ["versions", "name", "domain", "techniques"],
+        "properties": {
+            "versions": {
+                "type": "object",
+                "properties": {
+                    "layer": {"type": "string", "example": "4.5"},
+                    "attack": {"type": "string", "example": "17.0"},
+                    "navigator": {"type": "string", "example": "5.1.0"},
+                },
+                "required": ["layer", "attack", "navigator"],
+                "additionalProperties": False,
+            },
+            "name": {"type": "string"},
+            "domain": {
+                "type": "string",
+                "enum": ["enterprise-attack", "mobile-attack", "ics-attack"],
+            },
+            "description": {"type": "string"},
+            "gradient": {
+                "type": "object",
+                "required": ["colors", "minValue", "maxValue"],
+                "properties": {
+                    "colors": {
+                        "type": "array",
+                        "items": {"type": "string", "pattern": "^#[0-9A-Fa-f]{6}$"},
+                    },
+                    "minValue": {"type": "number"},
+                    "maxValue": {"type": "number"},
+                },
+            },
+            "legendItems": {
+                "type": "array",
+                "items": {
+                    "type": "object",
+                    "properties": {
+                        "label": {"type": "string"},
+                        "color": {"type": "string", "pattern": "^#[0-9A-Fa-f]{6}$"},
+                        "value": {"type": "number"},
+                    },
+                },
+            },
+            "showTacticsRowBackground": {"type": "boolean"},
+            "techniques": {
+                "type": "array",
+                "items": {
+                    "type": "object",
+                    "required": ["techniqueID"],
+                    "properties": {
+                        "techniqueID": {"type": "string"},
+                        "tactic": {"type": "string"},
+                        "score": {"type": ["number", "null"]},
+                        "color": {"type": "string", "pattern": "^#[0-9A-Fa-f]{6}$"},
+                        "comment": {"type": "string"},
+                        "enabled": {"type": "boolean"},
+                        "links": {
+                            "type": "array",
+                            "items": {
+                                "type": "object",
+                                "properties": {
+                                    "href": {"type": "string", "format": "uri"},
+                                    "text": {"type": "string"},
+                                },
+                                "required": ["href", "text"],
+                            },
+                        },
+                    },
+                    "additionalProperties": True,
+                },
+            },
+            "tacticUseIds": {"type": "array", "items": {"type": "string"}},
+            "filters": {
+                "type": "object",
+                "properties": {
+                    "includeSubtechniques": {"type": "boolean"},
+                    "showOnlyVisibleTechniques": {"type": "boolean"},
+                },
+            },
+        },
+        "additionalProperties": True,
+    }

siemrules/siemrules/utils.py

@@ -43,3 +43,13 @@ def parse(self, stream, media_type=None, parser_context=None):
             return yaml.safe_load(stream)
         except Exception as e:
             raise ParseError(f"parse yaml error: {e}") from e
+
+
+class PDFRenderer(renderers.BaseRenderer):
+    media_type = 'application/pdf'
+    format = 'pdf'
+    charset = None
+    render_style = 'binary'
+
+    def render(self, data, media_type=None, renderer_context=None):
+        return data  # You must return raw bytes here (PDF content)