Refine get_purldb_entries to compare on plain PURL #307 (#308)

* Refine get_purldb_entries to compare on plain PURL #307

Including the qualifiers and subpaths in the comparison is too restrictive.

Signed-off-by: tdruez <[email protected]>

* Add changelog entry #307

Signed-off-by: tdruez <[email protected]>

---------

Signed-off-by: tdruez <[email protected]>

Author: tdruez

CHANGELOG.rst

@@ -153,6 +153,10 @@ Release notes
   This tab displays related packages grouped by their normalized ("plain") Package URL.
   https://github.com/aboutcode-org/dejacode/issues/276
 
+- Refine get_purldb_entries to compare on plain PackageURL.
+  Including the qualifiers and subpaths in the comparison was too restrictive.
+  https://github.com/aboutcode-org/dejacode/issues/307
+
 ### Version 5.2.1
 
 - Fix the models documentation navigation.

component_catalog/models.py

@@ -75,6 +75,7 @@
 from dje.models import ParentChildRelationshipModel
 from dje.models import ReferenceNotesMixin
 from dje.tasks import logger as tasks_logger
+from dje.utils import get_plain_purl
 from dje.utils import is_purl_str
 from dje.utils import merge_common_non_empty_values
 from dje.utils import set_fields_from_object
@@ -2560,9 +2561,14 @@ def get_purldb_entries(self, user, max_request_call=0, timeout=10):
             return []
 
         # Cleanup the PurlDB entries:
-        # - Packages with different PURL are excluded.
+        # Packages with different "plain" PURL are excluded. The qualifiers and
+        # subpaths are not involved in this comparison.
         if package_url:
-            purldb_entries = [entry for entry in purldb_entries if entry.get("purl") == package_url]
+            purldb_entries = [
+                entry
+                for entry in purldb_entries
+                if get_plain_purl(entry.get("purl", "")) == package_url
+            ]
 
         return purldb_entries
 

component_catalog/tests/test_models.py

@@ -2558,27 +2558,35 @@ def test_package_model_inferred_url_property(self):
 
     @mock.patch("dejacode_toolkit.purldb.PurlDB.find_packages")
     def test_package_model_get_purldb_entries(self, mock_find_packages):
-        purl = "pkg:pypi/[email protected]"
-        package1 = make_package(self.dataspace, package_url=purl)
+        purl1 = "pkg:pypi/[email protected]"
+        purl2 = "pkg:pypi/[email protected]?file_name=Django-3.0.tar.gz"
+        purl3 = "pkg:pypi/django"
+        package1 = make_package(self.dataspace, package_url=purl1)
         purldb_entry1 = {
-            "purl": purl,
+            "purl": purl1,
             "type": "pypi",
             "name": "django",
             "version": "3.0",
         }
         purldb_entry2 = {
-            "purl": "pkg:pypi/django",
+            "purl": purl2,
+            "type": "pypi",
+            "name": "django",
+            "version": "3.0",
+        }
+        purldb_entry3 = {
+            "purl": purl3,
             "type": "pypi",
             "name": "django",
         }
 
         mock_find_packages.return_value = None
         purldb_entries = package1.get_purldb_entries(user=self.user)
 
-        mock_find_packages.return_value = [purldb_entry1, purldb_entry2]
+        mock_find_packages.return_value = [purldb_entry1, purldb_entry2, purldb_entry3]
         purldb_entries = package1.get_purldb_entries(user=self.user)
         # The purldb_entry2 is excluded as the PURL differs
-        self.assertEqual([purldb_entry1], purldb_entries)
+        self.assertEqual([purldb_entry1, purldb_entry2], purldb_entries)
 
     @mock.patch("component_catalog.models.Package.get_purldb_entries")
     def test_package_model_update_from_purldb(self, mock_get_purldb_entries):

dje/tests/test_utils.py

@@ -31,6 +31,7 @@
 from dje.utils import get_instance_from_referer
 from dje.utils import get_instance_from_resolver
 from dje.utils import get_object_compare_diff
+from dje.utils import get_plain_purl
 from dje.utils import get_referer_resolver
 from dje.utils import get_zipfile
 from dje.utils import group_by_name_version
@@ -526,6 +527,15 @@ def test_utils_is_purl_fragment(self):
         for fragment in invalid_fragments:
             self.assertFalse(is_purl_fragment(fragment), msg=fragment)
 
+    def test_utils_get_plain_purl(self):
+        self.assertEqual("", get_plain_purl(""))
+        self.assertEqual("not:a/purl", get_plain_purl("not:a/purl"))
+        self.assertEqual("not:a/purl", get_plain_purl("not:a/purl"))
+        self.assertEqual("pkg:npm/[email protected]", get_plain_purl("pkg:npm/[email protected]"))
+        self.assertEqual(
+            "pkg:npm/[email protected]", get_plain_purl("pkg:npm/[email protected]?qualifier=1#frament")
+        )
+
     def test_utils_localized_datetime(self):
         self.assertIsNone(localized_datetime(None))
 

dje/utils.py

@@ -656,6 +656,11 @@ def is_purl_fragment(string):
     return any(connector in string for connector in purl_connectors)
 
 
+def get_plain_purl(purl_str):
+    """Remove the qualifiers and subpath from the `purl_str```."""
+    return purl_str.split("?")[0]
+
+
 def remove_empty_values(input_dict):
     """
     Return a new dict not including empty value entries from `input_dict`.