Check if license cache exists after obtaining lock

Fix: #4273

Signed-off-by: Markus Obendrauf <[email protected]>

Author: Markus Obendrauf

src/licensedcode/cache.py

@@ -103,20 +103,11 @@ def load_or_build(
         create_dir(idx_cache_dir)
         cache_file = os.path.join(idx_cache_dir, LICENSE_INDEX_FILENAME)
 
-        has_cache = os.path.exists(cache_file) and os.path.getsize(cache_file)
-
         # bypass build if cache exists
-        if has_cache and not force:
-            try:
-                # save the list of additional directories included in the cache, or None if the cache does not
-                # include any additional directories
-                return load_cache_file(cache_file)
-            except Exception as e:
-                # work around some rare Windows quirks
-                import traceback
-                print('Inconsistent License cache: rebuilding index.')
-                print(str(e))
-                print(traceback.format_exc())
+        if not force:
+            license_cache = _load_cache_file_if_exists(cache_file)
+            if license_cache is not None:
+                return license_cache
 
         from licensedcode.models import licenses_data_dir as ldd
         from licensedcode.models import rules_data_dir as rdd
@@ -135,6 +126,10 @@ def load_or_build(
         try:
             # acquire lock and wait until timeout to get a lock or die
             with lockfile.FileLock(lock_file).locked(timeout=timeout):
+                if not force:
+                    license_cache = _load_cache_file_if_exists(cache_file)
+                    if license_cache is not None:
+                        return license_cache
 
                 additional_directories = []
                 if only_builtin:
@@ -207,6 +202,24 @@ def dump(self, cache_file):
             pickle.dump(self, fn, protocol=PICKLE_PROTOCOL)
 
 
+def _load_cache_file_if_exists(cache_file):
+    """
+    Return a LicenseCache loaded from ``cache_file`` if it exists.
+    """
+    if os.path.exists(cache_file) and os.path.getsize(cache_file):
+        try:
+            # save the list of additional directories included in the cache, or None if the cache does not
+            # include any additional directories
+            return load_cache_file(cache_file)
+        except Exception as e:
+            # work around some rare Windows quirks
+            import traceback
+            print('Inconsistent License cache: rebuilding index.')
+            print(str(e))
+            print(traceback.format_exc())
+    return None
+
+
 def build_index(
     licenses_db=None,
     licenses_data_dir=None,

tests/licensedcode/test_zzzz_cache.py

@@ -78,6 +78,7 @@ def test_LicenseCache_load_or_build_from_empty(self):
             rules_data_dir=rules_data_dir,
         )
 
+        # fail
         assert hash.sha1(cache_file) != idx_checksum_before
 
         # force=False loads an index
@@ -159,7 +160,7 @@ def test_get_spdx_symbols_checks_duplicates_with_deprecated_on_live_db(self):
     def test_build_spdx_license_expression(self):
         from licensedcode.cache import build_spdx_license_expression
         assert build_spdx_license_expression("mit")
-    
+
     def test_build_spdx_license_expression_fails_on_invalid_key_none(self):
         from licensedcode.cache import build_spdx_license_expression
         from licensedcode.cache import InvalidLicenseKeyError