Merge pull request #890 from CDLUC3/develop

Merge Develop to the main branch

Author: jsjiang

ezidapp/management/commands/proc-link-checker.py

@@ -60,6 +60,12 @@
 
 The link checker notices within a few seconds when the exclusion file has been modified.
 Examine the link checker's log file to confirm that it has been reloaded successfully.
+
+There is also the option to exclude identifiers based on what they start with which will
+usually be used to exclude shoulders.
+
+    LINKCHECKER_ID_EXCLUSION_ENABLED = True
+    LINKCHECKER_ID_EXCLUSION_FILE = 'path/to/id_exclusion_file.txt'
 """
 
 # noinspection PyUnresolvedReferences
@@ -119,12 +125,22 @@ def __init__(self):
         self._temporaryExcludes = []
         self._exclusionFile = None
 
+        self._idExclusionFileModifyTime = -1
+        self._idLastExclusionFileCheckTime = -1
+        self._idExclusionFile = None
+        self._idExclusionTuple = None
+
     def run(self):
         if django.conf.settings.LINKCHECKER_EXCLUSION_ENABLED:
             if django.conf.settings.LINKCHECKER_EXCLUSION_FILE is not None:
                 self._exclusionFile = django.conf.settings.LINKCHECKER_EXCLUSION_FILE
                 log.info(f"Link checker exclusion enabled with file: {self._exclusionFile}")
 
+        if django.conf.settings.LINKCHECKER_ID_EXCLUSION_ENABLED:
+            if django.conf.settings.LINKCHECKER_ID_EXCLUSION_FILE is not None:
+                self._idExclusionFile = django.conf.settings.LINKCHECKER_ID_EXCLUSION_FILE
+                log.info(f"Link checker ID exclusion enabled with file: {self._idExclusionFile}")
+
         while not self.terminated():
             self.check_all()
 
@@ -227,6 +243,44 @@ def remaining(self, start, cycle):
     def daysSince(self, when):
         return int((self.now() - when) / 86400)
 
+    def loadIdExclusionFile(self):
+        if self._idExclusionFile is None:
+            return
+        if self.now_int() - self._idLastExclusionFileCheckTime < 10:
+            return
+        self._idLastExclusionFileCheckTime = self.now_int()
+        f = None
+        s = None
+        try:
+            # noinspection PyTypeChecker
+            s = os.stat(self._idExclusionFile)
+            if s.st_mtime == self._idExclusionFileModifyTime:
+                return
+            # noinspection PyTypeChecker
+            f = open(self._idExclusionFile)
+
+            id_exclusion = []
+            n = 0
+            for l in f:
+                n += 1
+                if l.strip() == "" or l.startswith("#"):
+                    continue
+
+                id_exclusion.append(l.strip().casefold())  # Store as lowercase for case-insensitive matching
+
+            self._idExclusionTuple = tuple(id_exclusion)
+            self._exclusionFileModifyTime = s.st_mtime
+            log.info("id exclusion file successfully loaded")
+        except Exception as e:
+            log.error('Exception')
+            if s is not None:
+                self._exclusionFileModifyTime = s.st_mtime
+            log.error("error loading exclusion file: " + str(e))
+        finally:
+            if f is not None:
+                f.close()
+
+
     def loadExclusionFile(self):
         if self._exclusionFile is None:
             return
@@ -288,14 +342,20 @@ def harvest(self, model, only=None, filter=None):
             if len(qs) == 0:
                 break
             for o in qs:
+                # added to exclude ID patterns if they match the id exclusion tuple in addition to normal filtering
                 if filter is None or filter(o):
-                    # log.debug(f'Generator returning: {str(o)}')
-                    yield o
+                    if self._idExclusionTuple is None or (o.identifier is not None and
+                            not o.identifier.casefold().startswith(self._idExclusionTuple)):
+                        yield o
+                    else:
+                        log.debug('Skipping identifier %s due to ID shoulder exclusion', o.identifier)
+
             lastIdentifier = qs[-1].identifier
         yield None
 
     def updateDatabaseTable(self):
         self.loadExclusionFile()
+        self.loadIdExclusionFile()
         log.info("begin update table")
         numIdentifiers = 0
         numAdditions = 0
@@ -413,6 +473,7 @@ def updateDatabaseTable(self):
     def loadWorkset(self):
         self._workset = None
         self.loadExclusionFile()
+        self.loadIdExclusionFile()
         log.info("begin load workset")
         _workset = []
         numOwnersCapped = 0
@@ -490,6 +551,7 @@ def getNextLink(self):
         _lock.acquire()
         try:
             self.loadExclusionFile()
+            self.loadIdExclusionFile()
             startingIndex = self._index
             allFinished = True
             t = self.now()

settings/settings.py.j2

@@ -529,10 +529,14 @@ LINKCHECKER_MAX_READ = 104_857_600
 # EZID Link checker e-mails
 LINK_CHECKER_ADMIN = {{ link_checker_admin }}
 LINKCHECKER_EXCLUSION_ENABLED = True
+LINKCHECKER_ID_EXCLUSION_ENABLED = True
 # Full path to the file containing the list of user accounts to be excluded from link checking.
 # None or empty string means no exclusion file.
 LINKCHECKER_EXCLUSION_FILE = DATA_DIR / 'link_check_exclusion_list.txt'
 
+# the ID exclusion file contains startswith identifier patterns to be excluded.
+LINKCHECKER_ID_EXCLUSION_FILE = DATA_DIR / 'linkchecker_id_exclusion_list.txt'
+
 # Internal settings
 
 # Changes in these settings may require corresponding source code modifications.

templates/manage/details.html

@@ -117,7 +117,7 @@ <h2 class="heading__icon-9">{% trans "About the Identified Object" %}</h2>
   <hr/>
   <div class="row vertical-buffer-bot">
     <div class="inline-header">
-      <h2 class="heading__icon-4">{% trans "About the Identifier's Status" %} kdlkjklsjfkslj</h2>
+      <h2 class="heading__icon-4">{% trans "About the Identifier's Status" %}</h2>
     </div>
   </div>
   <div class="row vertical-buffer-bot">