Major Changes -

- kindle_epub_fixer.py has been completely rewritten to more closely replicate the function of the original JS tool written by innocenat
- Auto backup of files processed by kindle_epub_fixer is now added, enabled by default and able to be turned on or off by the user in the settings panel
- Entries for Files that have been processed by kindle_epub_fixer are now automatically added to cwa.db in a new table called epub_fixes. These entries are now also available to view in the CWA Stats page accessible via the Admin Panel
- CWA Stats pages has been rearranged to have the stats for the functions users care most about at the top and a bug was fixed in the see more pages where the title for all pages was for the enforcement statistics regardless of what was being shown
- Creation of the /config/processed_books archive folders as well the setting of their permissions is now the sole responsibility of the CWA Auto Zipper service
- Major improvements to exception handling in both convert_library and ingest_processor when handling files to improve performance, reliability as well as making it much easier to diagnose user errors
- The default CWA settings are now pulled from the schema instead of being hardcoded into the CWA_DB class
- Plus general refactoring and tidying up of the codebase

root/app/calibre-web/cps/cwa_functions.py

@@ -96,6 +96,10 @@ def cwa_library_refresh():
 @login_required_if_no_ano
 @admin_required
 def set_cwa_settings():
+    cwa_db = CWA_DB()
+    cwa_default_settings = cwa_db.cwa_default_settings
+    cwa_settings = cwa_db.cwa_settings
+
     ignorable_formats = ['azw', 'azw3', 'azw4', 'cbz',
                         'cbr', 'cb7', 'cbc', 'chm',
                         'djvu', 'docx', 'epub', 'fb2',
@@ -104,20 +108,23 @@ def set_cwa_settings():
                         'prc', 'pdb', 'pml', 'rb',
                         'rtf', 'snb', 'tcr', 'txt', 'txtz']
     target_formats = ['epub', 'azw3', 'kepub', 'mobi', 'pdf']
-    boolean_settings = ["auto_backup_imports",
-                        "auto_backup_conversions",
-                        "auto_zip_backups",
-                        "cwa_update_notifications",
-                        "auto_convert",
-                        "auto_metadata_enforcement",
-                        "kindle_epub_fixer"]
-    string_settings = ["auto_convert_target_format"]
+
+    boolean_settings = []
+    string_settings = []
+    list_settings = []
+    for setting in cwa_default_settings:
+        if type(cwa_default_settings[setting]) == int:
+            boolean_settings.append(setting)
+        elif type(cwa_default_settings[setting]) == str and cwa_default_settings[setting] != "":
+            string_settings.append(setting)
+        else:
+            list_settings.append(setting)
+
     for format in ignorable_formats:
         string_settings.append(f"ignore_ingest_{format}")
         string_settings.append(f"ignore_convert_{format}")
 
     if request.method == 'POST':
-        cwa_db = CWA_DB()
         if request.form['submit_button'] == "Submit":
             result = {"auto_convert_ignored_formats":[], "auto_ingest_ignored_formats":[]}
             # set boolean_settings
@@ -171,8 +178,9 @@ def set_cwa_settings():
             cwa_settings = cwa_db.get_cwa_settings()
 
     elif request.method == 'GET':
-        cwa_db = CWA_DB()
-        cwa_settings = cwa_db.cwa_settings
+        # cwa_db = CWA_DB()
+        # cwa_settings = cwa_db.cwa_settings
+        ...
 
     return render_title_template("cwa_settings.html", title=_("CWA Settings"), page="cwa-settings",
                                     cwa_settings=cwa_settings, ignorable_formats=ignorable_formats,
@@ -184,57 +192,99 @@ def set_cwa_settings():
 ##                                                                            ##
 ##————————————————————————————————————————————————————————————————————————————##
 
+### TABLE HEADERS
+headers = {
+    "enforcement":{
+        "no_paths":[
+            "Timestamp", "Book ID", "Book Title", "Book Author", "Trigger Type"],
+        "with_paths":[
+            "Timestamp","Book ID", "Filepath"]
+        },
+    "epub_fixer":{
+        "no_fixes":[
+            "Timestamp", "Filename", "Manual?", "No. Fixes", "Original Backed Up?"],
+        "with_fixes":[
+            "Timestamp", "Filename", "Filepath", "Fixes Applied"]
+        },
+    "imports":[
+        "Timestamp", "Filename", "Original Backed Up?"],
+    "conversions":[
+        "Timestamp", "Filename", "Original Format", "End Format", "Original Backed Up?"],
+}
+
 @cwa_history.route("/cwa-history-show", methods=["GET", "POST"])
 @login_required_if_no_ano
 @admin_required
 def cwa_history_show():
     cwa_db = CWA_DB()
-    data, table_headers = cwa_db.enforce_show(paths=False, verbose=False, web_ui=True)
-    data_p, table_headers_p = cwa_db.enforce_show(paths=True, verbose=False, web_ui=True)
-    data_i, table_headers_i = cwa_db.get_import_history(verbose=False)
-    data_c, table_headers_c = cwa_db.get_conversion_history(verbose=False)
+    data_enforcement= cwa_db.enforce_show(paths=False, verbose=False, web_ui=True)
+    data_enforcement_with_paths = cwa_db.enforce_show(paths=True, verbose=False, web_ui=True)
+    data_imports = cwa_db.get_import_history(verbose=False)
+    data_conversions = cwa_db.get_conversion_history(verbose=False)
+    data_epub_fixer = cwa_db.get_epub_fixer_history(fixes=False, verbose=False)
+    data_epub_fixer_with_fixes = cwa_db.get_epub_fixer_history(fixes=True, verbose=False)
 
     return render_title_template("cwa_history.html", title=_("Calibre-Web Automated Stats"), page="cwa-history",
-                                    table_headers=table_headers, data=data,
-                                    table_headers_p=table_headers_p, data_p=data_p,
-                                    data_i=data_i, table_headers_i=table_headers_i,
-                                    data_c=data_c, table_headers_c=table_headers_c)
+                                data_enforcement=data_enforcement, headers_enforcement=headers["enforcement"]["no_paths"], 
+                                data_enforcement_with_paths=data_enforcement_with_paths,headers_enforcement_with_paths=headers["enforcement"]["with_paths"], 
+                                data_imports=data_imports, headers_import=headers["imports"],
+                                data_conversions=data_conversions, headers_conversion=headers["conversions"],
+                                data_epub_fixer=data_epub_fixer, headers_epub_fixer=headers["epub_fixer"]["no_fixes"],
+                                data_epub_fixer_with_fixes=data_epub_fixer_with_fixes, headers_epub_fixer_with_fixes=headers["epub_fixer"]["with_fixes"])
 
 @cwa_history.route("/cwa-history-show/full-enforcement", methods=["GET", "POST"])
 @login_required_if_no_ano
 @admin_required
 def show_full_enforcement():
     cwa_db = CWA_DB()
-    data, table_headers = cwa_db.enforce_show(paths=False, verbose=True, web_ui=True)
+    data = cwa_db.enforce_show(paths=False, verbose=True, web_ui=True)
     return render_title_template("cwa_history_full.html", title=_("Calibre-Web Automated - Full Enforcement History"), page="cwa-history-full",
-                                    table_headers=table_headers, data=data)
+                                    table_headers=headers["enforcement"]["no_paths"], data=data)
 
-@cwa_history.route("/cwa-history-show/full-enforcement-path", methods=["GET", "POST"])
+@cwa_history.route("/cwa-history-show/full-enforcement-with-paths", methods=["GET", "POST"])
 @login_required_if_no_ano
 @admin_required
 def show_full_enforcement_path():
     cwa_db = CWA_DB()
-    data, table_headers = cwa_db.enforce_show(paths=True, verbose=True, web_ui=True)
-    return render_title_template("cwa_history_full.html", title=_("Calibre-Web Automated - Full Enforcement History (Paths)"), page="cwa-history-full",
-                                    table_headers=table_headers, data=data)
+    data = cwa_db.enforce_show(paths=True, verbose=True, web_ui=True)
+    return render_title_template("cwa_history_full.html", title=_("Calibre-Web Automated - Full Enforcement History (w/ Paths)"), page="cwa-history-full",
+                                    table_headers=headers["enforcement"]["with_paths"], data=data)
 
 @cwa_history.route("/cwa-history-show/full-imports", methods=["GET", "POST"])
 @login_required_if_no_ano
 @admin_required
 def show_full_imports():
     cwa_db = CWA_DB()
-    data, table_headers = cwa_db.get_import_history(verbose=True)
+    data = cwa_db.get_import_history(verbose=True)
     return render_title_template("cwa_history_full.html", title=_("Calibre-Web Automated - Full Import History"), page="cwa-history-full",
-                                    table_headers=table_headers, data=data)
+                                    table_headers=headers["imports"], data=data)
 
 @cwa_history.route("/cwa-history-show/full-conversions", methods=["GET", "POST"])
 @login_required_if_no_ano
 @admin_required
 def show_full_conversions():
     cwa_db = CWA_DB()
-    data, table_headers = cwa_db.get_conversion_history(verbose=True)
+    data = cwa_db.get_conversion_history(verbose=True)
     return render_title_template("cwa_history_full.html", title=_("Calibre-Web Automated - Full Conversion History"), page="cwa-history-full",
-                                    table_headers=table_headers, data=data)
+                                    table_headers=headers["conversions"], data=data)
+
+@cwa_history.route("/cwa-history-show/full-epub-fixer", methods=["GET", "POST"])
+@login_required_if_no_ano
+@admin_required
+def show_full_epub_fixer():
+    cwa_db = CWA_DB()
+    data = cwa_db.get_epub_fixer_history(fixes=False, verbose=True)
+    return render_title_template("cwa_history_full.html", title=_("Calibre-Web Automated - Full EPUB Fixer History (w/out Paths & Fixes)"), page="cwa-history-full",
+                                    table_headers=headers["epub_fixer"]["no_fixes"], data=data)
+
+@cwa_history.route("/cwa-history-show/full-epub-fixer-with-paths-fixes", methods=["GET", "POST"])
+@login_required_if_no_ano
+@admin_required
+def show_full_epub_fixer_with_paths_fixes():
+    cwa_db = CWA_DB()
+    data = cwa_db.get_epub_fixer_history(fixes=True, verbose=True)
+    return render_title_template("cwa_history_full.html", title=_("Calibre-Web Automated - Full EPUB Fixer History (w/ Paths & Fixes)"), page="cwa-history-full",
+                                    table_headers=headers["epub_fixer"]["with_fixes"], data=data)
 
 ##————————————————————————————————————————————————————————————————————————————##
 ##                                                                            ##

root/app/calibre-web/cps/templates/admin.html

@@ -203,7 +203,7 @@ <h2>{{_('Scheduled Tasks ⌛')}}</h2>
   <div class="row form-group">
     <h2>{{_('CWA Admin Functions⚡')}}</h2>
     <a class="btn btn-default" id="cwa-settings" href="{{url_for('cwa_settings.set_cwa_settings')}}">{{_('CWA Settings')}}</a>
-    <a class="btn btn-default" id="cwa_history" href="{{url_for('cwa_history.cwa_history_show')}}">{{_('Show CWA History')}}</a>
+    <a class="btn btn-default" id="cwa_history" href="{{url_for('cwa_history.cwa_history_show')}}">{{_('Show CWA Stats')}}</a>
     <a class="btn btn-default" id="check_mon_srvs" href="{{url_for('cwa_check_status.cwa_flash_status')}}">{{_('Check CWA Status')}}</a>
   </div>
   <div class="row form-group">

root/app/calibre-web/cps/templates/cwa_history.html

@@ -2,30 +2,36 @@
 {% block body %}
 <div class="discover">
   <h2>{{title}}</h2>
+
     <div>
-      <h3>Calibre-Web Automated Enforcement History</h3><a class="btn btn-default" id="show_full_enforcement" href="{{url_for('cwa_history.show_full_enforcement')}}">{{_('Click to See More')}}</a><br>
+      <h3>Calibre-Web Automated Conversion History</h3>
+      <a class="btn btn-default" href="{{ url_for('cwa_history.show_full_conversions') }}">{{_('Click to See More')}}</a><br>
       <table class="table table-striped">
         <tr>
-          {% for header in table_headers %}
+          {% for header in headers_conversion %}
           <th>{{ header }}</th>
           {% endfor %}
         </tr>
-        {% for row in data %}
+        {% for row in data_conversions %}
         <tr>
           {% for cell in row %}
           <td>{{ cell }}</td>
           {% endfor %}
         </tr>
         {% endfor %}
       </table>
-      <h4><i>Paths</i></h4><a class="btn btn-default" id="show_full_enforcement" href="{{ url_for('cwa_history.show_full_enforcement_path') }}">{{_('Click to See More')}}</a><br>
+    </div>
+
+    <div>
+      <h3>Calibre-Web Automated Import History</h3>
+      <a class="btn btn-default" href="{{ url_for('cwa_history.show_full_imports') }}">{{_('Click to See More')}}</a><br>
       <table class="table table-striped">
         <tr>
-          {% for header in table_headers_p %}
+          {% for header in headers_import %}
           <th>{{ header }}</th>
           {% endfor %}
         </tr>
-        {% for row in data_p %}
+        {% for row in data_imports %}
         <tr>
           {% for cell in row %}
           <td>{{ cell }}</td>
@@ -34,15 +40,33 @@ <h4><i>Paths</i></h4><a class="btn btn-default" id="show_full_enforcement" href=
         {% endfor %}
       </table>
     </div>
+
     <div>
-      <h3>Calibre-Web Automated Import History</h3><a class="btn btn-default" id="show_full_enforcement" href="{{ url_for('cwa_history.show_full_imports') }}">{{_('Click to See More')}}</a><br>
+      <h3>Calibre-Web Automated EPUB Fixer History</h3>
+      <a class="btn btn-default" href="{{url_for('cwa_history.show_full_epub_fixer')}}">{{_('Click to See More')}}</a><br>
+      <table class="table table-striped">
+        <tr>
+          {% for header in headers_epub_fixer %}
+          <th>{{ header }}</th>
+          {% endfor %}
+        </tr>
+        {% for row in data_epub_fixer %}
+        <tr>
+          {% for cell in row %}
+          <td>{{ cell }}</td>
+          {% endfor %}
+        </tr>
+        {% endfor %}
+      </table>
+      <h4><i>EPUB Fixer History with Paths & Fixes</i></h4>
+      <a class="btn btn-default" href="{{ url_for('cwa_history.show_full_epub_fixer_with_paths_fixes') }}">{{_('Click to See More')}}</a><br>
       <table class="table table-striped">
         <tr>
-          {% for header in table_headers_i %}
+          {% for header in headers_epub_fixer_with_fixes %}
           <th>{{ header }}</th>
           {% endfor %}
         </tr>
-        {% for row in data_i %}
+        {% for row in data_epub_fixer_with_fixes %}
         <tr>
           {% for cell in row %}
           <td>{{ cell }}</td>
@@ -51,15 +75,33 @@ <h3>Calibre-Web Automated Import History</h3><a class="btn btn-default" id="show
         {% endfor %}
       </table>
     </div>
+
     <div>
-      <h3>Calibre-Web Automated Conversion History</h3><a class="btn btn-default" id="show_full_enforcement" href="{{ url_for('cwa_history.show_full_conversions') }}">{{_('Click to See More')}}</a><br>
+      <h3>Calibre-Web Automated Enforcement History</h3>
+      <a class="btn btn-default" href="{{url_for('cwa_history.show_full_enforcement')}}">{{_('Click to See More')}}</a><br>
+      <table class="table table-striped">
+        <tr>
+          {% for header in headers_enforcement %}
+          <th>{{ header }}</th>
+          {% endfor %}
+        </tr>
+        {% for row in data_enforcement %}
+        <tr>
+          {% for cell in row %}
+          <td>{{ cell }}</td>
+          {% endfor %}
+        </tr>
+        {% endfor %}
+      </table>
+      <h4><i>Enforcement History with Paths</i></h4>
+      <a class="btn btn-default" href="{{ url_for('cwa_history.show_full_enforcement_path') }}">{{_('Click to See More')}}</a><br>
       <table class="table table-striped">
         <tr>
-          {% for header in table_headers_c %}
+          {% for header in headers_enforcement_with_paths %}
           <th>{{ header }}</th>
           {% endfor %}
         </tr>
-        {% for row in data_c %}
+        {% for row in data_enforcement_with_paths %}
         <tr>
           {% for cell in row %}
           <td>{{ cell }}</td>
@@ -68,5 +110,6 @@ <h3>Calibre-Web Automated Conversion History</h3><a class="btn btn-default" id="
         {% endfor %}
       </table>
     </div>
+
 </div>
 {% endblock %}

root/app/calibre-web/cps/templates/cwa_history_full.html

@@ -1,9 +1,9 @@
 {% extends "layout.html" %}
 {% block body %}
 <div class="discover">
-  <h2>{{title}}</h2>
+  <h2>Calibre-Web Automated Stats</h2>
     <div>
-      <h3>Calibre-Web Automated Enforcement History</h3><br>
+      <h3>{{title}}</h3><br>
       <table class="table table-striped">
         <tr>
           {% for header in table_headers %}

root/app/calibre-web/cps/templates/cwa_settings.html

@@ -67,6 +67,14 @@ <h4 style="padding-bottom: 14px;">Automatic Backup Settings</h4>
         <label for="auto_backup_conversions" style="padding-left: 10px;">Enable CWA Conversion Auto-Backup</label><br>
         <p class="cwa-settings-tooltip">When active, the originals of ingested files that undergo conversion will be stored in /config/processed_books/converted</p>
 
+        {% if cwa_settings['auto_backup_epub_fixes'] %}
+        <input type="checkbox" id="auto_backup_epub_fixes" name="auto_backup_epub_fixes" value="True" checked style="accent-color: var(--color-secondary);" data-toggle="tooltip" data-placement="right" title="If deactivated, the originals of EPUB's that have been fixed will not be stored (except in the case of a process failure)">
+        {% else %}
+        <input type="checkbox" id="auto_backup_epub_fixes" name="auto_backup_epub_fixes" value="True" style="accent-color: var(--color-secondary);" data-toggle="tooltip" data-placement="right" title="If deactivated, the originals of EPUB's that have been fixed will not be stored (except in the case of a process failure)">
+        {% endif %}
+        <label for="auto_backup_epub_fixes" style="padding-left: 10px;">Enable CWA EPUB Fixer Auto-Backup</label><br>
+        <p class="cwa-settings-tooltip">When active, the originals of EPUBs processed by the CWA Kindle EPUB Fixer service will be stored in /config/processed_books/fixed_originals</p>
+
         {% if cwa_settings['auto_zip_backups'] %}
         <input type="checkbox" id="auto_zip_backups" name="auto_zip_backups" value="True" checked style="accent-color: var(--color-secondary);" data-toggle="tooltip" data-placement="right" data-html="true" title="If deactivated, backed up files will be left alone in their respective folders">
         {% else %}

root/etc/s6-overlay/s6-rc.d/cwa-auto-zipper/run

@@ -6,6 +6,10 @@ echo "[cwa-auto-zipper] Matching internal localtime & timezone with the one prov
 mkdir -p /config/processed_books/converted
 mkdir -p /config/processed_books/imported
 mkdir -p /config/processed_books/failed
+mkdir -p /config/processed_books/fixed_originals
+
+chown -R abc:abc /config/processed_books
+
 
 tz=$(printcontenv TZ)
 if [[ $tz == "" ]]

scripts/auto_zip.py

@@ -13,7 +13,9 @@ def __init__(self):
         self.converted_dir = self.archive_dirs_stem + "converted/"
         self.failed_dir = self.archive_dirs_stem + "failed/"
         self.imported_dir = self.archive_dirs_stem + "imported/"
-        self.archive_dirs = [self.converted_dir, self.failed_dir, self.imported_dir]
+        self.fixed_originals_dir = self.archive_dirs_stem + "fixed_originals/"
+
+        self.archive_dirs = [self.converted_dir, self.failed_dir, self.imported_dir, self.fixed_originals_dir]
 
         self.current_date = datetime.today().strftime('%Y-%m-%d')