Feature - Issue #548 - Adds navigation to tA (unfoldingWord-dev#87)

* Feature - Issue #548 - Adds navigation to tA

* Properly handles adding classes to a soup element

client/client_webhook.py

@@ -180,6 +180,11 @@ def process_webhook(self):
                 job = json_data['job']
 
         cdn_handler = S3Handler(self.cdn_bucket)
+        s3_commit_key = 'u/{0}'.format(identifier)
+
+        # clear out the commit directory in the cdn bucket for this project revision
+        for obj in cdn_handler.get_objects(prefix=s3_commit_key):
+            cdn_handler.delete_file(obj.key)
 
         # Download the project.json file for this repo (create it if doesn't exist) and update it
         project_json_key = 'u/{0}/{1}/project.json'.format(repo_owner, repo_name)
@@ -207,9 +212,15 @@ def process_webhook(self):
         write_file(project_file, project_json)
         cdn_handler.upload_file(project_file, project_json_key, 0)
 
+        # Upload a manifest.yaml file for this project
+        manifest_file = os.path.join(self.base_temp_dir, 'manifest.yaml')
+        write_file(manifest_file, rc.as_dict())
+        cdn_handler.upload_file(manifest_file, s3_commit_key + '/manifest.yaml', 0)
+
         # Compile data for build_log.json
         build_log_json = job
         build_log_json['repo_name'] = repo_name
+        build_log_json['title'] = rc.resource.title
         build_log_json['repo_owner'] = repo_owner
         build_log_json['commit_id'] = commit_id
         build_log_json['committed_by'] = pusher_username
@@ -218,9 +229,6 @@ def process_webhook(self):
         build_log_json['commit_message'] = commit_message
 
         # Upload build_log.json to S3:
-        s3_commit_key = 'u/{0}'.format(identifier)
-        for obj in cdn_handler.get_objects(prefix=s3_commit_key):
-            cdn_handler.delete_file(obj.key)
         build_log_file = os.path.join(self.base_temp_dir, 'build_log.json')
         write_file(build_log_file, build_log_json)
         cdn_handler.upload_file(build_log_file, s3_commit_key + '/build_log.json', 0)

client/preprocessors.py

@@ -6,14 +6,15 @@
 from shutil import copy
 from resource_container.ResourceContainer import RC
 from glob import glob
+from resource_container.ResourceContainer import BIBLE_RESOURCE_TYPES
 
 
 def do_preprocess(rc, repo_dir, output_dir):
-    if rc.resource.identifier in ['obs']:
+    if rc.resource.identifier == 'obs':
         preprocessor = ObsPreprocessor(rc, repo_dir, output_dir)
-    elif rc.resource.identifier in ['bible', 'ulb', 'udb', 'reg']:
+    elif rc.resource.identifier in BIBLE_RESOURCE_TYPES:
         preprocessor = BiblePreprocessor(rc, repo_dir, output_dir)
-    elif rc.resource.identifier in ['ta']:
+    elif rc.resource.identifier == 'ta':
         preprocessor = TaPreprocessor(rc, repo_dir, output_dir)
     else:
         preprocessor = Preprocessor(rc, repo_dir, output_dir)
@@ -225,6 +226,7 @@ class TaPreprocessor(Preprocessor):
 
     def __init__(self, *args, **kwargs):
         super(TaPreprocessor, self).__init__(*args, **kwargs)
+        self.section_container_id = 1
 
     def get_title(self, project, link, alt_title=None):
         proj = None
@@ -271,12 +273,12 @@ def compile_section(self, project, section, level):
         :return: 
         """
         if 'link' in section:
-            markdown = '{0} <a name="{1}"/>{2}\n\n'.format('#'*level, section['link'], 
-                                                           self.get_title(project, section['link'], section['title']))
+            link = section['link']
         else:
-            markdown = '{0} {1}\n\n'.format('#'*level, section['title'])
+            link = 'section-container-{0}'.format(self.section_container_id)
+            self.section_container_id = self.section_container_id + 1
+        markdown = '{0} <a id="{1}"/>{2}\n\n'.format('#' * level, link, self.get_title(project, link, section['title']))
         if 'link' in section:
-            link = section['link']
             question = self.get_question(project, link)
             if question:
                 markdown += '*This page answers the question: {0}*\n\n'.format(question)
@@ -305,6 +307,7 @@ def compile_section(self, project, section, level):
 
     def run(self):
         for project in self.rc.projects:
+            self.section_container_id = 1
             toc = self.rc.toc(project.identifier)
             if project.identifier in self.manual_title_map:
                 title = self.manual_title_map[project.identifier]

converters/converter.py

@@ -1,7 +1,6 @@
 from __future__ import print_function, unicode_literals
 import os
 import tempfile
-import logging
 from aws_tools.s3_handler import S3Handler
 from general_tools.url_utils import download_file
 from general_tools.file_utils import unzip, add_contents_to_zip, remove_tree, remove

converters/md2html_converter.py

@@ -4,6 +4,7 @@
 import markdown
 import codecs
 from shutil import copyfile
+from bs4 import BeautifulSoup
 from general_tools.file_utils import write_file
 from converter import Converter
 from door43_tools.obs_handler import OBSInspection
@@ -27,7 +28,7 @@ def convert_obs(self):
         files = self.get_files()
 
         current_dir = os.path.dirname(os.path.realpath(__file__))
-        with open(os.path.join(current_dir, 'templates', 'obs-template.html')) as template_file:
+        with open(os.path.join(current_dir, 'templates', 'template.html')) as template_file:
             html_template = string.Template(template_file.read())
 
         found_chapters = {}
@@ -38,7 +39,7 @@ def convert_obs(self):
                 with codecs.open(filename, 'r', 'utf-8-sig') as md_file:
                     md = md_file.read()
                 html = markdown.markdown(md)
-                html = html_template.safe_substitute(content=html)
+                html = html_template.safe_substitute(title=self.resource.upper(), content=html)
                 base_name = os.path.splitext(os.path.basename(filename))[0]
                 found_chapters[base_name] = True
                 html_filename = base_name + ".html"
@@ -87,6 +88,16 @@ def convert_markdown(self):
                     md = md_file.read()
                 html = markdown.markdown(md)
                 html = html_template.safe_substitute(title=self.resource.upper(), content=html)
+
+                # Change headers like <h1><a id="verbs"/>Verbs</h1> to <h1 id="verbs">Verbs</h1>
+                soup = BeautifulSoup(html, 'html.parser')
+                for tag in soup.findAll('a', {'id': True}):
+                    if tag.parent and tag.parent.name in ['h1', 'h2', 'h3', 'h4', 'h5']:
+                        tag.parent['id'] = tag['id']
+                        tag.parent['class'] = tag.parent.get('class', []) + ['section-header']
+                        tag.extract()
+                html = unicode(soup)
+
                 base_name = os.path.splitext(os.path.basename(filename))[0]
                 found_chapters[base_name] = True
                 html_filename = base_name + ".html"

converters/templates/template.html

@@ -5,7 +5,7 @@
     <title>$title</title>
 </head>
 <body>
-<div id="content" class="content">
+<div id="content">
 $content
 </div>
 </body>

converters/usfm2html_converter.py

@@ -7,12 +7,13 @@
 from general_tools.file_utils import write_file, remove_tree
 from converter import Converter
 from usfm_tools.transform import UsfmTransform
+from resource_container.ResourceContainer import BIBLE_RESOURCE_TYPES
 
 
 class Usfm2HtmlConverter(Converter):
 
     def convert(self):
-        if self.resource in ['ulb', 'udb', 'bible', 'reg']:
+        if self.resource in BIBLE_RESOURCE_TYPES:
             self.convert_bible()
             return True
         else:
@@ -25,7 +26,7 @@ def convert_bible(self):
         files = self.get_files()
 
         current_dir = os.path.dirname(os.path.realpath(__file__))
-        with open(os.path.join(current_dir, 'templates', 'bible-template.html')) as template_file:
+        with open(os.path.join(current_dir, 'templates', 'template.html')) as template_file:
             template_html = template_file.read()
 
         for filename in files:
@@ -39,6 +40,7 @@ def convert_bible(self):
                 with codecs.open(os.path.join(scratch_dir, html_filename), 'r', 'utf-8-sig') as html_file:
                     converted_html = html_file.read()
                 template_soup = BeautifulSoup(template_html, 'html.parser')
+                template_soup.head.title.string = self.resource.upper()
                 converted_soup = BeautifulSoup(converted_html, 'html.parser')
                 content_div = template_soup.find('div', id='content')
                 content_div.clear()

door43_tools/project_deployer.py

@@ -66,16 +66,9 @@ def deploy_revision_to_door43(self, build_log_key):
 
         self.cdn_handler.download_dir(s3_commit_key, source_dir)
         source_dir = os.path.join(source_dir, s3_commit_key)
-
         resource_type = build_log['resource_type']
-        if resource_type == 'ulb' or resource_type == 'udb':
-            resource_type = 'bible'
-
-        if resource_type in ['bible', 'ulb', 'udb']:
-            template_key = 'templates/bible.html'
-        else:
-            template_key = 'templates/obs.html'
-        template_file = os.path.join(template_dir, 'template.html')
+        template_key = 'templates/project-page.html'
+        template_file = os.path.join(template_dir, 'project-page.html')
         self.logger.debug("Downloading {0} to {1}...".format(template_key, template_file))
         self.door43_handler.download_file(template_key, template_file)
 
@@ -103,9 +96,11 @@ def deploy_revision_to_door43(self, build_log_key):
                 """
                 content += '<ul><li>' + '</li><li>'.join(build_log['warnings']) + '</li></ul>'
             else:
-                content += '<h1>{0}</h1>'.format(build_log['message'])
+                content += '<h1 class="conversion-requested">{0}</h1>'.format(build_log['message'])
                 content += '<p><i>No content is available to show for {0} yet.</i></p>'.format(repo_name)
-
+                content += """
+                <script type="text/javascript">setTimeout(function(){window.location.reload(1);}, 10000);</script>
+                """
             html = """
                 <html lang="en">
                     <head>

door43_tools/templaters.py

@@ -1,24 +1,30 @@
 from __future__ import unicode_literals, print_function
 import os
-import json
 import codecs
 import logging
 from glob import glob
 from bs4 import BeautifulSoup
 from general_tools.file_utils import write_file
 from resource_container.ResourceContainer import RC
+from general_tools.file_utils import load_yaml_object
+from resource_container.ResourceContainer import BIBLE_RESOURCE_TYPES
 
 
 def do_template(resource_type, source_dir, output_dir, template_file):
-    if resource_type in ['udb', 'ulb', 'bible']:
-        templater = BibleTemplater(source_dir, output_dir, template_file)
+    if resource_type in BIBLE_RESOURCE_TYPES:
+        templater = BibleTemplater(resource_type, source_dir, output_dir, template_file)
+    elif resource_type == 'obs':
+        templater = ObsTemplater(resource_type, source_dir, output_dir, template_file)
+    elif resource_type == 'ta':
+        templater = TaTemplater(resource_type, source_dir, output_dir, template_file)
     else:
-        templater = Templater(source_dir, output_dir, template_file)
+        templater = Templater(resource_type, source_dir, output_dir, template_file)
     return templater.run()
 
 
 class Templater(object):
-    def __init__(self, source_dir, output_dir, template_file):
+    def __init__(self, resource_type, source_dir, output_dir, template_file):
+        self.resource_type = resource_type
         self.source_dir = source_dir  # Local directory
         self.output_dir = output_dir  # Local directory
         self.template_file = template_file  # Local file of template
@@ -33,6 +39,11 @@ def run(self):
         self.rc = RC(self.source_dir)
         with open(self.template_file) as template_file:
             self.template_html = template_file.read()
+            soup = BeautifulSoup(self.template_html, 'html.parser')
+            soup.body['class'] = soup.body.get('class', []) + [self.resource_type]
+            if self.resource_type in BIBLE_RESOURCE_TYPES and self.resource_type != 'bible':
+                soup.body['class'] = soup.body.get('class', []) + ['bible']
+            self.template_html = unicode(soup)
         self.apply_template()
         return True
 
@@ -161,6 +172,11 @@ def apply_template(self):
 
             # render the html as an unicode string
             html = unicode(soup)
+
+            # fix the footer message, removing the title of this page in parentheses as it doesn't get filled
+            html = html.replace(
+                '("<a xmlns:dct="http://purl.org/dc/terms/" href="https://live.door43.org/templates/project-page.html" rel="dct:source">{{ HEADING }}</a>") ',
+                '')
             # update the canonical URL - it is in several different locations
             html = html.replace(canonical, canonical.replace('/templates/', '/{0}/'.format(language_code)))
             # write to output directory
@@ -181,7 +197,7 @@ def __init__(self, *args, **kwargs):
     def build_page_nav(self, filename=None):
         html = """
         <nav class="affix-top hidden-print hidden-xs hidden-sm" id="right-sidebar-nav">
-            <ul id="sidebar-nav" class="affix affix-top nav nav-stacked books panel-group">
+            <ul id="sidebar-nav" class="nav nav-stacked books panel-group">
             """
         for fname in self.files:
             filebase = os.path.splitext(os.path.basename(fname))[0]
@@ -216,7 +232,7 @@ def build_page_nav(self, filename=None):
                 html += """
                        <li class="chapter"><a href="{0}#{1}">{2}</a></li>
                     """.format(os.path.basename(fname) if fname != filename else '', chapter['id'],
-                               chapter['id'].split('-')[1].lstrip('0'))
+                               chapter['id'].split('-')[2].lstrip('0'))
             html += """
                         </ul>
                     </div>
@@ -227,3 +243,73 @@ def build_page_nav(self, filename=None):
         </nav>
             """
         return html
+
+
+class TaTemplater(Templater):
+    def __init__(self, *args, **kwargs):
+        super(TaTemplater, self).__init__(*args, **kwargs)
+        self.section_container_id = 1
+
+    def build_section_toc(self, section):
+        """
+        Recursive section toc builder
+        :param dict section: 
+        :return: 
+        """
+        if 'link' in section:
+            link = section['link']
+        else:
+            link = 'section-container-{0}'.format(self.section_container_id)
+            self.section_container_id = self.section_container_id + 1
+        html = """
+            <li>
+                <a href="#{0}">{1}</a>
+            """.format(link, section['title'])
+        if 'sections' in section:
+            html += """
+                <ul>
+            """
+            for subsection in section['sections']:
+                html += self.build_section_toc(subsection)
+            html += """
+                </ul>
+            """
+        html += """
+            </li>
+        """
+        return html
+
+    def build_page_nav(self, filename=None):
+        self.section_container_id = 1
+        html = """
+            <nav class="affix-top hidden-print hidden-xs hidden-sm" id="right-sidebar-nav">
+                <ul id="sidebar-nav" class="nav nav-stacked">
+        """
+        for fname in self.files:
+            with codecs.open(fname, 'r', 'utf-8-sig') as f:
+                soup = BeautifulSoup(f.read(), 'html.parser')
+            if soup.find('h1'):
+                title = soup.find('h1').text
+            else:
+                title = os.path.splitext(os.path.basename(fname))[0].title()
+            if title == "Conversion requested..." or title == "Conversion successful" or title == "Index":
+                continue
+            if fname != filename:
+                html += """
+                <h4><a href="{0}">{1}</a></h4>
+                """.format(os.path.basename(fname), title)
+            else:
+                html += """
+                <h4>{0}</h4>
+                """.format(title)
+                toc = load_yaml_object(os.path.join('{0}-toc.yaml'.format(os.path.splitext(fname)[0])))
+                if toc:
+                    for section in toc['sections']:
+                        html += self.build_section_toc(section)
+                html += """
+                """
+        html += """
+                </ul>
+            </nav>
+        """
+        return html