From 0960c21f4aa793a346a7aec0ea9a7dd9c40a5139 Mon Sep 17 00:00:00 2001 From: Arthur Collet Date: Thu, 10 Apr 2025 13:04:50 -0500 Subject: [PATCH 1/3] - Updating deps - Updating syntax to support updated plugins - Added Dockerfile --- .dockerignore | 7 ++++ Dockerfile | 53 +++++++++++++++++++++++++++++++ build.sh | 32 +++++++++++++++++++ buildencyclopedia.py | 2 +- epel9.repo | 7 ++++ mkdocs.yml | 7 ++-- nginx.conf | 19 +++++++++++ pandoc_converter.patch | 11 +++++++ requirements.in | 8 +++++ requirements.txt | 58 +++++++++++++++++++++++++++++++--- tables.patch | 20 ++++++++++++ theme/404.html | 2 +- theme/base.html | 40 +++++++++++------------ theme/content.html | 6 ++-- theme/nav-section-map-toc.html | 2 +- theme/nav-section-map.html | 2 +- theme/nav.html | 8 ++--- theme/toc.html | 2 +- 18 files changed, 246 insertions(+), 40 deletions(-) create mode 100644 .dockerignore create mode 100644 Dockerfile create mode 100644 build.sh create mode 100644 epel9.repo create mode 100644 nginx.conf create mode 100644 pandoc_converter.patch create mode 100644 requirements.in create mode 100644 tables.patch diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..2de1448f9 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,7 @@ +logs/ +.git +*.md +.cache +*Dockerfile* +.dockerignore +.dive-ci diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..cdee3bd44 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,53 @@ +# syntax=docker/dockerfile:1.0 +ARG BASE_VERSION=3.2.1 +ARG REGISTRY=docker.osdc.io/ncigdc +ARG SERVICE_NAME=gene-expression +ARG PYTHON_VERSION=python3.9 + +FROM ${REGISTRY}/${PYTHON_VERSION}-builder:${BASE_VERSION} as build +ARG SERVICE_NAME +ARG PIP_INDEX_URL +ENV PIP_INDEX_URL=$PIP_INDEX_URL + +WORKDIR /build +# Install dependencies first so Docker can reuse the cached layer as +# long as we don't update any pins. +COPY epel9.repo /etc/yum.repos.d/ +RUN dnf install -y pandoc --enablerepo=epel9 && \ + dnf install -y sed texlive texlive-xetex + + +COPY requirements.txt . +RUN pip install -r requirements.txt +RUN pip install git+https://github.com/jgrassler/mkdocs-pandoc + +COPY pandoc_converter.patch . +COPY tables.patch . +RUN patch -u /venv/lib/python3.9/site-packages/mkdocs_pandoc/pandoc_converter.py -i pandoc_converter.patch && \ + patch -u /venv/lib/python3.9/site-packages/mkdocs_pandoc/filters/tables.py -i tables.patch +COPY . /apps + +WORKDIR /apps +RUN bash build.sh +#RUN mkdocs build --clean + + +FROM ${REGISTRY}/nginx:${BASE_VERSION} +ARG NAME +ARG PYTHON_VERSION +ARG SERVICE_NAME +ARG GIT_BRANCH +ARG COMMIT +ARG BUILD_DATE + +LABEL org.opencontainers.image.title="${SERVICE_NAME}" \ + org.opencontainers.image.description="${SERVICE_NAME} container image" \ + org.opencontainers.image.source="https://github.com/NCI-GDC/${SERVICE_NAME}" \ + org.opencontainers.image.vendor="NCI GDC" \ + org.opencontainers.image.ref.name="${SERVICE_NAME}:${GIT_BRANCH}" \ + org.opencontainers.image.revision="${COMMIT}" \ + org.opencontainers.image.created="${BUILD_DATE}" + +COPY --chown=app:app nginx.conf /etc/nginx/conf.d/default.conf +COPY --chown=app:app --from=build /app /app +EXPOSE 80 443 diff --git a/build.sh b/build.sh new file mode 100644 index 000000000..49e5711e7 --- /dev/null +++ b/build.sh @@ -0,0 +1,32 @@ +#!/bin/bash +echo "$(date +'%d %B %Y - %k:%M'): ${ENVIRONMENT}: Looking for User Guides" +userGuides=() +for i in $( ls *_UG.yml ); do + userGuides+=(${i::${#i}-7}) +done +echo "$(date +'%d %B %Y - %k:%M'): ${ENVIRONMENT}: Number of User Guides found: ${#userGuides[@]}" + +for userGuide in "${userGuides[@]}"; do + echo "$(date +'%d %B %Y - %k:%M'): ${ENVIRONMENT}: ${userGuide}: Starting creation" + if [ ! -d "docs/Data_Portal/PDF/" ]; then + echo "$(date +'%d %B %Y - %k:%M'): ${ENVIRONMENT}: ${userGuide}: PDF Directory does not exists, creating ..." + mkdir docs/${userGuide}/PDF/ + fi + echo "$(date +'%d %B %Y - %k:%M'): ${ENVIRONMENT}: ${userGuide}: Building pandoc document" + /venv/bin/mkdocs2pandoc -f ${userGuide}_UG.yml -o docs/${userGuide}/PDF/${userGuide}_UG.pd + echo "$(date +'%d %B %Y - %k:%M'): ${ENVIRONMENT}: ${userGuide}: Replacing strings in pandoc document " + /bin/sed -i -e 's/# / /g' docs/${userGuide}/PDF/${userGuide}_UG.pd + /bin/sed -i -e 's/### /## /g' docs/${userGuide}/PDF/${userGuide}_UG.pd + /bin/sed -i -e 's/\/site\//\/docs\//g' docs/${userGuide}/PDF/${userGuide}_UG.pd + /bin/sed -i -e "s/(images/(https:\/\/gdc-docs.nci.nih.gov\/"$userGuide"\/Users_Guide\/images/g" docs/${userGuide}/PDF/${userGuide}_UG.pd #To make images clickable in the PDF + echo "$(date +'%d %B %Y - %k:%M'): ${ENVIRONMENT}: ${userGuide}: Building PDF from pandoc document " + /usr/bin/pandoc --listings -H theme/latex/listings-setup.tex --toc -V documentclass=report -V geometry:"top=2cm, bottom=1.5cm, left=1cm, right=1cm" -f markdown+grid_tables+table_captions docs/${userGuide}/PDF/${userGuide}_Title.txt -o docs/${userGuide}/PDF/${userGuide}_UG.pdf docs/${userGuide}/PDF/${userGuide}_UG.pd +done + + +echo "$(date +'%d %B %Y - %k:%M'): ${ENVIRONMENT}: Build Encyclopedia" +python buildencyclopedia.py + + +echo "$(date +'%d %B %Y - %k:%M'): ${ENVIRONMENT}: Deploying new version to /app" +/venv/bin/mkdocs build -v --site-dir /app/ diff --git a/buildencyclopedia.py b/buildencyclopedia.py index cbf60a5f5..9e93a43ec 100644 --- a/buildencyclopedia.py +++ b/buildencyclopedia.py @@ -9,7 +9,7 @@ FILEARRAY = sorted(FILEARRAY, key=str.lower) with open(ABSFILEPATH + '/mkdocs.yml', 'r') as f: - doc = yaml.load(f) + doc = yaml.full_load(f) encycdict = next(d for (index, d) in enumerate(doc['pages']) \ if d.get('EncyclopediaEntries', False) != False) diff --git a/epel9.repo b/epel9.repo new file mode 100644 index 000000000..9f41d026f --- /dev/null +++ b/epel9.repo @@ -0,0 +1,7 @@ +[epel9] +baseurl=https://nexus.osdc.io/repository/rpm-epel/9/Everything/x86_64 +gpgcheck=0 +metadata_expire=1d +mode=0640 +name=Extra Packages for Enterprise Linux 9 +enabled=0 diff --git a/mkdocs.yml b/mkdocs.yml index c655fe291..136f3b4a1 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -142,7 +142,7 @@ pages: - Release Number: Encyclopedia/pages/Release_Number.md - REST API: Encyclopedia/pages/REST_API.md - RNA-Seq: Encyclopedia/pages/RNA-Seq.md - - Sequence Trace File: Encyclopedia/pages/Sequence_Trace_File.md +# - Sequence Trace File: Encyclopedia/pages/Sequence_Trace_File.md - SNP Array-Based Data: Encyclopedia/pages/SNP_Array-Based_Data.md - TCGA Barcode: Encyclopedia/pages/TCGA_Barcode.md - TCIA: Encyclopedia/pages/TCIA.md @@ -152,5 +152,6 @@ pages: repo_url: https://github.com/NCI-GDC/gdc-docs site_name: GDC Docs site_url: http://docs.gdc.cancer.gov -theme: mkdocs -theme_dir: theme +theme: + name: mkdocs + custom_dir: theme diff --git a/nginx.conf b/nginx.conf new file mode 100644 index 000000000..ec7c7a0eb --- /dev/null +++ b/nginx.conf @@ -0,0 +1,19 @@ +server { + listen 80 default_server; + listen 443 ssl default_server; + listen [::]:80 default_server; + listen [::]:443 ssl default_server; + + server_name _; + + set $empty ""; + + ssl_ciphers aNULL; + ssl_certificate data:$empty; + ssl_certificate_key data:$empty; + + location / { + root /app; + index index.html; + } +} diff --git a/pandoc_converter.patch b/pandoc_converter.patch new file mode 100644 index 000000000..216fa840e --- /dev/null +++ b/pandoc_converter.patch @@ -0,0 +1,11 @@ +--- /venv/lib/python3.9/site-packages/mkdocs_pandoc/pandoc_converter.py 2025-04-10 13:54:36.348159540 +0000 ++++ /venv/lib/python3.9/site-packages/mkdocs_pandoc/pandoc_converter.py 2025-04-10 13:55:01.268162893 +0000 +@@ -34,7 +34,7 @@ + raise FatalError("Couldn't open %s for reading: %s" % (self.config_file, + e.strerror), 1) + +- self.config = yaml.load(cfg) ++ self.config = yaml.full_load(cfg) + + if not 'docs_dir' in self.config: + self.config['docs_dir'] = 'docs' diff --git a/requirements.in b/requirements.in new file mode 100644 index 000000000..fd5795f95 --- /dev/null +++ b/requirements.in @@ -0,0 +1,8 @@ +mkdocs~=0.17.5 +mkdocs-bootstrap~=0.1.1 +mkdocs-bootswatch~=0.4 +BSCodeTabs +Markdown~=2.6.11 +MarkupSafe~=1.0 +mkdocs-pandoc +mkdocs-pandoc-plugin \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index dc48490de..b65c7ef4a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,54 @@ -mkdocs==0.15.1 +# +# This file is autogenerated by pip-compile with Python 3.9 +# by the following command: +# +# pip-compile --output-file=requirements.txt requirements.in +# +--index-url https://nexus.osdc.io/repository/pypi-all/simple +--trusted-host nexus.osdc.io + +beautifulsoup4==4.13.3 + # via mkdocs-pandoc-plugin +bscodetabs==1.0 + # via -r requirements.in +click==8.1.8 + # via mkdocs +jinja2==2.11.3 + # via mkdocs +livereload==2.7.1 + # via mkdocs +markdown==2.6.11 + # via + # -r requirements.in + # bscodetabs + # markdown-include + # mkdocs +markdown-include==0.6.0 + # via mkdocs-pandoc +markupsafe==1.1.1 + # via + # -r requirements.in + # jinja2 +mkdocs==0.17.5 + # via + # -r requirements.in + # mkdocs-pandoc + # mkdocs-pandoc-plugin mkdocs-bootstrap==0.1.1 -mkdocs-bootswatch==0.4 -BSCodeTabs -Markdown==2.6.11 -MarkupSafe==1.0 \ No newline at end of file + # via -r requirements.in +mkdocs-bootswatch==0.5.0 + # via -r requirements.in +mkdocs-pandoc==0.2.6 + # via -r requirements.in +mkdocs-pandoc-plugin==1.0.1 + # via -r requirements.in +pyyaml==6.0.2 + # via mkdocs +soupsieve==2.6 + # via beautifulsoup4 +tornado==4.5.3 + # via + # livereload + # mkdocs +typing-extensions==4.13.1 + # via beautifulsoup4 diff --git a/tables.patch b/tables.patch new file mode 100644 index 000000000..30ee91869 --- /dev/null +++ b/tables.patch @@ -0,0 +1,20 @@ +--- /venv/lib/python3.9/site-packages/mkdocs_pandoc/filters/tables.py 2025-04-10 09:11:33 ++++ /venv/lib/python3.9/site-packages/mkdocs_pandoc/filters/tables.py 2025-04-10 11:18:15 +@@ -88,7 +88,7 @@ + + # Initialize width arrays + +- for i in range(0, len(self._split_row(lines_orig[0], has_border))): ++ for i in range(0, len(self._split_row(lines_orig[0]))): + widest_cell.append(0) + widest_word.append(0) + widths.append(0) +@@ -96,7 +96,7 @@ + # Parse lines into array of cells and record width of widest cell/word + + for line in lines_orig: +- row = self._split_row(line, has_border) ++ row = self._split_row(line) + # pad widest_cell to account for under length first row + for i in range(0, len(row) - len(widest_cell)): + widest_cell.append(0) diff --git a/theme/404.html b/theme/404.html index a1a8abf40..ad00e30a6 100644 --- a/theme/404.html +++ b/theme/404.html @@ -19,7 +19,7 @@ {% if page_description %}{% endif %} {% if site_author %}{% endif %} - {% if canonical_url %}{% endif %} + {% if page.canonical_url %}{% endif %} {% if favicon %} {% else %}{% endif %} diff --git a/theme/base.html b/theme/base.html index 5b78f7d9a..a45467e88 100644 --- a/theme/base.html +++ b/theme/base.html @@ -5,20 +5,20 @@ - {% if current_page.title == config.extra.gdcmvs_viewer_page_id %} + {% if page.title == config.extra.gdcmvs_viewer_page_id %} {% else %} {% endif %} {% if page_description %}{% endif %} {% if site_author %}{% endif %} - {% if canonical_url %}{% endif %} + {% if page.canonical_url %}{% endif %} {% if favicon %} {% else %}{% endif %} - {% if page_title %}{{ page_title }} - {% endif %}{{ site_name }} + {% if page.title %}{{ page.title }} - {% endif %}{{ site_name }} @@ -43,14 +43,14 @@ {% endif %} {% endfor %} - {% if current_page.title == config.extra.gdcmvs_viewer_page_id %} + {% if page.title == config.extra.gdcmvs_viewer_page_id %} {% endif %} - {% if current_page.title == "Home" %} + {% if page.title == "Home" %} {% endif %} @@ -72,7 +72,7 @@ {% endif %} - {% if current_page.title == config.extra.dictionary_viewer_page_id %} + {% if page.title == config.extra.dictionary_viewer_page_id %} {% include "apps/dictionary/html-shards/dictionary-deps.html" %} {% endif %} @@ -86,7 +86,7 @@
- {% if current_page.title != config.extra.dictionary_viewer_page_id and current_page.title != config.extra.gdcmvs_viewer_page_id %} + {% if page.title != config.extra.dictionary_viewer_page_id and page.title != config.extra.gdcmvs_viewer_page_id %}
@@ -96,31 +96,31 @@ {% include "nav.html" %}
-
+
- {% if current_page.title == "Home" %} + {% if page.title == "Home" %} {% include "homepage.html" %} {% else %} - {% if "/Encyclopedia" in current_page.abs_url %} + {% if "/Encyclopedia" in page.abs_url %} {% include "encyclopedia/menu.html" %} - {% elif current_page.title != config.extra.dictionary_viewer_page_id and current_page.title != 'Home' and current_page.title != config.extra.gdcmvs_viewer_page_id %} + {% elif page.title != config.extra.dictionary_viewer_page_id and page.title != 'Home' and page.title != config.extra.gdcmvs_viewer_page_id %} {% include "nav-section-map.html" %} - {% elif current_page.title == 'Home' %} - {% if toc and ((toc | first) is defined) %} + {% elif page.title == 'Home' %} + {% if page.toc and ((page.toc | first) is defined) %} {% endif %} {% endif %} - {% if current_page.title == config.extra.dictionary_viewer_page_id or current_page.title == config.extra.gdcmvs_viewer_page_id %} + {% if page.title == config.extra.dictionary_viewer_page_id or page.title == config.extra.gdcmvs_viewer_page_id %}
{% else %}
{% endif %} - {% if current_page.title == config.extra.encyclopedia_page_title%} + {% if page.title == config.extra.encyclopedia_page_title%} {% include "encyclopedia/index.html" %} {% else %} {% include "content.html" %} @@ -128,12 +128,12 @@
{% if next_page or previous_page %} diff --git a/theme/content.html b/theme/content.html index a3726582b..ecf0a37e5 100644 --- a/theme/content.html +++ b/theme/content.html @@ -1,9 +1,9 @@ -{% if meta.source %} +{% if page.meta.source %} {% endif %} -{{ content }} +{{ page.content }} diff --git a/theme/nav-section-map-toc.html b/theme/nav-section-map-toc.html index 66fbaa22e..21b5abab9 100644 --- a/theme/nav-section-map-toc.html +++ b/theme/nav-section-map-toc.html @@ -1,5 +1,5 @@