diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 00000000..5bf4860b --- /dev/null +++ b/.editorconfig @@ -0,0 +1,26 @@ +root = true + +[*] +charset = utf-8 +insert_final_newline = true +trim_trailing_whitespace = true + +[*.md] +indent_size = 2 +indent_style = space +max_line_length = 100 # Please keep this in sync with bin/lesson_check.py! +trim_trailing_whitespace = false # keep trailing spaces in markdown - 2+ spaces are translated to a hard break (
) + +[*.r] +max_line_length = 80 + +[*.py] +indent_size = 4 +indent_style = space +max_line_length = 79 + +[*.sh] +end_of_line = lf + +[Makefile] +indent_style = tab diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index 6cc9e527..077de4cf 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -1,9 +1,21 @@ -Please delete the text below before submitting your contribution. +
+Instructions ---- +Thanks for contributing! :heart: -Thanks for contributing! If this contribution is for instructor training, please send an email to checkout@carpentries.org with a link to this contribution so we can record your progress. You’ve completed your contribution step for instructor checkout just by submitting this contribution. +If this contribution is for instructor training, please email the link to this contribution to +checkout@carpentries.org so we can record your progress. You've completed your contribution +step for instructor checkout by submitting this contribution! -Please keep in mind that lesson maintainers are volunteers and it may be some time before they can respond to your contribution. Although not all contributions can be incorporated into the lesson materials, we appreciate your time and effort to improve the curriculum. If you have any questions about the lesson maintenance process or would like to volunteer your time as a contribution reviewer, please contact Kate Hertweck (k8hertweck@gmail.com). +If this issue is about a specific episode within a lesson, please provide its link or filename. ---- +Keep in mind that **lesson maintainers are volunteers** and it may take them some time to +respond to your contribution. Although not all contributions can be incorporated into the lesson +materials, we appreciate your time and effort to improve the curriculum. If you have any questions +about the lesson maintenance process or would like to volunteer your time as a contribution +reviewer, please contact The Carpentries Team at team@carpentries.org. + +You may delete these instructions from your comment. + +\- The Carpentries +
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 6cc9e527..d3368998 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,9 +1,19 @@ -Please delete the text below before submitting your contribution. +
+Instructions ---- +Thanks for contributing! :heart: -Thanks for contributing! If this contribution is for instructor training, please send an email to checkout@carpentries.org with a link to this contribution so we can record your progress. You’ve completed your contribution step for instructor checkout just by submitting this contribution. +If this contribution is for instructor training, please email the link to this contribution to +instructor.training@carpentries.org so we can record your progress. You've completed your contribution +step for instructor checkout by submitting this contribution! -Please keep in mind that lesson maintainers are volunteers and it may be some time before they can respond to your contribution. Although not all contributions can be incorporated into the lesson materials, we appreciate your time and effort to improve the curriculum. If you have any questions about the lesson maintenance process or would like to volunteer your time as a contribution reviewer, please contact Kate Hertweck (k8hertweck@gmail.com). +Keep in mind that **lesson maintainers are volunteers** and it may take them some time to +respond to your contribution. Although not all contributions can be incorporated into the lesson +materials, we appreciate your time and effort to improve the curriculum. If you have any questions +about the lesson maintenance process or would like to volunteer your time as a contribution +reviewer, please contact The Carpentries Team at team@carpentries.org. ---- +You may delete these instructions from your comment. + +\- The Carpentries +
diff --git a/.github/workflows/template.yml b/.github/workflows/template.yml new file mode 100644 index 00000000..0665e0cf --- /dev/null +++ b/.github/workflows/template.yml @@ -0,0 +1,185 @@ +name: Test template +on: + push: + branches: gh-pages + pull_request: +jobs: + check-template: + name: ${{ matrix.lesson-name }} (${{ matrix.os-name }}) + if: github.repository == 'carpentries/styles' + runs-on: ${{ matrix.os }} + continue-on-error: ${{ matrix.experimental }} + strategy: + fail-fast: false + matrix: + lesson: [swcarpentry/shell-novice, datacarpentry/r-intro-geospatial, librarycarpentry/lc-git] + os: [ubuntu-20.04, macos-latest, windows-latest] + experimental: [false] + include: + - os: ubuntu-20.04 + os-name: Linux + - os: macos-latest + os-name: macOS + - os: windows-latest + os-name: Windows + - lesson: swcarpentry/shell-novice + lesson-name: (SWC) Shell novice + - lesson: datacarpentry/r-intro-geospatial + lesson-name: (DC) R Intro Geospatial + - lesson: librarycarpentry/lc-git + lesson-name: (LC) Intro to Git + - lesson: datacarpentry/astronomy-python + lesson-name: (DC) Foundations of Astronomical Data Science + experimental: true + os: ubuntu-20.04 + os-name: Linux + - lesson: carpentries/lesson-example + lesson-name: (CP) Lesson Example + experimental: false + os: ubuntu-20.04 + os-name: Linux + defaults: + run: + shell: bash # forces 'Git for Windows' on Windows + env: + RSPM: 'https://packagemanager.rstudio.com/cran/__linux__/focal/latest' + steps: + - name: Set up Ruby + uses: ruby/setup-ruby@v1 + with: + ruby-version: '2.7' + bundler-cache: true + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.x' + + - name: Install GitHub Pages, Bundler, and kramdown gems + run: | + gem install github-pages bundler kramdown kramdown-parser-gfm + + - name: Install Python modules + run: | + if [[ $RUNNER_OS == macOS || $RUNNER_OS == Linux ]]; then + python3 -m pip install --upgrade pip setuptools wheel pyyaml==5.3.1 requests + elif [[ $RUNNER_OS == Windows ]]; then + python -m pip install --upgrade pip setuptools wheel pyyaml==5.3.1 requests + fi + + - name: Checkout the ${{ matrix.lesson }} lesson + uses: actions/checkout@master + with: + repository: ${{ matrix.lesson }} + path: lesson + fetch-depth: 0 + + - name: Sync lesson with carpentries/styles + working-directory: lesson + run: | + echo "::group::Fetch Styles" + if [[ -n "${{ github.event.pull_request.number }}" ]] + then + ref="refs/pull/${{ github.event.pull_request.number }}/head" + else + ref="gh-pages" + fi + + git config --global user.email "team@carpentries.org" + git config --global user.name "The Carpentries Bot" + + git remote add styles https://github.com/carpentries/styles.git + git fetch styles $ref:styles-ref + echo "::endgroup::" + echo "::group::Synchronize Styles" + # Sync up only if necessary + if [[ $(git rev-list --count HEAD..styles-ref) != 0 ]] + then + + # The merge command below might fail for lessons that use remote theme + # https://github.com/carpentries/carpentries-theme + echo "Testing merge using recursive strategy, accepting upstream changes without committing" + if ! git merge -s recursive -Xtheirs --no-commit styles-ref + then + + # Remove "deleted by us, unmerged" files from the staging area. + # these are the files that were removed from the lesson + # but are still present in the carpentries/styles repo + echo "Removing previously deleted files" + git rm $(git diff --name-only --diff-filter=DU) + + # If there are still "unmerged" files, + # let's raise an error and look into this more closely + if [[ -n $(git diff --name-only --diff-filter=U) ]] + then + echo "There were unmerged files in ${{ matrix.lesson-name }}:" + echo "$(git diff --compact-summary --diff-filter=U)" + exit 1 + fi + fi + + echo "Committing changes" + git commit -m "Sync lesson with carpentries/styles" + fi + echo "::endgroup::" + + - name: Look for R-markdown files + id: check-rmd + working-directory: lesson + run: | + echo "count=$(shopt -s nullglob; files=($(find . -iname '*.Rmd')); echo ${#files[@]})" >> $GITHUB_OUTPUT + + - name: Set up R + if: steps.check-rmd.outputs.count != 0 + uses: r-lib/actions/setup-r@v2 + with: + use-public-rspm: true + install-r: false + + - name: Install needed packages + if: steps.check-rmd.outputs.count != 0 + working-directory: lesson + run: | + source('bin/dependencies.R') + install_required_packages(.libPaths()[1]) + shell: Rscript {0} + + - name: Query dependencies + if: steps.check-rmd.outputs.count != 0 + working-directory: lesson + run: | + source('bin/dependencies.R') + deps <- identify_dependencies() + create_description(deps) + use_bioc_repos() + saveRDS(remotes::dev_package_deps(dependencies = TRUE), ".github/depends.Rds", version = 2) + writeLines(sprintf("R-%i.%i", getRversion()$major, getRversion()$minor), ".github/R-version") + shell: Rscript {0} + + - name: Restore Package Cache + if: runner.os != 'Windows' && steps.check-rmd.outputs.count != 0 + uses: actions/cache@v2 + with: + path: ${{ env.R_LIBS_USER }} + key: ${{ runner.os }}-${{ hashFiles('.github/R-version') }}-1-${{ hashFiles('.github/depends.Rds') }} + restore-keys: ${{ runner.os }}-${{ hashFiles('.github/R-version') }}-1- + + - name: Install stringi from source + if: runner.os == 'Linux' && steps.check-rmd.outputs.count != 0 + run: install.packages('stringi', repos='https://cloud.r-project.org') + shell: Rscript {0} + + - name: Install system dependencies for R packages + if: runner.os == 'Linux' && steps.check-rmd.outputs.count != 0 + working-directory: lesson + run: | + while read -r cmd + do + eval sudo $cmd || echo "Nothing to update" + done < <(Rscript -e 'cat(remotes::system_requirements("ubuntu", "20.04"), sep = "\n")') + + - run: make site + working-directory: lesson + + - run: make lesson-check-all + working-directory: lesson diff --git a/.github/workflows/website.yml b/.github/workflows/website.yml new file mode 100644 index 00000000..be0b1875 --- /dev/null +++ b/.github/workflows/website.yml @@ -0,0 +1,123 @@ +name: Website +on: + push: + branches: + - gh-pages + - main + pull_request: [] +jobs: + build-website: + if: ${{ !endsWith(github.repository, '/styles') }} + runs-on: ubuntu-20.04 + env: + RSPM: "https://packagemanager.rstudio.com/cran/__linux__/focal/latest" + GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} + defaults: + run: + shell: bash + steps: + - name: Set up Ruby + uses: ruby/setup-ruby@v1 + with: + ruby-version: '2.7' + bundler-cache: true + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.x' + + - name: Install GitHub Pages, Bundler, and kramdown gems + run: | + gem install github-pages bundler kramdown kramdown-parser-gfm + + - name: Install Python modules + run: | + python3 -m pip install --upgrade pip setuptools wheel pyyaml==5.3.1 requests + + - name: Checkout the lesson + uses: actions/checkout@master + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha }} + + - name: Look for R-markdown files + id: check-rmd + run: | + echo "count=$(shopt -s nullglob; files=($(find . -iname '*.Rmd')); echo ${#files[@]})" >> $GITHUB_OUTPUT + + - name: Set up R + if: steps.check-rmd.outputs.count != 0 + uses: r-lib/actions/setup-r@v2 + with: + use-public-rspm: true + install-r: false + + - name: Restore R Cache + if: steps.check-rmd.outputs.count != 0 + uses: actions/cache@v2 + with: + path: ${{ env.R_LIBS_USER }} + key: ${{ runner.os }}-${{ hashFiles('.github/R-version') }}-1-${{ hashFiles('.github/depends.Rds') }} + restore-keys: ${{ runner.os }}-${{ hashFiles('.github/R-version') }}-1- + + - name: Install needed packages + if: steps.check-rmd.outputs.count != 0 + run: | + source('bin/dependencies.R') + install_required_packages() + shell: Rscript {0} + + - name: Query dependencies + if: steps.check-rmd.outputs.count != 0 + run: | + source('bin/dependencies.R') + deps <- identify_dependencies() + create_description(deps) + use_bioc_repos() + saveRDS(remotes::dev_package_deps(dependencies = TRUE), ".github/depends.Rds", version = 2) + writeLines(sprintf("R-%i.%i", getRversion()$major, getRversion()$minor), ".github/R-version") + shell: Rscript {0} + + + - name: Install system dependencies for R packages + if: steps.check-rmd.outputs.count != 0 + run: | + while read -r cmd + do + eval sudo $cmd || echo "Nothing to update" + done < <(Rscript -e 'cat(remotes::system_requirements("ubuntu", "20.04"), sep = "\n")') + + - name: Render the markdown and confirm that the site can be built + run: make site + + - name: Checkout github pages + if: ${{ github.event_name == 'push' && steps.check-rmd.outputs.count != 0 && github.ref != 'refs/heads/gh-pages'}} + uses: actions/checkout@master + with: + ref: gh-pages + path: gh-pages + + - name: Commit and Push + if: ${{ github.event_name == 'push' && steps.check-rmd.outputs.count != 0 && github.ref != 'refs/heads/gh-pages'}} + run: | + # clean up gh-pages + cd gh-pages + git rm -rf . # remove all previous files + git restore --staged . # remove things from the stage + cd .. + # copy everything into gh-pages site + cp -r `ls -A | grep -v 'gh-pages' | grep -v '.git' | grep -v '.bundle/' | grep -v '_site'` gh-pages + # move into gh-pages, add, commit, and push + cd gh-pages + # setup git + git config --local user.email "actions@github.com" + git config --local user.name "GitHub Actions" + git add -A . + git commit --allow-empty -m "[Github Actions] render website (via ${{ github.sha }})" + git push origin gh-pages + # return + cd .. + + - run: make lesson-check-all + if: always() diff --git a/.gitignore b/.gitignore index 1844e365..e3ce3b79 100644 --- a/.gitignore +++ b/.gitignore @@ -3,10 +3,16 @@ .DS_Store .ipynb_checkpoints .sass-cache +.jekyll-cache/ +.jekyll-metadata __pycache__ _site .Rproj.user .Rhistory .RData - -/Gemfile.lock +.bundle/ +.vendor/ +vendor/ +.docker-vendor/ +Gemfile.lock +.*history diff --git a/404.md b/404.md new file mode 100644 index 00000000..c1cf9dd4 --- /dev/null +++ b/404.md @@ -0,0 +1,25 @@ +--- +layout: base +root: . +permalink: 404.html +title: "Page not found" +--- + +# Oops! We cannot find that page. +{: style="text-align: center;"} + +> ## Our apologies! +> +> We cannot seem to find the page you are looking for. +> Try going back to the previous page or +> navigate to any other page using the navigation bar above +> {%- if site.kind == "lesson" -%} or the schedule below {%- endif -%}. +> If you got here by clicking on a link in the +> {%- if site.kind == "lesson" -%} lesson {%- else -%} workshop {%- endif -%}, +> please report this link to the +> {%- if site.kind == "lesson" -%} lesson developers {%- else -%} workshop organizers {%- endif -%}. +{: .caution} + +{% if site.kind == "lesson" %} + {% include syllabus.html %} +{% endif%} diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index c3b96690..2cf1133b 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -8,4 +8,5 @@ we pledge to follow the [Carpentry Code of Conduct][coc]. Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by following our [reporting guidelines][coc-reporting]. -{% include links.md %} +[coc]: https://docs.carpentries.org/topic_folders/policies/code-of-conduct.html +[coc-reporting]: https://docs.carpentries.org/topic_folders/policies/incident-reporting.html diff --git a/Gemfile b/Gemfile index fb4cf18c..8d69492b 100644 --- a/Gemfile +++ b/Gemfile @@ -1,3 +1,14 @@ +# frozen_string_literal: true + source 'https://rubygems.org' + +git_source(:github) { |repo_name| "https://github.com/#{repo_name}" } + +# Synchronize with https://pages.github.com/versions +ruby '>=2.7.1' + gem 'github-pages', group: :jekyll_plugins -gem 'wdm', '>= 0.1.0' + +if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new('3.0.0') + gem 'webrick', '>= 1.6.1' +end \ No newline at end of file diff --git a/LICENSE.md b/LICENSE.md index 8c24b2a0..d09b18ce 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -5,7 +5,7 @@ root: . --- ## Instructional Material -All Software Carpentry and Data Carpentry instructional material is +All Software Carpentry, Data Carpentry, and Library Carpentry instructional material is made available under the [Creative Commons Attribution license][cc-by-human]. The following is a human-readable summary of (and not a substitute for) the [full legal text of the CC BY 4.0 @@ -23,13 +23,15 @@ license terms. Under the following terms: -* **Attribution**---You must give appropriate credit (mentioning that - your work is derived from work that is Copyright © Software - Carpentry or Data Carpentry and, where practical, linking to - http://software-carpentry.org/ or https://datacarpentry.org), provide a [link to the - license][cc-by-human], and indicate if changes were made. You may do - so in any reasonable manner, but not in any way that suggests the - licensor endorses you or your use. +* **Attribution**---You must give appropriate credit by: + - mentioning that your work is derived from work that is + Copyright © Software Carpentry, Data Carpentry, Library Carpentry, + or The Carpentries. + - where practical, linking to the respective lesson program website + (https://software-carpentry.org/, https://datacarpentry.org, https://librarycarpentry.org, or + https://carpentries.org), provide a [link to the license][cc-by-human] + - and indicate if changes were made. You may do so in any reasonable manner, but not in any way + that suggests the licensor endorses you or your use. **No additional restrictions**---You may not apply legal terms or technological measures that legally restrict others from doing @@ -74,8 +76,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ## Trademark -"Software Carpentry" and "Data Carpentry" and their respective logos -are registered trademarks of [Community Initiatives][CI]. +"The Carpentries", "Software Carpentry" and "Data Carpentry" and their respective logos are +registered trademarks of [Community Initiatives][CI]. [cc-by-human]: https://creativecommons.org/licenses/by/4.0/ [cc-by-legal]: https://creativecommons.org/licenses/by/4.0/legalcode diff --git a/Makefile b/Makefile index 3ba06f60..6bf02c6d 100644 --- a/Makefile +++ b/Makefile @@ -3,66 +3,110 @@ # Settings MAKEFILES=Makefile $(wildcard *.mk) -JEKYLL=bundle exec jekyll -JEKYLL_VERSION=3.7.3 +JEKYLL=bundle config set --local path .vendor/bundle && bundle install && bundle update && bundle exec jekyll PARSER=bin/markdown_ast.rb DST=_site -# Controls -.PHONY : commands clean files -.NOTPARALLEL: -all : commands - -## commands : show all commands. -commands : - @grep -h -E '^##' ${MAKEFILES} | sed -e 's/## //g' - -## docker-serve : use docker to build the site -docker-serve : - docker run --rm -it -v ${PWD}:/srv/jekyll -p 127.0.0.1:4000:4000 jekyll/jekyll:${JEKYLL_VERSION} make serve - -## serve : run a local server. -serve : lesson-md +# Find Docker +DOCKER := $(shell which docker 2>/dev/null) + +# Check Python 3 is installed and determine if it's called via python3 or python +# (https://stackoverflow.com/a/4933395) +PYTHON3_EXE := $(shell which python3 2>/dev/null) +ifneq (, $(PYTHON3_EXE)) + ifeq (,$(findstring Microsoft/WindowsApps/python3,$(subst \,/,$(PYTHON3_EXE)))) + PYTHON := $(PYTHON3_EXE) + endif +endif + +ifeq (,$(PYTHON)) + PYTHON_EXE := $(shell which python 2>/dev/null) + ifneq (, $(PYTHON_EXE)) + PYTHON_VERSION_FULL := $(wordlist 2,4,$(subst ., ,$(shell python --version 2>&1))) + PYTHON_VERSION_MAJOR := $(word 1,${PYTHON_VERSION_FULL}) + ifeq (3, ${PYTHON_VERSION_MAJOR}) + PYTHON := $(PYTHON_EXE) + else + PYTHON_NOTE = "Your system does not appear to have Python 3 installed." + endif + else + PYTHON_NOTE = "Your system does not appear to have any Python installed." + endif +endif + + +# Default target +.DEFAULT_GOAL := commands + +## I. Commands for both workshop and lesson websites +## ================================================= + +.PHONY: site docker-serve repo-check clean clean-rmd + +## * serve : render website and run a local server +serve : lesson-md index.md ${JEKYLL} serve -## site : build files but do not run a server. -site : lesson-md +## * site : build website but do not run a server +site : lesson-md index.md ${JEKYLL} build -# repo-check : check repository settings. -repo-check : - @bin/repo_check.py -s . - -## clean : clean up junk files. +## * docker-serve : use Docker to serve the site +docker-serve : +ifeq (, $(DOCKER)) + $(error Your system does not appear to have Docker installed) +else + @$(DOCKER) pull carpentries/lesson-docker:latest + @$(DOCKER) run --rm -it \ + -v $${PWD}:/home/rstudio \ + -p 4000:4000 \ + -p 8787:8787 \ + -e USERID=$$(id -u) \ + -e GROUPID=$$(id -g) \ + carpentries/lesson-docker:latest +endif + +## * repo-check : check repository settings +repo-check : python + @${PYTHON} bin/repo_check.py -s . + +## * clean : clean up junk files clean : @rm -rf ${DST} @rm -rf .sass-cache @rm -rf bin/__pycache__ + @rm -rf .vendor + @rm -rf .bundle + @rm -f Gemfile.lock @find . -name .DS_Store -exec rm {} \; @find . -name '*~' -exec rm {} \; @find . -name '*.pyc' -exec rm {} \; -## clean-rmd : clean intermediate R files (that need to be committed to the repo). -clear-rmd : +## * clean-rmd : clean intermediate R files (that need to be committed to the repo) +clean-rmd : @rm -rf ${RMD_DST} @rm -rf fig/rmd-* -## ---------------------------------------- -## Commands specific to workshop websites. + +## +## II. Commands specific to workshop websites +## ================================================= .PHONY : workshop-check -## workshop-check : check workshop homepage. -workshop-check : - @bin/workshop_check.py . +## * workshop-check : check workshop homepage +workshop-check : python + @${PYTHON} bin/workshop_check.py . + -## ---------------------------------------- -## Commands specific to lesson websites. +## +## III. Commands specific to lesson websites +## ================================================= -.PHONY : lesson-check lesson-md lesson-files lesson-fixme +.PHONY : lesson-check lesson-md lesson-files lesson-fixme install-rmd-deps # RMarkdown files -RMD_SRC = $(wildcard _episodes_rmd/??-*.Rmd) +RMD_SRC = $(wildcard _episodes_rmd/*.Rmd) RMD_DST = $(patsubst _episodes_rmd/%.Rmd,_episodes/%.md,$(RMD_SRC)) # Lesson source files in the order they appear in the navigation menu. @@ -81,42 +125,64 @@ HTML_DST = \ ${DST}/conduct/index.html \ ${DST}/setup/index.html \ $(patsubst _episodes/%.md,${DST}/%/index.html,$(sort $(wildcard _episodes/*.md))) \ - ${DST}/reference/index.html \ + ${DST}/reference.html \ $(patsubst _extras/%.md,${DST}/%/index.html,$(sort $(wildcard _extras/*.md))) \ ${DST}/license/index.html -## lesson-md : convert Rmarkdown files to markdown +## * install-rmd-deps : Install R packages dependencies to build the RMarkdown lesson +install-rmd-deps: + @${SHELL} bin/install_r_deps.sh + +## * lesson-md : convert Rmarkdown files to markdown lesson-md : ${RMD_DST} -# Use of .NOTPARALLEL makes rule execute only once -${RMD_DST} : ${RMD_SRC} - @bin/knit_lessons.sh ${RMD_SRC} +_episodes/%.md: _episodes_rmd/%.Rmd install-rmd-deps + @mkdir -p _episodes + @$(SHELL) bin/knit_lessons.sh $< $@ -## lesson-check : validate lesson Markdown. -lesson-check : lesson-fixme - @bin/lesson_check.py -s . -p ${PARSER} -r _includes/links.md +## * lesson-check : validate lesson Markdown +lesson-check : python lesson-fixme + @${PYTHON} bin/lesson_check.py -s . -p ${PARSER} -r _includes/links.md -## lesson-check-all : validate lesson Markdown, checking line lengths and trailing whitespace. -lesson-check-all : - @bin/lesson_check.py -s . -p ${PARSER} -r _includes/links.md -l -w --permissive +## * lesson-check-all : validate lesson Markdown, checking line lengths and trailing whitespace +lesson-check-all : python + @${PYTHON} bin/lesson_check.py -s . -p ${PARSER} -r _includes/links.md -l -w --permissive -## unittest : run unit tests on checking tools. -unittest : - @bin/test_lesson_check.py +## * unittest : run unit tests on checking tools +unittest : python + @${PYTHON} bin/test_lesson_check.py -## lesson-files : show expected names of generated files for debugging. +## * lesson-files : show expected names of generated files for debugging lesson-files : @echo 'RMD_SRC:' ${RMD_SRC} @echo 'RMD_DST:' ${RMD_DST} @echo 'MARKDOWN_SRC:' ${MARKDOWN_SRC} @echo 'HTML_DST:' ${HTML_DST} -## lesson-fixme : show FIXME markers embedded in source files. +## * lesson-fixme : show FIXME markers embedded in source files lesson-fixme : - @fgrep -i -n FIXME ${MARKDOWN_SRC} || true + @grep --fixed-strings --word-regexp --line-number --no-messages FIXME ${MARKDOWN_SRC} || true + +## +## IV. Auxililary (plumbing) commands +## ================================================= -#------------------------------------------------------------------------------- -# Include extra commands if available. -#------------------------------------------------------------------------------- +.PHONY : commands python --include commands.mk +## * commands : show all commands. +commands : + @sed -n -e '/^##/s|^##[[:space:]]*||p' $(MAKEFILE_LIST) + +python : +ifeq (, $(PYTHON)) + $(error $(PYTHON_NOTE)) +else + @: +endif + +index.md : +ifeq (, $(wildcard index.md)) + $(error index.md not found) +else + @: +endif diff --git a/aio.md b/aio.md index a91fb0f5..8d1806d2 100644 --- a/aio.md +++ b/aio.md @@ -1,36 +1,13 @@ --- -layout: page -title: Aio +permalink: /aio/index.html --- - + {% comment %} -Create anchor for each one of the episodes. +As a maintainer, you do not need to edit this file. +If you notice that something does not work, please +open an issue: https://github.com/carpentries/styles/issues/new {% endcomment %} -{% for episode in site.episodes %} -
-{% endfor %} + +{% include base_path.html %} + +{% include aio-script.md %} diff --git a/bin/boilerplate/.travis.yml b/bin/boilerplate/.travis.yml deleted file mode 100644 index d30f78a4..00000000 --- a/bin/boilerplate/.travis.yml +++ /dev/null @@ -1,23 +0,0 @@ -# dist: trusty # Ubuntu 14.04 -language: python -python: 3.6 -branches: - only: - - gh-pages - - /.*/ -before_install: - - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E084DAB9 - - echo "deb https://cran.rstudio.com/bin/linux/ubuntu trusty/" | sudo tee -a /etc/apt/sources.list - - sudo apt-get update -y - - sudo apt-get install -y r-base - - sudo Rscript -e "install.packages('knitr', repos = 'https://', dependencies = TRUE)" - - sudo Rscript -e "install.packages('stringr', repos = 'https://cran.rstudio.com', dependencies = TRUE)" - - sudo Rscript -e "install.packages('checkpoint', repos = 'https://cran.rstudio.com', dependencies = TRUE)" - - sudo Rscript -e "install.packages('ggplot2', repos = 'https://cran.rstudio.com', dependencies = TRUE)" - - rvm default - - gem install json kramdown jekyll -install: - - pip install pyyaml -script: - - make lesson-check-all - - make --always-make site diff --git a/bin/boilerplate/CODE_OF_CONDUCT.md b/bin/boilerplate/CODE_OF_CONDUCT.md deleted file mode 100644 index c3b96690..00000000 --- a/bin/boilerplate/CODE_OF_CONDUCT.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -layout: page -title: "Contributor Code of Conduct" ---- -As contributors and maintainers of this project, -we pledge to follow the [Carpentry Code of Conduct][coc]. - -Instances of abusive, harassing, or otherwise unacceptable behavior -may be reported by following our [reporting guidelines][coc-reporting]. - -{% include links.md %} diff --git a/bin/boilerplate/CONTRIBUTING.md b/bin/boilerplate/CONTRIBUTING.md index 63d53e8f..8c095d86 100644 --- a/bin/boilerplate/CONTRIBUTING.md +++ b/bin/boilerplate/CONTRIBUTING.md @@ -1,6 +1,6 @@ # Contributing -[Software Carpentry][swc-site] and [Data Carpentry][dc-site] are open source projects, +[The Carpentries][c-site] ([Software Carpentry][swc-site], [Data Carpentry][dc-site], and [Library Carpentry][lc-site]) are open source projects, and we welcome contributions of all kinds: new lessons, fixes to existing material, @@ -14,7 +14,7 @@ you agree that we may redistribute your work under [our license](LICENSE.md). In exchange, we will address your issues and/or assess your change proposal as promptly as we can, and help you become a member of our community. -Everyone involved in [Software Carpentry][swc-site] and [Data Carpentry][dc-site] +Everyone involved in [The Carpentries][c-site] agrees to abide by our [code of conduct](CODE_OF_CONDUCT.md). ## How to Contribute @@ -70,16 +70,16 @@ There are many ways to contribute, from writing new exercises and improving existing ones to updating or filling in the documentation and submitting [bug reports][issues] -about things that don't work, aren't clear, or are missing. +about things that do not work, are not clear, or are missing. If you are looking for ideas, please see the 'Issues' tab for a list of issues associated with this repository, -or you may also look at the issues for [Data Carpentry][dc-issues] -and [Software Carpentry][swc-issues] projects. +or you may also look at the issues for [Data Carpentry][dc-issues], +[Software Carpentry][swc-issues], and [Library Carpentry][lc-issues] projects. Comments on issues and reviews of pull requests are just as welcome: we are smarter together than we are on our own. Reviews from novices and newcomers are particularly valuable: -it's easy for people who have been using these lessons for a while +it is easy for people who have been using these lessons for a while to forget how impenetrable some of this material can be, so fresh eyes are always welcome. @@ -94,8 +94,8 @@ and (b) explain what you would take out to make room for it. The first encourages contributors to be honest about requirements; the second, to think hard about priorities. -We are also not looking for exercises or other material that only run on one platform. -Our workshops typically contain a mixture of Windows, Mac OS X, and Linux users; +We are also not looking for exercises or other material that will only run on one platform. +Our workshops typically contain a mixture of Windows, macOS, and Linux users; in order to be usable, our lessons must run equally well on all three. @@ -104,7 +104,7 @@ our lessons must run equally well on all three. If you choose to contribute via GitHub, you may want to look at [How to Contribute to an Open Source Project on GitHub][how-contribute]. To manage changes, we follow [GitHub flow][github-flow]. -Each lesson has two maintainers who review issues and pull requests or encourage others to do so. +Each lesson has at least two maintainers who review issues and pull requests or encourage others to do so. The maintainers are community volunteers and have final say over what gets merged into the lesson. To use the web interface for contributing to a lesson: @@ -128,21 +128,24 @@ repository for reference while revising. ## Other Resources -General discussion of [Software Carpentry][swc-site] and [Data Carpentry][dc-site] +General discussion of [Software Carpentry][swc-site], [Data Carpentry][dc-site], and [Library Carpentry][lc-site] happens on the [discussion mailing list][discuss-list], which everyone is welcome to join. You can also [reach us by email][email]. -[email]: mailto:admin@software-carpentry.org +[email]: mailto:team@carpentries.org [dc-issues]: https://github.com/issues?q=user%3Adatacarpentry [dc-lessons]: http://datacarpentry.org/lessons/ [dc-site]: http://datacarpentry.org/ -[discuss-list]: http://lists.software-carpentry.org/listinfo/discuss +[discuss-list]: https://carpentries.topicbox.com/groups/discuss [github]: https://github.com [github-flow]: https://guides.github.com/introduction/flow/ [github-join]: https://github.com/join -[how-contribute]: https://egghead.io/series/how-to-contribute-to-an-open-source-project-on-github +[how-contribute]: https://app.egghead.io/playlists/how-to-contribute-to-an-open-source-project-on-github [issues]: https://guides.github.com/features/issues/ [swc-issues]: https://github.com/issues?q=user%3Aswcarpentry [swc-lessons]: https://software-carpentry.org/lessons/ [swc-site]: https://software-carpentry.org/ +[c-site]: https://carpentries.org/ +[lc-site]: https://librarycarpentry.org/ +[lc-issues]: https://github.com/issues?q=user%3Alibrarycarpentry diff --git a/bin/boilerplate/README.md b/bin/boilerplate/README.md index 43a6b722..060994ae 100644 --- a/bin/boilerplate/README.md +++ b/bin/boilerplate/README.md @@ -2,7 +2,7 @@ [![Create a Slack Account with us](https://img.shields.io/badge/Create_Slack_Account-The_Carpentries-071159.svg)](https://swc-slack-invite.herokuapp.com/) -FIXME +This repository generates the corresponding lesson website from [The Carpentries](https://carpentries.org/) repertoire of lessons. ## Contributing @@ -13,9 +13,21 @@ We'd like to ask you to familiarize yourself with our [Contribution Guide](CONTR the [more detailed guidelines][lesson-example] on proper formatting, ways to render the lesson locally, and even how to write new episodes. +Please see the current list of [issues][FIXME] for ideas for contributing to this +repository. For making your contribution, we use the GitHub flow, which is +nicely explained in the chapter [Contributing to a Project](http://git-scm.com/book/en/v2/GitHub-Contributing-to-a-Project) in Pro Git +by Scott Chacon. +Look for the tag ![good_first_issue](https://img.shields.io/badge/-good%20first%20issue-gold.svg). This indicates that the maintainers will welcome a pull request fixing this issue. + + ## Maintainer(s) +Current maintainers of this lesson are + * FIXME +* FIXME +* FIXME + ## Authors diff --git a/bin/boilerplate/_config.yml b/bin/boilerplate/_config.yml index a62f808b..f2ca3734 100644 --- a/bin/boilerplate/_config.yml +++ b/bin/boilerplate/_config.yml @@ -7,11 +7,33 @@ # dc: Data Carpentry # lc: Library Carpentry # cp: Carpentries (to use for instructor traning for instance) +# incubator: Carpentries Incubator carpentry: "swc" # Overall title for pages. title: "Lesson Title" +# Life cycle stage of the lesson +# See this page for more details: https://cdh.carpentries.org/the-lesson-life-cycle.html +# Possible values: "pre-alpha", "alpha", "beta", "stable" +# +# Lessons that are going through the transition to the +# Carpentries Workbench will go through 3 steps: +# 'transition-step-1': notice indicating a new version +# 'transition-step-2': notice encouraging to use new version +# 'transition-step-3': notice indicating the lesson is deprecated, +# with automated redirect +life_cycle: "pre-alpha" + +# For lessons in the life stages in 'transition-step-1' or later: +# - 'transition_url' holds the URL for the version of the lesson that +# uses the Workbench (needed for all 3 steps) +# - 'transition_date' (in yyyy-mm-dd format) is the date when the lesson +# will transition to being deprecated. The date only needs to be decided +# when the lesson is in 'transition-step-2'. +transition_url: +transition_date: + #------------------------------------------------------------ # Generic settings (should not need to change). #------------------------------------------------------------ @@ -28,14 +50,15 @@ repository: / email: "team@carpentries.org" # Sites. -amy_site: "https://amy.software-carpentry.org/workshops" +coc: "https://docs.carpentries.org/topic_folders/policies/code-of-conduct.html" +amy_site: "https://amy.carpentries.org/" carpentries_github: "https://github.com/carpentries" carpentries_pages: "https://carpentries.github.io" carpentries_site: "https://carpentries.org/" -dc_site: "http://datacarpentry.org" +dc_site: "https://datacarpentry.org" example_repo: "https://github.com/carpentries/lesson-example" example_site: "https://carpentries.github.io/lesson-example" -lc_site: "https://librarycarpentry.github.io/" +lc_site: "https://librarycarpentry.org/" swc_github: "https://github.com/swcarpentry" swc_pages: "https://swcarpentry.github.io" swc_site: "https://software-carpentry.org" @@ -43,11 +66,16 @@ template_repo: "https://github.com/carpentries/styles" training_site: "https://carpentries.github.io/instructor-training" workshop_repo: "https://github.com/carpentries/workshop-template" workshop_site: "https://carpentries.github.io/workshop-template" +cc_by_human: "https://creativecommons.org/licenses/by/4.0/" # Surveys. -pre_survey: "https://www.surveymonkey.com/r/swc_pre_workshop_v1?workshop_id=" -post_survey: "https://www.surveymonkey.com/r/swc_post_workshop_v1?workshop_id=" -training_post_survey: "https://www.surveymonkey.com/r/post-instructor-training" +pre_survey: "https://carpentries.typeform.com/to/wi32rS#slug=" +post_survey: "https://carpentries.typeform.com/to/UgVdRQ#slug=" +instructor_pre_survey: "https://carpentries.typeform.com/to/QVOarK#slug=" +instructor_post_survey: "https://carpentries.typeform.com/to/cjJ9UP#slug=" + +# Set to 'true' for instructor training websites only. +instructor_training: false # Start time in minutes (0 to be clock-independent, 540 to show a start at 09:00 am). start_time: 0 @@ -84,6 +112,9 @@ exclude: - Makefile - bin/ - .Rproj.user/ + - .vendor/ + - vendor/ + - .docker-vendor/ # Turn on built-in syntax highlighting. highlighter: rouge diff --git a/bin/boilerplate/_episodes/01-introduction.md b/bin/boilerplate/_episodes/01-introduction.md index fcfdda17..2e156c26 100644 --- a/bin/boilerplate/_episodes/01-introduction.md +++ b/bin/boilerplate/_episodes/01-introduction.md @@ -5,10 +5,11 @@ exercises: 0 questions: - "Key question (FIXME)" objectives: -- "First objective. (FIXME)" +- "First learning objective. (FIXME)" keypoints: -- "First key point. (FIXME)" +- "First key point. Brief Answer to questions. (FIXME)" --- FIXME {% include links.md %} + diff --git a/bin/boilerplate/_extras/figures.md b/bin/boilerplate/_extras/figures.md index c1511e8c..0012c88e 100644 --- a/bin/boilerplate/_extras/figures.md +++ b/bin/boilerplate/_extras/figures.md @@ -1,40 +1,79 @@ --- title: Figures --- + +{% include base_path.html %} +{% include manual_episode_order.html %} + -{% comment %} -Create anchor for each one of the episodes. -{% endcomment %} -{% for episode in site.episodes %} -
+ +{% comment %} Create anchor for each one of the episodes. {% endcomment %} + +{% for lesson_episode in lesson_episodes %} + {% if site.episode_order %} + {% assign episode = site.episodes | where: "slug", lesson_episode | first %} + {% else %} + {% assign episode = lesson_episode %} + {% endif %} +
{% endfor %} {% include links.md %} diff --git a/bin/boilerplate/aio.md b/bin/boilerplate/aio.md deleted file mode 100644 index a418f3b4..00000000 --- a/bin/boilerplate/aio.md +++ /dev/null @@ -1,34 +0,0 @@ ---- ---- - -{% comment %} -Create anchor for each one of the episodes. -{% endcomment %} -{% for episode in site.episodes %} -
-{% endfor %} diff --git a/bin/boilerplate/index.md b/bin/boilerplate/index.md index 7fb812c2..95ccdbdc 100644 --- a/bin/boilerplate/index.md +++ b/bin/boilerplate/index.md @@ -1,10 +1,14 @@ --- layout: lesson -root: . # Is the only page that don't follow the partner /:path/index.html -permalink: index.html # Is the only page that don't follow the partner /:path/index.html +root: . # Is the only page that doesn't follow the pattern /:path/index.html +permalink: index.html # Is the only page that doesn't follow the pattern /:path/index.html --- FIXME: home page introduction + + +{% comment %} This is a comment in Liquid {% endcomment %} + > ## Prerequisites > > FIXME diff --git a/bin/chunk-options.R b/bin/chunk-options.R index f9435842..8e0d62af 100644 --- a/bin/chunk-options.R +++ b/bin/chunk-options.R @@ -20,7 +20,7 @@ knitr_fig_path <- function(prefix) { opts_chunk$set(fig.path = new_path) } -## We use the rmd- prefix for the figures generated by the lssons so +## We use the rmd- prefix for the figures generated by the lessons so ## they can be easily identified and deleted by `make clean-rmd`. The ## working directory when the lessons are generated is the root so the ## figures need to be saved in fig/, but when the site is generated, @@ -29,30 +29,42 @@ knitr_fig_path <- function(prefix) { opts_chunk$set(tidy = FALSE, results = "markup", comment = NA, fig.align = "center", fig.path = "fig/rmd-", - fig.process = fix_fig_path) + fig.process = fix_fig_path, + fig.width = 8.5, fig.height = 8.5, + fig.retina = 2) # The hooks below add html tags to the code chunks and their output so that they # are properly formatted when the site is built. hook_in <- function(x, options) { + lg <- tolower(options$engine) + style <- paste0(".language-", lg) + stringr::str_c("\n\n~~~\n", - paste0(x, collapse="\n"), - "\n~~~\n{: .language-r}\n\n") + paste0(x, collapse="\n"), + "\n~~~\n{: ", style, "}\n\n") } hook_out <- function(x, options) { x <- gsub("\n$", "", x) stringr::str_c("\n\n~~~\n", - paste0(x, collapse="\n"), - "\n~~~\n{: .output}\n\n") + paste0(x, collapse="\n"), + "\n~~~\n{: .output}\n\n") } hook_error <- function(x, options) { x <- gsub("\n$", "", x) stringr::str_c("\n\n~~~\n", - paste0(x, collapse="\n"), - "\n~~~\n{: .error}\n\n") + paste0(x, collapse="\n"), + "\n~~~\n{: .error}\n\n") +} + +hook_warning <- function(x, options) { + x <- gsub("\n$", "", x) + stringr::str_c("\n\n~~~\n", + paste0(x, collapse = "\n"), + "\n~~~\n{: .warning}\n\n") } -knit_hooks$set(source = hook_in, output = hook_out, warning = hook_error, - error = hook_error, message = hook_out) +knit_hooks$set(source = hook_in, output = hook_out, warning = hook_warning, + error = hook_error, message = hook_out) diff --git a/bin/dependencies.R b/bin/dependencies.R new file mode 100644 index 00000000..4eeeb215 --- /dev/null +++ b/bin/dependencies.R @@ -0,0 +1,107 @@ +install_required_packages <- function(lib = NULL, repos = getOption("repos", default = c(CRAN = "https://cran.rstudio.com/"))) { + + if (is.null(lib)) { + lib <- .libPaths()[[1]] + } + + message("lib paths: ", paste(lib, collapse = ", ")) + # Note: RMarkdown is needed for renv to detect packages in Rmd documents. + required_pkgs <- c("rprojroot", "desc", "remotes", "renv", "BiocManager", "rmarkdown") + installed_pkgs <- rownames(installed.packages(lib.loc = lib)) + missing_pkgs <- setdiff(required_pkgs, installed_pkgs) + + # The default installation of R will have "@CRAN@" as the default repository, + # which directs contrib.url() to either force the user to choose a mirror if + # interactive or fail if not. Since we are not interactve, we need to force + # the mirror here. + if ("@CRAN@" %in% repos) { + repos <- c(CRAN = "https://cran.rstudio.com/") + } + + if (length(missing_pkgs) != 0) { + install.packages(missing_pkgs, lib = lib, repos = repos) + } +} + +find_root <- function() { + + cfg <- rprojroot::has_file_pattern("^_config.y*ml$") + root <- rprojroot::find_root(cfg) + + root +} + +# set the BiocManager repositories and return a function that resets the default +# repositories. +# +# @example +# bioc_repos_example <- function() { +# message("User repos") +# as.data.frame(getOption("repos")) +# reset_repos <- use_bioc_repos() +# on.exit(reset_repos()) +# message("Bioc repos") +# as.data.frame(getOption("repos")) +# } +# bioc_repos_example() +# as.data.frame(getOption("repos") +use_bioc_repos <- function() { + repos <- getOption("repos") + suppressMessages(options(repos = BiocManager::repositories())) + function() { + options(repos = repos) + } +} + +identify_dependencies <- function() { + + root <- find_root() + + reset_repos <- use_bioc_repos() + on.exit(reset_repos(), add = TRUE) + eps <- file.path(root, "_episodes_rmd") + bin <- file.path(root, "bin") + + required_pkgs <- unique(c( + ## Packages for episodes + renv::dependencies(eps, progress = FALSE, error = "ignored")$Package, + ## Packages for tools + renv::dependencies(bin, progress = FALSE, error = "ignored")$Package + )) + + required_pkgs +} + +create_description <- function(required_pkgs) { + d <- desc::description$new("!new") + d$set_deps(data.frame(type = "Imports", package = required_pkgs, version = "*")) + d$write("DESCRIPTION") + # We have to write the description twice to get the hidden dependencies + # because renv only considers explicit dependencies. + # + # This is needed because some of the hidden dependencis will require system + # libraries to be configured. + suppressMessages(repo <- BiocManager::repositories()) + deps <- remotes::dev_package_deps(dependencies = TRUE, repos = repo) + deps <- deps$package[deps$diff < 0] + if (length(deps)) { + # only create new DESCRIPTION file if there are dependencies to install + d$set_deps(data.frame(type = "Imports", package = deps, version = "*")) + d$write("DESCRIPTION") + } +} + +install_dependencies <- function(required_pkgs, ...) { + + reset_repos <- use_bioc_repos() + on.exit(reset_repos(), add = TRUE) + + create_description(required_pkgs) + on.exit(file.remove("DESCRIPTION"), add = TRUE) + remotes::install_deps(dependencies = TRUE, ...) + + if (require("knitr") && packageVersion("knitr") < '1.9.19') { + stop("knitr must be version 1.9.20 or higher") + } + +} diff --git a/bin/generate_md_episodes.R b/bin/generate_md_episodes.R index 6c27d9c9..7fb4c5a5 100644 --- a/bin/generate_md_episodes.R +++ b/bin/generate_md_episodes.R @@ -1,52 +1,44 @@ generate_md_episodes <- function() { - library("methods") - - if (require("knitr") && packageVersion("knitr") < '1.9.19') - stop("knitr must be version 1.9.20 or higher") + # avoid ansi color characters from being printed in the output + op <- options() + on.exit(options(op), add = TRUE) + options(crayon.enabled = FALSE) + ## get the Rmd file to process from the command line, and generate the path + ## for their respective outputs + args <- commandArgs(trailingOnly = TRUE) + if (!identical(length(args), 2L)) { + stop("input and output file must be passed to the script") + } - if (!require("stringr")) - stop("The package stringr is required for generating the lessons.") + src_rmd <- args[1] + dest_md <- args[2] - if (require("checkpoint") && packageVersion("checkpoint") >= '0.4.0') { - required_pkgs <- - checkpoint:::scanForPackages(project = "_episodes_rmd", - verbose=FALSE, use.knitr = TRUE)$pkgs - } else { - stop("The checkpoint package (>= 0.4.0) is required to build the lessons.") - } + ## knit the Rmd into markdown + knitr::knit(src_rmd, output = dest_md) - missing_pkgs <- required_pkgs[!(required_pkgs %in% rownames(installed.packages()))] + # Read the generated md files and add comments advising not to edit them + add_no_edit_comment <- function(y) { + con <- file(y) + mdfile <- readLines(con) + if (mdfile[1] != "---") + stop("Input file does not have a valid header") + mdfile <- append( + mdfile, + "# Please do not edit this file directly; it is auto generated.", + after = 1 + ) + mdfile <- append( + mdfile, + paste("# Instead, please edit", basename(y), "in _episodes_rmd/"), + after = 2 + ) + writeLines(mdfile, con) + close(con) + return(paste("Warning added to YAML header of", y)) + } - if (length(missing_pkgs)) { - message("Installing missing required packages: ", - paste(missing_pkgs, collapse=", ")) - install.packages(missing_pkgs) - } - - ## find all the Rmd files, and generate the paths for their respective outputs - src_rmd <- list.files(pattern = "??-*.Rmd$", path = "_episodes_rmd", full.names = TRUE) - dest_md <- file.path("_episodes", gsub("Rmd$", "md", basename(src_rmd))) - - ## knit the Rmd into markdown - mapply(function(x, y) { - knitr::knit(x, output = y) - }, src_rmd, dest_md) - - # Read the generated md files and add comments advising not to edit them - vapply(dest_md, function(y) { - con <- file(y) - mdfile <- readLines(con) - if (mdfile[1] != "---") - stop("Input file does not have a valid header") - mdfile <- append(mdfile, "# Please do not edit this file directly; it is auto generated.", after = 1) - mdfile <- append(mdfile, paste("# Instead, please edit", - basename(y), "in _episodes_rmd/"), after = 2) - writeLines(mdfile, con) - close(con) - return(paste("Warning added to YAML header of", y)) - }, - character(1)) + vapply(dest_md, add_no_edit_comment, character(1)) } generate_md_episodes() diff --git a/bin/install_r_deps.sh b/bin/install_r_deps.sh new file mode 100755 index 00000000..0280f241 --- /dev/null +++ b/bin/install_r_deps.sh @@ -0,0 +1 @@ +Rscript -e "source(file.path('bin', 'dependencies.R')); install_required_packages(); install_dependencies(identify_dependencies())" diff --git a/bin/knit_lessons.sh b/bin/knit_lessons.sh index 3a2395f3..141c136a 100755 --- a/bin/knit_lessons.sh +++ b/bin/knit_lessons.sh @@ -3,6 +3,6 @@ # Only try running R to translate files if there are some files present. # The Makefile passes in the names of files. -if [ $# -ne 0 ] ; then - Rscript -e "source('bin/generate_md_episodes.R')" +if [ $# -eq 2 ] ; then + Rscript -e "source('bin/generate_md_episodes.R')" "$@" fi diff --git a/bin/lesson_check.py b/bin/lesson_check.py old mode 100755 new mode 100644 index 8ed1bf89..86e42495 --- a/bin/lesson_check.py +++ b/bin/lesson_check.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python3 - """ Check lesson files and their contents. """ @@ -8,16 +6,22 @@ import os import glob import re +import sys from argparse import ArgumentParser -from util import (Reporter, read_markdown, load_yaml, check_unwanted_files, - require) +# This uses the `__all__` list in `util.py` to determine what objects to import +# see https://docs.python.org/3/tutorial/modules.html#importing-from-a-package +from util import * +from reporter import Reporter __version__ = '0.3' # Where to look for source Markdown files. SOURCE_DIRS = ['', '_episodes', '_extras'] +# Where to look for source Rmd files. +SOURCE_RMD_DIRS = ['_episodes_rmd'] + # Required files: each entry is ('path': YAML_required). # FIXME: We do not yet validate whether any files have the required # YAML headers, but should in the future. @@ -26,19 +30,19 @@ # specially. This list must include all the Markdown files listed in the # 'bin/initialize' script. REQUIRED_FILES = { - '%/CODE_OF_CONDUCT.md': True, - '%/CONTRIBUTING.md': False, - '%/LICENSE.md': True, - '%/README.md': False, - '%/_extras/discuss.md': True, - '%/_extras/guide.md': True, - '%/index.md': True, - '%/reference.md': True, - '%/setup.md': True, + 'CODE_OF_CONDUCT.md': True, + 'CONTRIBUTING.md': False, + 'LICENSE.md': True, + 'README.md': False, + os.path.join('_extras', 'discuss.md'): True, + os.path.join('_extras', 'guide.md'): True, + 'index.md': True, + 'reference.md': True, + 'setup.md': True, } # Episode filename pattern. -P_EPISODE_FILENAME = re.compile(r'/_episodes/(\d\d)-[-\w]+.md$') +P_EPISODE_FILENAME = re.compile(r'(\d\d)-[-\w]+.md$') # Pattern to match lines ending with whitespace. P_TRAILING_WHITESPACE = re.compile(r'\s+$') @@ -52,9 +56,27 @@ # Pattern to match reference links (to resolve internally-defined references). P_INTERNAL_LINK_DEF = re.compile(r'^\[([^\]]+)\]:\s*(.+)') +# Pattern to match {% include ... %} statements +P_INTERNAL_INCLUDE_LINK = re.compile(r'^{% include ([^ ]*) %}$') + +# Pattern to match image-only and link-only lines +P_LINK_IMAGE_LINE = re.compile(r''' + [> #]* # any number of '>', '#', and spaces + \W{,3} # up to 3 non-word characters + !? # ! or nothing + \[[^]]+\] # [any text] + [([] # ( or [ + [^])]+ # 1+ characters that are neither ] nor ) + [])] # ] or ) + (?:{:[^}]+})? # {:any text} or nothing + \W{,3} # up to 3 non-word characters + [ ]* # any number of spaces + \\?$ # \ or nothing + end of line''', re.VERBOSE) + # What kinds of blockquotes are allowed? KNOWN_BLOCKQUOTES = { 'callout', + 'caution', 'challenge', 'checklist', 'discussion', @@ -63,22 +85,17 @@ 'prereq', 'quotation', 'solution', - 'testimonial' + 'testimonial', + 'warning' } # What kinds of code fragments are allowed? +# Below we allow all 'language-*' code blocks KNOWN_CODEBLOCKS = { 'error', 'output', 'source', - 'language-bash', - 'html', - 'language-make', - 'language-matlab', - 'language-python', - 'language-r', - 'language-shell', - 'language-sql' + 'warning' } # What fields are required in teaching episode metadata? @@ -99,15 +116,31 @@ } # How long are lines allowed to be? +# Please keep this in sync with .editorconfig! MAX_LINE_LEN = 100 +# Contents of _config.yml +CONFIG = {} def main(): """Main driver.""" args = parse_args() args.reporter = Reporter() - check_config(args.reporter, args.source_dir) + + global CONFIG + config_file = os.path.join(args.source_dir, '_config.yml') + CONFIG = load_yaml(config_file) + CONFIG["config_file"] = config_file + + life_cycle = CONFIG.get('life_cycle', None) + # pre-alpha lessons should report without error + if life_cycle == "pre-alpha": + args.permissive = True + + check_config(args.reporter) + check_source_rmd(args.reporter, args.source_dir, args.parser) + args.references = read_references(args.reporter, args.reference_path) docs = read_all_markdown(args.source_dir, args.parser) @@ -118,8 +151,16 @@ def main(): checker.check() args.reporter.report() - if args.reporter.messages and not args.permissive: - exit(1) + if args.reporter.messages: + if args.permissive: + print("Problems detected but ignored (permissive mode).") + else: + print("Problems detected.") + sys.exit(1) + else: + print("No problems found.") + + return def parse_args(): @@ -156,63 +197,106 @@ def parse_args(): args, extras = parser.parse_known_args() require(args.parser is not None, - 'Path to Markdown parser not provided') + 'Path to Markdown parser not provided', + True) require(not extras, 'Unexpected trailing command-line arguments "{0}"'.format(extras)) return args - -def check_config(reporter, source_dir): +def check_config(reporter): """Check configuration file.""" - config_file = os.path.join(source_dir, '_config.yml') - config = load_yaml(config_file) - reporter.check_field(config_file, 'configuration', - config, 'kind', 'lesson') - reporter.check_field(config_file, 'configuration', - config, 'carpentry', ('swc', 'dc', 'lc', 'cp')) - reporter.check_field(config_file, 'configuration', config, 'title') - reporter.check_field(config_file, 'configuration', config, 'email') + reporter.check_field(CONFIG["config_file"], 'configuration', + CONFIG, 'kind', 'lesson') + reporter.check_field(CONFIG["config_file"], 'configuration', + CONFIG, 'carpentry', ('swc', 'dc', 'lc', 'cp', 'incubator')) + reporter.check_field(CONFIG["config_file"], 'configuration', CONFIG, 'title') + reporter.check_field(CONFIG["config_file"], 'configuration', CONFIG, 'email') for defaults in [ {'values': {'root': '.', 'layout': 'page'}}, {'values': {'root': '..', 'layout': 'episode'}, 'scope': {'type': 'episodes', 'path': ''}}, {'values': {'root': '..', 'layout': 'page'}, 'scope': {'type': 'extras', 'path': ''}} ]: - reporter.check(defaults in config.get('defaults', []), - 'configuration', - '"root" not set to "." in configuration') - + error_text = 'incorrect settings for: root "{0}" layout "{1}"' + root = defaults["values"]["root"] + layout = defaults["values"]["layout"] + error_message = error_text.format(root, layout) + + defaults_test = defaults in CONFIG.get('defaults', []) + reporter.check(defaults_test, 'configuration', error_message) + +def check_source_rmd(reporter, source_dir, parser): + """Check that Rmd episode files include `source: Rmd`""" + + episode_rmd_dir = [os.path.join(source_dir, d) for d in SOURCE_RMD_DIRS] + episode_rmd_files = [os.path.join(d, '*.Rmd') for d in episode_rmd_dir] + results = {} + for pat in episode_rmd_files: + for f in glob.glob(pat): + data = read_markdown(parser, f) + dy = data['metadata'] + if dy: + reporter.check_field(f, 'episode_rmd', + dy, 'source', 'Rmd') def read_references(reporter, ref_path): """Read shared file of reference links, returning dictionary of valid references {symbolic_name : URL} """ + if 'remote_theme' in CONFIG: + return {} + + if not ref_path: + raise Warning("No filename has been provided.") + result = {} urls_seen = set() - if ref_path: - with open(ref_path, 'r') as reader: - for (num, line) in enumerate(reader): - line_num = num + 1 - m = P_INTERNAL_LINK_DEF.search(line) - require(m, - '{0}:{1} not valid reference:\n{2}'.format(ref_path, line_num, line.rstrip())) - name = m.group(1) - url = m.group(2) - require(name, - 'Empty reference at {0}:{1}'.format(ref_path, line_num)) - reporter.check(name not in result, - ref_path, - 'Duplicate reference {0} at line {1}', - name, line_num) - reporter.check(url not in urls_seen, - ref_path, - 'Duplicate definition of URL {0} at line {1}', - url, line_num) - result[name] = url - urls_seen.add(url) + + with open(ref_path, 'r', encoding='utf-8') as reader: + for (num, line) in enumerate(reader, 1): + + # Skip empty lines + if len(line.strip()) == 0: + continue + + # Skip HTML comments + if line.strip().startswith(""): + continue + + # Skip Liquid's {% include ... %} lines + if P_INTERNAL_INCLUDE_LINK.search(line): + continue + + m = P_INTERNAL_LINK_DEF.search(line) + + message = '{}: {} not a valid reference: {}' + require(m, message.format(ref_path, num, line.rstrip())) + + name = m.group(1) + url = m.group(2) + + message = 'Empty reference at {0}:{1}' + require(name, message.format(ref_path, num)) + + unique_name = name not in result + unique_url = url not in urls_seen + + reporter.check(unique_name, + ref_path, + 'Duplicate reference name {0} at line {1}', + name, num) + + reporter.check(unique_url, + ref_path, + 'Duplicate definition of URL {0} at line {1}', + url, num) + + result[name] = url + urls_seen.add(url) + return result @@ -236,7 +320,7 @@ def check_fileset(source_dir, reporter, filenames_present): """Are all required files present? Are extraneous files present?""" # Check files with predictable names. - required = [p.replace('%', source_dir) for p in REQUIRED_FILES] + required = [os.path.join(source_dir, p) for p in REQUIRED_FILES] missing = set(required) - set(filenames_present) for m in missing: reporter.add(None, 'Missing required file {0}', m) @@ -246,7 +330,10 @@ def check_fileset(source_dir, reporter, filenames_present): for filename in filenames_present: if '_episodes' not in filename: continue - m = P_EPISODE_FILENAME.search(filename) + + # split path to check episode name + base_name = os.path.basename(filename) + m = P_EPISODE_FILENAME.search(base_name) if m and m.group(1): seen.append(m.group(1)) else: @@ -320,12 +407,19 @@ def check_line_lengths(self): """Check the raw text of the lesson body.""" if self.args.line_lengths: - over = [i for (i, l, n) in self.lines if ( - n > MAX_LINE_LEN) and (not l.startswith('!'))] - self.reporter.check(not over, + over_limit = [] + + for (i, l, n) in self.lines: + # Report lines that are longer than the suggested + # line length limit only if they're not + # link-only or image-only lines. + if n > MAX_LINE_LEN and not P_LINK_IMAGE_LINE.match(l): + over_limit.append(i) + + self.reporter.check(not over_limit, self.filename, - 'Line(s) are too long: {0}', - ', '.join([str(i) for i in over])) + 'Line(s) too long: {0}', + ', '.join([str(i) for i in over_limit])) def check_trailing_whitespace(self): """Check for whitespace at the ends of lines.""" @@ -353,7 +447,8 @@ def check_codeblock_classes(self): for node in self.find_all(self.doc, {'type': 'codeblock'}): cls = self.get_val(node, 'attr', 'class') - self.reporter.check(cls in KNOWN_CODEBLOCKS, + self.reporter.check(cls is not None and (cls in KNOWN_CODEBLOCKS or + cls.startswith('language-')), (self.filename, self.get_loc(node)), 'Unknown or missing code block type {0}', cls) @@ -483,6 +578,9 @@ def check_metadata_fields(self, expected): def check_reference_inclusion(self): """Check that links file has been included.""" + if 'remote_theme' in CONFIG: + return + if not self.args.reference_path: return @@ -520,8 +618,8 @@ def __init__(self, args, filename, metadata, metadata_len, text, lines, doc): (re.compile(r'README\.md'), CheckNonJekyll), (re.compile(r'index\.md'), CheckIndex), (re.compile(r'reference\.md'), CheckReference), - (re.compile(r'_episodes/.*\.md'), CheckEpisode), - (re.compile(r'aio\.md'), CheckNonJekyll), + # '.' below is what's passed on the command line via '-s' flag + (re.compile(os.path.join('.','_episodes', '[^/]*\.md')), CheckEpisode), (re.compile(r'.*\.md'), CheckGeneric) ] diff --git a/bin/lesson_initialize.py b/bin/lesson_initialize.py old mode 100755 new mode 100644 index a96ba383..79ec05cf --- a/bin/lesson_initialize.py +++ b/bin/lesson_initialize.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python3 - """Initialize a newly-created repository.""" @@ -8,19 +6,16 @@ import shutil BOILERPLATE = ( - '.travis.yml', 'AUTHORS', 'CITATION', - 'CODE_OF_CONDUCT.md', 'CONTRIBUTING.md', 'README.md', '_config.yml', - '_episodes/01-introduction.md', - '_extras/about.md', - '_extras/discuss.md', - '_extras/figures.md', - '_extras/guide.md', - 'aio.md', + os.path.join('_episodes', '01-introduction.md'), + os.path.join('_extras', 'about.md'), + os.path.join('_extras', 'discuss.md'), + os.path.join('_extras', 'figures.md'), + os.path.join('_extras', 'guide.md'), 'index.md', 'reference.md', 'setup.md', @@ -43,7 +38,7 @@ def main(): # Create. for path in BOILERPLATE: shutil.copyfile( - "bin/boilerplate/{}".format(path), + os.path.join('bin', 'boilerplate', path), path ) diff --git a/bin/markdown_ast.rb b/bin/markdown_ast.rb index c3fd0b5e..2ef3f772 100755 --- a/bin/markdown_ast.rb +++ b/bin/markdown_ast.rb @@ -1,11 +1,13 @@ #!/usr/bin/env ruby +# frozen_string_literal: true # Use Kramdown parser to produce AST for Markdown document. -require "kramdown" -require "json" +require 'kramdown' +require 'kramdown-parser-gfm' +require 'json' -markdown = STDIN.read() -doc = Kramdown::Document.new(markdown) +markdown = $stdin.read +doc = Kramdown::Document.new(markdown, input: 'GFM', hard_wrap: false) tree = doc.to_hash_a_s_t puts JSON.pretty_generate(tree) diff --git a/bin/repo_check.py b/bin/repo_check.py old mode 100755 new mode 100644 index af4b7823..6988ca59 --- a/bin/repo_check.py +++ b/bin/repo_check.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python3 - """ Check repository settings. """ @@ -11,7 +9,8 @@ import re from argparse import ArgumentParser -from util import Reporter, require +from util import require +from reporter import Reporter # Import this way to produce a more useful error message. try: @@ -22,7 +21,7 @@ # Pattern to match Git command-line output for remotes => (user name, project name). -P_GIT_REMOTE = re.compile(r'upstream\s+[^:]+:([^/]+)/([^.]+)\.git\s+\(fetch\)') +P_GIT_REMOTE = re.compile(r'upstream\s+(?:https://|git@)github.com[:/]([^/]+)/([^.]+)(\.git)?\s+\(fetch\)') # Repository URL format string. F_REPO_URL = 'https://github.com/{0}/{1}/' @@ -104,7 +103,7 @@ def get_repo_url(repo_url): # Guess. cmd = 'git remote -v' p = Popen(cmd, shell=True, stdin=PIPE, stdout=PIPE, - close_fds=True, universal_newlines=True) + close_fds=True, universal_newlines=True, encoding='utf-8') stdout_data, stderr_data = p.communicate() stdout_data = stdout_data.split('\n') matches = [P_GIT_REMOTE.match(line) for line in stdout_data] diff --git a/bin/reporter.py b/bin/reporter.py new file mode 100644 index 00000000..550dbf05 --- /dev/null +++ b/bin/reporter.py @@ -0,0 +1,75 @@ +import sys + +class Reporter: + """Collect and report errors.""" + + # Marker to show that an expected value hasn't been provided. + # (Can't use 'None' because that might be a legitimate value.) + _DEFAULT_REPORTER = [] + + def __init__(self): + """Constructor.""" + self.messages = [] + + def check_field(self, filename, name, values, key, expected=_DEFAULT_REPORTER): + """Check that a dictionary has an expected value.""" + + if key not in values: + self.add(filename, '{0} does not contain {1}', name, key) + elif expected is self._DEFAULT_REPORTER: + pass + elif type(expected) in (tuple, set, list): + if values[key] not in expected: + self.add( + filename, '{0} {1} value {2} is not in {3}', name, key, values[key], expected) + elif values[key] != expected: + self.add(filename, '{0} {1} is {2} not {3}', + name, key, values[key], expected) + + def check(self, condition, location, fmt, *args): + """Append error if condition not met.""" + + if not condition: + self.add(location, fmt, *args) + + def add(self, location, fmt, *args): + """Append error unilaterally.""" + + self.messages.append((location, fmt.format(*args))) + + @staticmethod + def pretty(item): + location, message = item + if isinstance(location, type(None)): + return message + elif isinstance(location, str): + return location + ': ' + message + elif isinstance(location, tuple): + return '{0}:{1}: '.format(*location) + message + + print('Unknown item "{0}"'.format(item), file=sys.stderr) + return NotImplemented + + @staticmethod + def key(item): + location, message = item + if isinstance(location, type(None)): + return ('', -1, message) + elif isinstance(location, str): + return (location, -1, message) + elif isinstance(location, tuple): + return (location[0], location[1], message) + + print('Unknown item "{0}"'.format(item), file=sys.stderr) + return NotImplemented + + def report(self, stream=sys.stdout): + """Report all messages in order.""" + + if not self.messages: + return + + for m in sorted(self.messages, key=self.key): + print(self.pretty(m), file=stream) + + diff --git a/bin/run-make-docker-serve.sh b/bin/run-make-docker-serve.sh new file mode 100755 index 00000000..1e091781 --- /dev/null +++ b/bin/run-make-docker-serve.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +set -o errexit +set -o pipefail +set -o nounset + + +bundle install +bundle update +exec bundle exec jekyll serve --host 0.0.0.0 diff --git a/bin/test_lesson_check.py b/bin/test_lesson_check.py old mode 100755 new mode 100644 index 960059e8..7a6d603a --- a/bin/test_lesson_check.py +++ b/bin/test_lesson_check.py @@ -1,21 +1,17 @@ -#!/usr/bin/env python3 - import unittest import lesson_check -import util +import reporter class TestFileList(unittest.TestCase): def setUp(self): - self.reporter = util.Reporter() # TODO: refactor reporter class. + self.reporter = reporter.Reporter() # TODO: refactor reporter class. def test_file_list_has_expected_entries(self): # For first pass, simply assume that all required files are present - all_filenames = [filename.replace('%', '') - for filename in lesson_check.REQUIRED_FILES] - lesson_check.check_fileset('', self.reporter, all_filenames) + lesson_check.check_fileset('', self.reporter, lesson_check.REQUIRED_FILES) self.assertEqual(len(self.reporter.messages), 0) diff --git a/bin/util.py b/bin/util.py index 522f5dfc..1398c378 100644 --- a/bin/util.py +++ b/bin/util.py @@ -10,94 +10,13 @@ print('Unable to import YAML module: please install PyYAML', file=sys.stderr) sys.exit(1) - -# Things an image file's name can end with. -IMAGE_FILE_SUFFIX = { - '.gif', - '.jpg', - '.png', - '.svg' -} +__all__ = ['check_unwanted_files', 'load_yaml', 'read_markdown', 'require'] # Files that shouldn't be present. UNWANTED_FILES = [ '.nojekyll' ] -# Marker to show that an expected value hasn't been provided. -# (Can't use 'None' because that might be a legitimate value.) -REPORTER_NOT_SET = [] - - -class Reporter: - """Collect and report errors.""" - - def __init__(self): - """Constructor.""" - self.messages = [] - - def check_field(self, filename, name, values, key, expected=REPORTER_NOT_SET): - """Check that a dictionary has an expected value.""" - - if key not in values: - self.add(filename, '{0} does not contain {1}', name, key) - elif expected is REPORTER_NOT_SET: - pass - elif type(expected) in (tuple, set, list): - if values[key] not in expected: - self.add( - filename, '{0} {1} value {2} is not in {3}', name, key, values[key], expected) - elif values[key] != expected: - self.add(filename, '{0} {1} is {2} not {3}', - name, key, values[key], expected) - - def check(self, condition, location, fmt, *args): - """Append error if condition not met.""" - - if not condition: - self.add(location, fmt, *args) - - def add(self, location, fmt, *args): - """Append error unilaterally.""" - - self.messages.append((location, fmt.format(*args))) - - @staticmethod - def pretty(item): - location, message = item - if isinstance(location, type(None)): - return message - elif isinstance(location, str): - return location + ': ' + message - elif isinstance(location, tuple): - return '{0}:{1}: '.format(*location) + message - - print('Unknown item "{0}"'.format(item), file=sys.stderr) - return NotImplemented - - @staticmethod - def key(item): - location, message = item - if isinstance(location, type(None)): - return ('', -1, message) - elif isinstance(location, str): - return (location, -1, message) - elif isinstance(location, tuple): - return (location[0], location[1], message) - - print('Unknown item "{0}"'.format(item), file=sys.stderr) - return NotImplemented - - def report(self, stream=sys.stdout): - """Report all messages in order.""" - - if not self.messages: - return - - for m in sorted(self.messages, key=self.key): - print(self.pretty(m), file=stream) - - def read_markdown(parser, path): """ Get YAML and AST for Markdown file, returning @@ -105,7 +24,7 @@ def read_markdown(parser, path): """ # Split and extract YAML (if present). - with open(path, 'r') as reader: + with open(path, 'r', encoding='utf-8') as reader: body = reader.read() metadata_raw, metadata_yaml, body = split_metadata(path, body) @@ -115,9 +34,9 @@ def read_markdown(parser, path): for (i, line) in enumerate(body.split('\n'))] # Parse Markdown. - cmd = 'ruby {0}'.format(parser) + cmd = 'bundle exec ruby {0}'.format(parser) p = Popen(cmd, shell=True, stdin=PIPE, stdout=PIPE, - close_fds=True, universal_newlines=True) + close_fds=True, universal_newlines=True, encoding='utf-8') stdout_data, stderr_data = p.communicate(body) doc = json.loads(stdout_data) @@ -144,11 +63,10 @@ def split_metadata(path, text): metadata_raw = pieces[1] text = pieces[2] try: - metadata_yaml = yaml.load(metadata_raw) + metadata_yaml = yaml.load(metadata_raw, Loader=yaml.SafeLoader) except yaml.YAMLError as e: - print('Unable to parse YAML header in {0}:\n{1}'.format( - path, e), file=sys.stderr) - sys.exit(1) + message = 'Unable to parse YAML header in {0}:\n{1}' + print(message.format(path, e), file=sys.stderr) return metadata_raw, metadata_yaml, text @@ -160,13 +78,16 @@ def load_yaml(filename): """ try: - with open(filename, 'r') as reader: - return yaml.load(reader) - except (yaml.YAMLError, IOError) as e: - print('Unable to load YAML file {0}:\n{1}'.format( - filename, e), file=sys.stderr) - sys.exit(1) + with open(filename, 'r', encoding='utf-8') as reader: + return yaml.load(reader, Loader=yaml.SafeLoader) + except yaml.YAMLError as e: + message = 'ERROR: Unable to load YAML file {0}:\n{1}' + print(message.format(filename, e), file=sys.stderr) + except (FileNotFoundError, IOError): + message = 'ERROR: File {} not found' + print(message.format(filename), file=sys.stderr) + return {} def check_unwanted_files(dir_path, reporter): """ @@ -180,9 +101,11 @@ def check_unwanted_files(dir_path, reporter): "Unwanted file found") -def require(condition, message): +def require(condition, message, fatal=False): """Fail if condition not met.""" if not condition: print(message, file=sys.stderr) - sys.exit(1) + + if fatal: + sys.exit(1) diff --git a/bin/workshop_check.py b/bin/workshop_check.py old mode 100755 new mode 100644 index bd15210a..312b1a1d --- a/bin/workshop_check.py +++ b/bin/workshop_check.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python3 - '''Check that a workshop's index.html metadata is valid. See the docstrings on the checking functions for a summary of the checks. ''' @@ -9,7 +7,8 @@ import os import re from datetime import date -from util import Reporter, split_metadata, load_yaml, check_unwanted_files +from util import split_metadata, load_yaml, check_unwanted_files +from reporter import Reporter # Metadata field patterns. EMAIL_PATTERN = r'[^@]+@[^@]+\.[^@]+' @@ -18,8 +17,8 @@ URL_PATTERN = r'https?://.+' # Defaults. -CARPENTRIES = ("dc", "swc") -DEFAULT_CONTACT_EMAIL = 'admin@software-carpentry.org' +CARPENTRIES = ("dc", "swc", "lc", "cp") +DEFAULT_CONTACT_EMAIL = 'team@carpentries.org' USAGE = 'Usage: "workshop_check.py path/to/root/directory"' @@ -91,7 +90,7 @@ def check_layout(layout): @look_for_fixme def check_carpentry(layout): - '''"carpentry" in YAML header must be "dc" or "swc".''' + '''"carpentry" in YAML header must be "dc", "swc", "lc", or "cp".''' return layout in CARPENTRIES @@ -117,7 +116,7 @@ def check_humandate(date): and 4-digit year. Examples include 'Feb 18-20, 2025' and 'Feb 18 and 20, 2025'. It may be in languages other than English, but the month name should be kept short to aid formatting of the main - Software Carpentry web site. + Carpentries web site. """ if ',' not in date: @@ -390,7 +389,7 @@ def check_config(reporter, filename): kind) carpentry = config.get('carpentry', None) - reporter.check(carpentry in ('swc', 'dc'), + reporter.check(carpentry in ('swc', 'dc', 'lc', 'cp'), filename, 'Missing or unknown carpentry: {0}', carpentry) @@ -410,7 +409,7 @@ def main(): reporter = Reporter() check_config(reporter, config_file) check_unwanted_files(root_dir, reporter) - with open(index_file) as reader: + with open(index_file, encoding='utf-8') as reader: data = reader.read() check_file(reporter, index_file, data) reporter.report()