Update Makefile & scripts from upstream

.github/workflows/test_and_build.yml

@@ -20,7 +20,7 @@ jobs:
         pip3 install codespell
     - name: Check spelling
       run: |
-        codespell --skip="assets,*.svg,bin" --quiet-level=2  -L "rouge,dropse,namd,hist"
+        make spellcheck
 
   check_lesson_and_build:
     runs-on: ubuntu-latest
@@ -37,6 +37,12 @@ jobs:
         bundle config set path '.vendor/bundle'
         bundle config build.nokogiri --use-system-libraries
         bundle install
+    - name: "Lint episode markdown"
+      run: |
+        find _episodes -name \*.md -exec bundle exec mdl -r MD001,MD003,MD005,MD006,MD007,MD008,MD009,MD010,MD011,MD012,MD015,MD016,MD017,MD018,MD019,MD020,MD021,MD022,MD023,MD025,MD035,MD036,MD037,MD038,MD039,MD046 {} \;
+    - name: "Lint snippet markdown"
+      run: |
+        find _includes/snippets -name \*.md -exec bundle exec mdl -r MD001,MD003,MD005,MD006,MD007,MD008,MD009,MD010,MD011,MD012,MD015,MD016,MD017,MD018,MD019,MD020,MD021,MD022,MD023,MD025,MD035,MD036,MD037,MD038,MD039,MD046 {} \;
     - name: "Check lesson for warnings"
       run: |
         make lesson-check-all

.gitignore

@@ -1,6 +1,7 @@
 *.pyc
 *~
 .DS_Store
+.bundle
 .ipynb_checkpoints
 .jekyll-cache
 .sass-cache

Gemfile

@@ -1,3 +1,4 @@
 source "https://rubygems.org"
 gem "github-pages", group: :jekyll_plugins
 gem "kramdown-parser-gfm"
+gem "mdl"

Makefile

@@ -6,7 +6,8 @@ export SHELL = /bin/bash
 
 # Settings
 MAKEFILES=Makefile $(wildcard *.mk)
-JEKYLL=bundle config set path '.vendor/bundle' && bundle install && bundle update && bundle exec jekyll
+JEKYLL=bundle config --local set path .vendor/bundle && bundle install && bundle update && bundle exec jekyll
+HPC_JEKYLL_CONFIG?=
 PARSER=bin/markdown_ast.rb
 DST=_site
 
@@ -45,11 +46,11 @@ endif
 
 ## * serve            : render website and run a local server
 serve : lesson-md
-	${JEKYLL} serve
+	${JEKYLL} serve --config _config.yml,${HPC_JEKYLL_CONFIG}
 
 ## * site             : build website but do not run a server
 site : lesson-md
-	${JEKYLL} build
+	${JEKYLL} build --config _config.yml,${HPC_JEKYLL_CONFIG}
 
 ## * docker-serve     : use Docker to serve the site
 docker-serve :
@@ -96,7 +97,7 @@ workshop-check :
 ## III. Commands specific to lesson websites
 ## =================================================
 
-.PHONY : lesson-check lesson-md lesson-files lesson-fixme
+.PHONY : lesson-check lesson-md lesson-files lesson-fixme spellcheck
 
 # RMarkdown files
 RMD_SRC = $(wildcard _episodes_rmd/??-*.Rmd)
@@ -151,6 +152,10 @@ lesson-files :
 lesson-fixme :
 	@fgrep -i -n FIXME ${MARKDOWN_SRC} || true
 
+## spellcheck
+spellcheck:
+	codespell --skip="assets,*.svg,.vendor" --quiet-level=2  -L "dropse,hist,namd,rouge"
+
 ##
 ## IV. Auxililary (plumbing) commands
 ## =================================================

_config.yml

@@ -2,6 +2,8 @@
 # Values for this lesson.
 #------------------------------------------------------------
 
+snippets: "/snippets"
+
 # Which carpentry is this ("swc", "dc", "lc", or "cp")?
 # swc: Software Carpentry
 # dc: Data Carpentry

_episodes/00-hpc-intro.md

@@ -33,8 +33,8 @@ laptop computer where they started:
   their laptop will take over a month!
 * A genomics researcher has been using small datasets of sequence data, but
   soon will be receiving a new type of sequencing data that is 10 times as
-  large. It's already challenging to open the datasets on their computer —
-  analyzing these larger datasets will probably crash it.
+  large. It's already challenging to open the datasets on their computer
+  — analyzing these larger datasets will probably crash it.
 * An engineer is using a fluid dynamics package that has an option to run in
   parallel. So far, they haven't used this option on their desktop, but in
   going from 2D to 3D simulations, simulation time has more than tripled and it

_episodes/01-connecting.md

@@ -1,6 +1,6 @@
 ---
 title: "Connecting to the remote HPC system"
-teaching: 20 
+teaching: 20
 exercises: 10
 questions:
 - How do I open a terminal?

_episodes/02-navigation.md

@@ -1,6 +1,6 @@
 ---
 title: "Moving around and looking at things"
-teaching: 15 
+teaching: 15
 exercises: 5
 questions:
 - How do I navigate and look around the system?
@@ -74,7 +74,7 @@ $ pwd
 ~~~
 {: .output}
 
-So, we know where we are. How do we look and see what's in our current 
+So, we know where we are. How do we look and see what's in our current
 directory?
 ```
 $ ls
@@ -382,7 +382,7 @@ Mandatory arguments to long options are mandatory for short options too.
 > [remote]$ ls -j
 > ~~~
 > {: .language-bash}
-> 
+>
 > ~~~
 > ls: invalid option -- 'j'
 > Try 'ls --help' for more information.
@@ -417,7 +417,7 @@ Mandatory arguments to long options are mandatory for short options too.
 > > 2. No: `/` stands for the root directory.
 > > 3. No: Amanda's home directory is `/Users/amanda`.
 > > 4. No: this goes up two levels, i.e. ends in `/Users`.
-> > 5. Yes: `~` stands for the user's home directory, in this case 
+> > 5. Yes: `~` stands for the user's home directory, in this case
 > >    `/Users/amanda`.
 > > 6. No: this would navigate into a directory `home` in the current directory
 > >    if it exists.

_episodes/03-files.md

@@ -31,7 +31,6 @@ $ cd hpc-test
 ```
 {: .language-bash}
 
-
 ## Creating and Editing Text Files
 
 When working on an HPC system, we will frequently need to create or edit text
@@ -72,7 +71,6 @@ editor. Here are the shortcut keys for a few common actions:
 * <kbd>Ctrl</kbd>+<kbd>U</kbd> &mdash; paste the cut text line (or lines). This
   command can be repeated to paste the same text elsewhere.
 
-
 > ## Using `vim` as a text editor
 >
 > From time to time, you may encounter the `vim` text editor. Although `vim`
@@ -104,7 +102,6 @@ draft.txt
 ```
 {: .output}
 
-
 ## Reading Files
 
 Let's read the file we just created now. There are a few different ways of
@@ -146,7 +143,6 @@ it's "share and thrive".
 > print the contents of `draft.txt`, `chap1.txt`, and `chap2.txt`.
 {: .challenge}
 
-
 ## Creating Directory
 
 We've successfully created a file. What about a directory? We've actually done
@@ -162,7 +158,6 @@ draft.txt  files
 ```
 {: .output}
 
-
 ## Moving, Renaming, Copying Files
 
 **Moving** &mdash; We will move `draft.txt` to the `files` directory with `mv`

_episodes/04-wildcards-pipes.md

@@ -22,7 +22,7 @@ keypoints:
 >
 > If you didn't get them in the last lesson, make sure to download the example
 > files used in the next few sections:
-> 
+>
 > **Using wget**: `wget {{site.url}}{{site.baseurl}}/files/bash-lesson.tar.gz`
 >
 > **Using a web browser**:
@@ -145,12 +145,12 @@ $ wc -l *
 > file with "fb" in it?
 >
 > > ## Solution
-> > 
+> >
 > > ```
 > > wc -l *fb*
 > > ```
 > > {: .language-bash}
-> > 
+> >
 > > i.e. *anything or nothing* then `fb` then *anything or nothing*
 > {: .solution}
 {: .challenge}
@@ -161,7 +161,7 @@ $ wc -l *
 > "fastq" and move all of our .fastq files there in one `mv` command.
 >
 > > ## Solution
-> > 
+> >
 > > ```
 > > mkdir fastq
 > > mv *.fastq fastq/
@@ -318,14 +318,7 @@ The output is nearly unintelligible since there is so much of it. Let's send
 the output of that `grep` command to `head` so we can just take a peek at the
 first line. The `|` operator lets us send output from one command to the next:
 
-```
-$ grep Act5C dmel-all-r6.19.gtf | head -n 1
-```
-{: .language-bash}
-```
-X	FlyBase	gene	5900861	5905399	.	+	.	gene_id "FBgn0000042"; gene_symbol "Act5C";
-```
-{: .output}
+{% include {{ site.snippets }}/04/grep-act5c.snip %}
 
 Nice work, we sent the output of `grep` to `head`. Let's try counting the
 number of entries for Act5C with `wc -l`. We can do the same trick to send
@@ -350,12 +343,12 @@ the number of lines from that file.
 > shell to do this.)
 >
 > > ## Solution
-> > 
+> >
 > > ```
 > > ls fastq/ | wc -l
 > > ```
 > > {: .language-bash}
-> > 
+> >
 > > Output of `ls` is one line per item, when chaining commands together like
 > > this, so counting lines gives the number of files.
 > {: .solution}
@@ -376,12 +369,12 @@ the number of lines from that file.
 > the file itself)?
 >
 > > ## Solution
-> > 
+> >
 > > ```
 > > zcat gene_association.fb.gz | head
 > > ```
 > > {: .language-bash}
-> > 
+> >
 > > The `head` command without any options shows the first 10 lines of a file.
 > {: .solution}
 {: .challenge}

_episodes/05-scripts.md

@@ -21,7 +21,6 @@ essentially a text file containing a list of UNIX commands to be executed in a
 sequential manner. These shell scripts can be run whenever we want, and are a
 great way to automate our work.
 
-
 ## Writing a Script
 
 So how do we write a shell script, exactly? It turns out we can do this with a
@@ -48,11 +47,11 @@ echo "Our script worked!"
 ```
 {: .language-bash}
 
-Ready to run our program? 
+Ready to run our program?
 Let's try running it:
 
 ```
-$ demo.sh 
+$ demo.sh
 ```
 {: .language-bash}
 
@@ -109,15 +108,15 @@ working left to right.
    `x` bits indicate permission to **R**ead, **W**rite, and e**X**ecute a file.
    There are three fields of `rwx` permissions following the spot for `d`. If a
    user is missing a permission to do something, it's indicated by a `-`.
-   - The first set of `rwx` are the permissions that the owner has (in this
-     case the owner is `yourUsername`).
-   - The second set of `rwx`s are permissions that other members of the owner's
-     group share (in this case, the group is named `tc001`).
-   - The third set of `rwx`s are permissions that anyone else with access to
-     this computer can do with a file. Though files are typically created with
-     read permissions for everyone, typically the permissions on your home
-     directory prevent others from being able to access the file in the first
-     place.
+   1. The first set of `rwx` are the permissions that the owner has (in this
+      case the owner is `yourUsername`).
+   1. The second set of `rwx`s are permissions that other members of the
+      owner's group share (in this case, the group is named `tc001`).
+   1. The third set of `rwx`s are permissions that anyone else with access to
+      this computer can do with a file. Though files are typically created with
+      read permissions for everyone, typically the permissions on your home
+      directory prevent others from being able to access the file in the first
+      place.
 2. **References:** This counts the number of references ([hard
    links](https://en.wikipedia.org/wiki/Hard_link)) to the item (file, folder,
    symbolic link or "shortcut").
@@ -154,7 +153,7 @@ drwxrwxr-x 2 yourUsername tc001     4096 Jan 16 19:16 fastq
 ```
 {: .output}
 
-Now that we have executable permissions for that file, we can run it. 
+Now that we have executable permissions for that file, we can run it.
 
 ```
 $ ./demo.sh
@@ -293,15 +292,8 @@ $ TEST=ls -l
 {: .error}
 
 What does work (we need to surround any command with `$(command)`):
-```
-$ TEST=$(ls -l)
-$ echo $TEST
-```
-{: .language-bash}
-```
-total 90372 -rw-rw-r-- 1 jeff jeff 12534006 Jan 16 18:50 bash-lesson.tar.gz -rwxrwxr-x. 1 jeff jeff 40 Jan 1619:41 demo.sh -rw-rw-r-- 1 jeff jeff 77426528 Jan 16 18:50 dmel-all-r6.19.gtf -rw-r--r-- 1 jeff jeff 721242 Jan 25 2016 dmel_unique_protein_isoforms_fb_2016_01.tsv drwxrwxr-x. 2 jeff jeff 4096 Jan 16 19:16 fastq -rw-r--r-- 1 jeff jeff 1830516 Jan 25 2016 gene_association.fb.gz -rw-rw-r-- 1 jeff jeff 15 Jan 16 19:17 test.txt -rw-rw-r-- 1 jeff jeff 245 Jan 16 19:24 word_counts.txt
-```
-{: .output}
+
+{% include {{ site.snippets }}/05/ls-l.snip %}
 
 Note that everything got printed on the same line. This is a feature, not a
 bug, as it allows us to use `$(commands)` inside lines of script without
@@ -404,11 +396,11 @@ gene_association.fb.gz
 > Is there a way to only run the loop on fastq files ending in `_1.fastq`?
 >
 > > ## Solution
-> > 
+> >
 > > Create the following script in a file called `head_all.sh`
 > > ```
 > > #!/bin/bash
-> > 
+> >
 > > for FILE in *.fastq
 > > do
 > >    echo $FILE
@@ -442,11 +434,11 @@ gene_association.fb.gz
 > with ".processed" added to it?
 >
 > > ## Solution
-> > 
+> >
 > > Create the following script in a file called `process.sh`
 > > ```
 > > #!/bin/bash
-> > 
+> >
 > > for FILE in *
 > > do
 > >    echo ${FILE}.processed
@@ -460,7 +452,7 @@ gene_association.fb.gz
 > >
 > > ```
 > > #!/bin/bash
-> > 
+> >
 > > for FILE in $(find . -max-depth 1 -type f)
 > > do
 > >    echo ${FILE}.processed
@@ -472,8 +464,7 @@ gene_association.fb.gz
 > {: .solution}
 {: .challenge}
 
-
-> ## Special permissions 
+> ## Special permissions
 >
 > What if we want to give different sets of users different permissions.
 > `chmod` actually accepts special numeric codes instead of stuff like `chmod