From 297fae8a6b6ad8a8f528a0f55fe6592bf6facbc6 Mon Sep 17 00:00:00 2001 From: Trevor Keller Date: Thu, 18 Mar 2021 00:34:15 -0400 Subject: [PATCH 1/4] update Makefile & scripts --- .github/workflows/test_and_build.yml | 2 +- Makefile | 13 +- _extras/figures.md | 11 +- bin/chunk-options.R | 32 +-- bin/generate_md_episodes.R | 83 ++---- bin/knit_lessons.sh | 2 +- bin/lesson_check.py | 36 ++- bin/lesson_initialize.py | 368 +++++++++++++++++++++++++-- bin/markdown_ast.rb | 10 +- bin/repo_check.py | 84 +++--- bin/test_lesson_check.py | 6 +- bin/workshop_check.py | 61 ++--- setup.md | 4 +- 13 files changed, 493 insertions(+), 219 deletions(-) diff --git a/.github/workflows/test_and_build.yml b/.github/workflows/test_and_build.yml index 1813585..25f39ac 100644 --- a/.github/workflows/test_and_build.yml +++ b/.github/workflows/test_and_build.yml @@ -20,7 +20,7 @@ jobs: pip3 install codespell - name: Check spelling run: | - codespell --skip="assets,*.svg,bin" --quiet-level=2 -L "rouge,dropse,namd,hist" + make spellcheck check_lesson_and_build: runs-on: ubuntu-latest diff --git a/Makefile b/Makefile index e42e769..ee0f25c 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,8 @@ export SHELL = /bin/bash # Settings MAKEFILES=Makefile $(wildcard *.mk) -JEKYLL=bundle config set path '.vendor/bundle' && bundle install && bundle update && bundle exec jekyll +JEKYLL=bundle config --local set path .vendor/bundle && bundle install && bundle update && bundle exec jekyll +HPC_JEKYLL_CONFIG?= PARSER=bin/markdown_ast.rb DST=_site @@ -45,11 +46,11 @@ endif ## * serve : render website and run a local server serve : lesson-md - ${JEKYLL} serve + ${JEKYLL} serve --config _config.yml,${HPC_JEKYLL_CONFIG} ## * site : build website but do not run a server site : lesson-md - ${JEKYLL} build + ${JEKYLL} build --config _config.yml,${HPC_JEKYLL_CONFIG} ## * docker-serve : use Docker to serve the site docker-serve : @@ -96,7 +97,7 @@ workshop-check : ## III. Commands specific to lesson websites ## ================================================= -.PHONY : lesson-check lesson-md lesson-files lesson-fixme +.PHONY : lesson-check lesson-md lesson-files lesson-fixme spellcheck # RMarkdown files RMD_SRC = $(wildcard _episodes_rmd/??-*.Rmd) @@ -151,6 +152,10 @@ lesson-files : lesson-fixme : @fgrep -i -n FIXME ${MARKDOWN_SRC} || true +## spellcheck +spellcheck: + codespell --skip="assets,*.svg,.vendor" --quiet-level=2 -L "dropse,hist,namd,rouge" + ## ## IV. Auxililary (plumbing) commands ## ================================================= diff --git a/_extras/figures.md b/_extras/figures.md index ee5b650..19dfce6 100644 --- a/_extras/figures.md +++ b/_extras/figures.md @@ -12,7 +12,7 @@ title: Figures {% endfor %} ]; - var xmlHttp = []; /* Required since we are going to query every episode. */ + var xmlHttp = []; /*Required since we are going to query every episode.*/ for (i=0; i < lesson_episodes.length; i++) { xmlHttp[i] = new XMLHttpRequest(); @@ -22,13 +22,13 @@ title: Figures if (this.readyState == 4 && this.status == 200) { var parser = new DOMParser(); var htmlDoc = parser.parseFromString(this.responseText,"text/html"); - var htmlDocArticle = htmlDoc.getElementsByTagName("article")[0]; + var htmlArticle = htmlDoc.getElementsByTagName("article")[0]; var article_here = document.getElementById(this.episode); - var images = htmlDocArticle.getElementsByTagName("img"); + var images = htmlArticle.getElementsByTagName("img"); if (images.length > 0) { - var h1text = htmlDocArticle.getElementsByTagName("h1")[0].innerHTML; + var h1text = htmlArticle.getElementsByTagName("h1")[0].innerHTML; var htitle = document.createElement('h2'); htitle.innerHTML = h1text; @@ -39,7 +39,8 @@ title: Figures image_num++; var title = document.createElement('p'); - title.innerHTML = "Figure " + image_num + ". " + image.alt; + title.innerHTML = "Figure " + image_num + + ". " + image.alt; article_here.appendChild(title); article_here.appendChild(image.cloneNode(false)); diff --git a/bin/chunk-options.R b/bin/chunk-options.R index 8e0d62a..d956f60 100644 --- a/bin/chunk-options.R +++ b/bin/chunk-options.R @@ -20,7 +20,7 @@ knitr_fig_path <- function(prefix) { opts_chunk$set(fig.path = new_path) } -## We use the rmd- prefix for the figures generated by the lessons so +## We use the rmd- prefix for the figures generated by the lssons so ## they can be easily identified and deleted by `make clean-rmd`. The ## working directory when the lessons are generated is the root so the ## figures need to be saved in fig/, but when the site is generated, @@ -29,42 +29,30 @@ knitr_fig_path <- function(prefix) { opts_chunk$set(tidy = FALSE, results = "markup", comment = NA, fig.align = "center", fig.path = "fig/rmd-", - fig.process = fix_fig_path, - fig.width = 8.5, fig.height = 8.5, - fig.retina = 2) + fig.process = fix_fig_path) # The hooks below add html tags to the code chunks and their output so that they # are properly formatted when the site is built. hook_in <- function(x, options) { - lg <- tolower(options$engine) - style <- paste0(".language-", lg) - stringr::str_c("\n\n~~~\n", - paste0(x, collapse="\n"), - "\n~~~\n{: ", style, "}\n\n") + paste0(x, collapse="\n"), + "\n~~~\n{: .r}\n\n") } hook_out <- function(x, options) { x <- gsub("\n$", "", x) stringr::str_c("\n\n~~~\n", - paste0(x, collapse="\n"), - "\n~~~\n{: .output}\n\n") + paste0(x, collapse="\n"), + "\n~~~\n{: .output}\n\n") } hook_error <- function(x, options) { x <- gsub("\n$", "", x) stringr::str_c("\n\n~~~\n", - paste0(x, collapse="\n"), - "\n~~~\n{: .error}\n\n") -} - -hook_warning <- function(x, options) { - x <- gsub("\n$", "", x) - stringr::str_c("\n\n~~~\n", - paste0(x, collapse = "\n"), - "\n~~~\n{: .warning}\n\n") + paste0(x, collapse="\n"), + "\n~~~\n{: .error}\n\n") } -knit_hooks$set(source = hook_in, output = hook_out, warning = hook_warning, - error = hook_error, message = hook_out) +knit_hooks$set(source = hook_in, output = hook_out, warning = hook_error, + error = hook_error, message = hook_out) diff --git a/bin/generate_md_episodes.R b/bin/generate_md_episodes.R index 0b4c69e..6d8a2d7 100644 --- a/bin/generate_md_episodes.R +++ b/bin/generate_md_episodes.R @@ -1,71 +1,36 @@ generate_md_episodes <- function() { - library("methods") - - if (!require("remotes", quietly = TRUE)) { - install.packages("remotes", repos = c(CRAN = "https://cloud.r-project.org/")) - } + if (require("knitr") && packageVersion("knitr") < '1.9.19') + stop("knitr must be version 1.9.20 or higher") - if (!require("requirements", quietly = TRUE)) { - remotes::install_github("hadley/requirements") - } + if (!require("stringr")) + stop("The package stringr is required for generating the lessons.") - required_pkgs <- unique(c( - ## Packages for episodes - requirements:::req_dir("_episodes_rmd"), - ## Pacakges for tools - requirements:::req_dir("bin") - )) + if (require("checkpoint")) { + required_pkgs <- + checkpoint:::projectScanPackages(project = "_episodes_rmd", + verbose=FALSE, use.knitr = TRUE)$pkgs + } else { + stop("The checkpoint package is required to build the lessons.") + } - missing_pkgs <- setdiff(required_pkgs, rownames(installed.packages())) + missing_pkgs <- required_pkgs[!(required_pkgs %in% rownames(installed.packages()))] - if (length(missing_pkgs)) { - message("Installing missing required packages: ", - paste(missing_pkgs, collapse=", ")) - install.packages(missing_pkgs) - } + if (length(missing_pkgs)) { + message("Installing missing required packages: ", + paste(missing_pkgs, collapse=", ")) + install.packages(missing_pkgs) + } - if (require("knitr") && packageVersion("knitr") < '1.9.19') - stop("knitr must be version 1.9.20 or higher") + ## find all the Rmd files, and generates the paths for their respective outputs + src_rmd <- list.files(pattern = "??-*.Rmd$", path = "_episodes_rmd", full.names = TRUE) + dest_md <- file.path("_episodes", gsub("Rmd$", "md", basename(src_rmd))) - ## get the Rmd file to process from the command line, and generate the path for their respective outputs - args <- commandArgs(trailingOnly = TRUE) - if (!identical(length(args), 2L)) { - stop("input and output file must be passed to the script") - } + ## knit the Rmd into markdown + mapply(function(x, y) { + knitr::knit(x, output = y) + }, src_rmd, dest_md) - src_rmd <- args[1] - dest_md <- args[2] - - ## knit the Rmd into markdown - knitr::knit(src_rmd, output = dest_md) - - # Read the generated md files and add comments advising not to edit them - vapply(dest_md, function(y) { - con <- file(y) - mdfile <- readLines(con) - if (mdfile[1] != "---") - stop("Input file does not have a valid header") - mdfile <- append(mdfile, "# Please do not edit this file directly; it is auto generated.", after = 1) - mdfile <- append(mdfile, paste("# Instead, please edit", - basename(y), "in _episodes_rmd/"), after = 2) - writeLines(mdfile, con) - close(con) - return(paste("Warning added to YAML header of", y)) - }, - character(1)) -} - -generate_md_episodes() -ead, please edit", basename(y), "in _episodes_rmd/"), - after = 2 - ) - writeLines(mdfile, con) - close(con) - return(paste("Warning added to YAML header of", y)) - } - - vapply(dest_md, add_no_edit_comment, character(1)) } generate_md_episodes() diff --git a/bin/knit_lessons.sh b/bin/knit_lessons.sh index 194f124..3a2395f 100755 --- a/bin/knit_lessons.sh +++ b/bin/knit_lessons.sh @@ -4,5 +4,5 @@ # The Makefile passes in the names of files. if [ $# -ne 0 ] ; then - Rscript -e "source('bin/generate_md_episodes.R')" "$@" + Rscript -e "source('bin/generate_md_episodes.R')" fi diff --git a/bin/lesson_check.py b/bin/lesson_check.py index 50e8bb0..6503008 100755 --- a/bin/lesson_check.py +++ b/bin/lesson_check.py @@ -18,6 +18,9 @@ # Where to look for source Markdown files. SOURCE_DIRS = ['', '_episodes', '_extras'] +# Where to look for snippet files (which should be Markdown syntax) +SNIPPET_DIR = os.path.join('_includes', 'snippets_library') + # Where to look for source Rmd files. SOURCE_RMD_DIRS = ['_episodes_rmd'] @@ -80,12 +83,15 @@ 'output', 'source', 'language-bash', - 'language-html', + 'html', 'language-make', 'language-matlab', 'language-python', 'language-r', - 'language-sql' + 'language-shell', + 'language-sql', + 'bash', + 'python' } # What fields are required in teaching episode metadata? @@ -107,7 +113,7 @@ # How long are lines allowed to be? # Please keep this in sync with .editorconfig! -MAX_LINE_LEN = 80 +MAX_LINE_LEN = 79 def main(): @@ -161,7 +167,7 @@ def parse_args(): default=False, action="store_true", dest='permissive', - help='Do not raise an error even if issues are detected') + help='Do not raise an error even if there are issues') args, extras = parser.parse_known_args() require(args.parser is not None, @@ -266,6 +272,13 @@ def read_all_markdown(source_dir, parser): data = read_markdown(parser, filename) if data: result[filename] = data + + # Also read our snippets + snippet_pattern = os.path.join(SNIPPET_DIR, '**/*.snip') + for filename in glob.glob(snippet_pattern, recursive=True): + data = read_markdown(parser, filename) + if data: + result[filename] = data return result @@ -357,8 +370,14 @@ def check_line_lengths(self): """Check the raw text of the lesson body.""" if self.args.line_lengths: - over = [i for (i, l, n) in self.lines if ( - n > MAX_LINE_LEN) and (not l.startswith('!'))] + code = '^[> ]*{{' # regular expression for [> > > ]{{ site... }} + link = '^[[].+[]]:' # regex for [link-abbrv]: address + http = '^.*http' # regex for ...http: + over = [i for (i, l, n) in self.lines if (n > MAX_LINE_LEN) and + (not l.startswith('!')) and + (not re.search(code, l)) and + (not re.search(link, l)) and + (not re.search(http, l)) ] self.reporter.check(not over, self.filename, 'Line(s) too long: {0}', @@ -462,6 +481,8 @@ def get_loc(self, node): class CheckNonJekyll(CheckBase): """Check a file that isn't translated by Jekyll.""" + def __init__(self, args, filename, metadata, metadata_len, text, lines, doc): + super().__init__(args, filename, metadata, metadata_len, text, lines, doc) def check_metadata(self): self.reporter.check(self.metadata is None, @@ -553,7 +574,8 @@ def __init__(self, args, filename, metadata, metadata_len, text, lines, doc): (re.compile(r'index\.md'), CheckIndex), (re.compile(r'reference\.md'), CheckReference), (re.compile(r'_episodes/.*\.md'), CheckEpisode), - (re.compile(r'.*\.md'), CheckGeneric) + (re.compile(r'.*\.md'), CheckGeneric), + (re.compile(r'.*\.snip'), CheckNonJekyll) ] diff --git a/bin/lesson_initialize.py b/bin/lesson_initialize.py index 65079c1..540bb08 100755 --- a/bin/lesson_initialize.py +++ b/bin/lesson_initialize.py @@ -1,29 +1,351 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python """Initialize a newly-created repository.""" +from __future__ import print_function import sys import os -import shutil + +ROOT_AUTHORS = '''\ +FIXME: list authors' names and email addresses. +''' + +ROOT_CITATION = '''\ +FIXME: describe how to cite this lesson. +''' + +ROOT_CONTRIBUTING_MD = '''\ +# Contributing + +[Software Carpentry][swc-site] and [Data Carpentry][dc-site] are open source projects, +and we welcome contributions of all kinds: +new lessons, +fixes to existing material, +bug reports, +and reviews of proposed changes are all welcome. + +## Contributor Agreement + +By contributing, +you agree that we may redistribute your work under [our license](LICENSE.md). +In exchange, +we will address your issues and/or assess your change proposal as promptly as we can, +and help you become a member of our community. +Everyone involved in [Software Carpentry][swc-site] and [Data Carpentry][dc-site] +agrees to abide by our [code of conduct](CONDUCT.md). + +## How to Contribute + +The easiest way to get started is to file an issue +to tell us about a spelling mistake, +some awkward wording, +or a factual error. +This is a good way to introduce yourself +and to meet some of our community members. + +1. If you do not have a [GitHub][github] account, + you can [send us comments by email][contact]. + However, + we will be able to respond more quickly if you use one of the other methods described below. + +2. If you have a [GitHub][github] account, + or are willing to [create one][github-join], + but do not know how to use Git, + you can report problems or suggest improvements by [creating an issue][issues]. + This allows us to assign the item to someone + and to respond to it in a threaded discussion. + +3. If you are comfortable with Git, + and would like to add or change material, + you can submit a pull request (PR). + Instructions for doing this are [included below](#using-github). + +## Where to Contribute + +1. If you wish to change this lesson, + please work in , + which can be viewed at . + +2. If you wish to change the example lesson, + please work in , + which documents the format of our lessons + and can be viewed at . + +3. If you wish to change the template used for workshop websites, + please work in . + The home page of that repository explains how to set up workshop websites, + while the extra pages in + provide more background on our design choices. + +4. If you wish to change CSS style files, tools, + or HTML boilerplate for lessons or workshops stored in `_includes` or `_layouts`, + please work in . + +## What to Contribute + +There are many ways to contribute, +from writing new exercises and improving existing ones +to updating or filling in the documentation +and and submitting [bug reports][issues] +about things that don't work, aren't clear, or are missing. +If you are looking for ideas, +please see [the list of issues for this repository][issues], +or the issues for [Data Carpentry][dc-issues] +and [Software Carpentry][swc-issues] projects. + +Comments on issues and reviews of pull requests are just as welcome: +we are smarter together than we are on our own. +Reviews from novices and newcomers are particularly valuable: +it's easy for people who have been using these lessons for a while +to forget how impenetrable some of this material can be, +so fresh eyes are always welcome. + +## What *Not* to Contribute + +Our lessons already contain more material than we can cover in a typical workshop, +so we are usually *not* looking for more concepts or tools to add to them. +As a rule, +if you want to introduce a new idea, +you must (a) estimate how long it will take to teach +and (b) explain what you would take out to make room for it. +The first encourages contributors to be honest about requirements; +the second, to think hard about priorities. + +We are also not looking for exercises or other material that only run on one platform. +Our workshops typically contain a mixture of Windows, Mac OS X, and Linux users; +in order to be usable, +our lessons must run equally well on all three. + +## Using GitHub + +If you choose to contribute via GitHub, +you may want to look at +[How to Contribute to an Open Source Project on GitHub][how-contribute]. +In brief: + +1. The published copy of the lesson is in the `gh-pages` branch of the repository + (so that GitHub will regenerate it automatically). + Please create all branches from that, + and merge the [master repository][repo]'s `gh-pages` branch into your `gh-pages` branch + before starting work. + Please do *not* work directly in your `gh-pages` branch, + since that will make it difficult for you to work on other contributions. + +2. We use [GitHub flow][github-flow] to manage changes: + 1. Create a new branch in your desktop copy of this repository for each significant change. + 2. Commit the change in that branch. + 3. Push that branch to your fork of this repository on GitHub. + 4. Submit a pull request from that branch to the [master repository][repo]. + 5. If you receive feedback, + make changes on your desktop and push to your branch on GitHub: + the pull request will update automatically. + +Each lesson has two maintainers who review issues and pull requests +or encourage others to do so. +The maintainers are community volunteers, +and have final say over what gets merged into the lesson. + +## Other Resources + +General discussion of [Software Carpentry][swc-site] and [Data Carpentry][dc-site] +happens on the [discussion mailing list][discuss-list], +which everyone is welcome to join. +You can also [reach us by email][contact]. + +[contact]: mailto:admin@software-carpentry.org +[dc-issues]: https://github.com/issues?q=user%3Adatacarpentry +[dc-lessons]: http://datacarpentry.org/lessons/ +[dc-site]: http://datacarpentry.org/ +[discuss-list]: http://lists.software-carpentry.org/listinfo/discuss +[github]: http://github.com +[github-flow]: https://guides.github.com/introduction/flow/ +[github-join]: https://github.com/join +[how-contribute]: https://egghead.io/series/how-to-contribute-to-an-open-source-project-on-github +[issues]: https://github.com/swcarpentry/FIXME/issues/ +[repo]: https://github.com/swcarpentry/FIXME/ +[swc-issues]: https://github.com/issues?q=user%3Aswcarpentry +[swc-lessons]: http://software-carpentry.org/lessons/ +[swc-site]: http://software-carpentry.org/ +''' + +ROOT_CONFIG_YML = '''\ +#------------------------------------------------------------ +# Values for this lesson. +#------------------------------------------------------------ + +# Which carpentry is this ("swc", "dc", or "lc")? +carpentry: "swc" + +# Overall title for pages. +title: "Lesson Title" + +# Contact. This *must* include the protocol: if it's an email +# address, it must look like "mailto:lessons@software-carpentry.org", +# or if it's a URL, "https://gitter.im/username/ProjectName". +contact: "mailto:lessons@software-carpentry.org" + +#------------------------------------------------------------ +# Generic settings (should not need to change). +#------------------------------------------------------------ + +# What kind of thing is this ("workshop" or "lesson")? +kind: "lesson" + +# Magic to make URLs resolve both locally and on GitHub. +# See https://help.github.com/articles/repository-metadata-on-github-pages/. +repository: / + +# Sites. +amy_site: "https://amy.software-carpentry.org/workshops" +dc_site: "http://datacarpentry.org" +swc_github: "https://github.com/swcarpentry" +swc_site: "https://software-carpentry.org" +swc_pages: "https://swcarpentry.github.io" +lc_site: "http://librarycarpentry.github.io/" +template_repo: "https://github.com/swcarpentry/styles" +example_repo: "https://github.com/swcarpentry/lesson-example" +example_site: "https://swcarpentry.github.com/lesson-example" +workshop_repo: "https://github.com/swcarpentry/workshop-template" +workshop_site: "https://swcarpentry.github.io/workshop-template" +training_site: "https://swcarpentry.github.io/instructor-training" + +# Surveys. +pre_survey: "https://www.surveymonkey.com/r/swc_pre_workshop_v1?workshop_id=" +post_survey: "https://www.surveymonkey.com/r/swc_post_workshop_v1?workshop_id=" +training_post_survey: "https://www.surveymonkey.com/r/post-instructor-training" + +# Start time in minutes (0 to be clock-independent, 540 to show a start at 09:00 am). +start_time: 0 + +# Specify that things in the episodes collection should be output. +collections: + episodes: + output: true + permalink: /:path/ + extras: + output: true + +# Set the default layout for things in the episodes collection. +defaults: + - values: + root: .. + - scope: + path: "" + type: episodes + values: + layout: episode + +# Files and directories that are not to be copied. +exclude: + - Makefile + - bin + +# Turn off built-in syntax highlighting. +highlighter: false +''' + +ROOT_INDEX_MD = '''\ +--- +layout: lesson +root: . +--- +FIXME: home page introduction + +> ## Prerequisites +> +> FIXME +{: .prereq} +''' + +ROOT_REFERENCE_MD = '''\ +--- +layout: reference +permalink: /reference/ +--- + +## Glossary + +FIXME +''' + +ROOT_SETUP_MD = '''\ +--- +layout: page +title: Setup +permalink: /setup/ +--- +FIXME +''' + +EPISODES_INTRODUCTION_MD = '''\ +--- +title: "Introduction" +teaching: 0 +exercises: 0 +questions: +- "Key question" +objectives: +- "First objective." +keypoints: +- "First key point." +--- +''' + +EXTRAS_ABOUT_MD = '''\ +--- +layout: page +title: About +permalink: /about/ +--- +{% include carpentries.html %} +''' + +EXTRAS_DISCUSS_MD = '''\ +--- +layout: page +title: Discussion +permalink: /discuss/ +--- +FIXME +''' + +EXTRAS_FIGURES_MD = '''\ +--- +layout: page +title: Figures +permalink: /figures/ +--- +{% include all_figures.html %} +''' + +EXTRAS_GUIDE_MD = '''\ +--- +layout: page +title: "Instructor Notes" +permalink: /guide/ +--- +FIXME +''' + +INCLUDES_ALL_FIGURES_HTML = '''\ + +''' BOILERPLATE = ( - '.travis.yml', - 'AUTHORS', - 'CITATION', - 'CONTRIBUTING.md', - 'MAINTENANCE.md', - 'README.md', - '_config.yml', - '_episodes/01-introduction.md', - '_extras/about.md', - '_extras/discuss.md', - '_extras/figures.md', - '_extras/guide.md', - 'aio.md', - 'index.md', - 'reference.md', - 'setup.md', + ('AUTHORS', ROOT_AUTHORS), + ('CITATION', ROOT_CITATION), + ('CONTRIBUTING.md', ROOT_CONTRIBUTING_MD), + ('_config.yml', ROOT_CONFIG_YML), + ('index.md', ROOT_INDEX_MD), + ('reference.md', ROOT_REFERENCE_MD), + ('setup.md', ROOT_SETUP_MD), + ('_episodes/01-introduction.md', EPISODES_INTRODUCTION_MD), + ('_extras/about.md', EXTRAS_ABOUT_MD), + ('_extras/discuss.md', EXTRAS_DISCUSS_MD), + ('_extras/figures.md', EXTRAS_FIGURES_MD), + ('_extras/guide.md', EXTRAS_GUIDE_MD), + ('_includes/all_figures.html', INCLUDES_ALL_FIGURES_HTML) ) @@ -32,7 +354,7 @@ def main(): # Check. errors = False - for path in BOILERPLATE: + for (path, _) in BOILERPLATE: if os.path.exists(path): print('Warning: {0} already exists.'.format(path), file=sys.stderr) errors = True @@ -41,11 +363,9 @@ def main(): sys.exit(1) # Create. - for path in BOILERPLATE: - shutil.copyfile( - "bin/boilerplate/{}".format(path), - path - ) + for (path, content) in BOILERPLATE: + with open(path, 'w') as writer: + writer.write(content) if __name__ == '__main__': diff --git a/bin/markdown_ast.rb b/bin/markdown_ast.rb index 2ef3f77..c3fd0b5 100755 --- a/bin/markdown_ast.rb +++ b/bin/markdown_ast.rb @@ -1,13 +1,11 @@ #!/usr/bin/env ruby -# frozen_string_literal: true # Use Kramdown parser to produce AST for Markdown document. -require 'kramdown' -require 'kramdown-parser-gfm' -require 'json' +require "kramdown" +require "json" -markdown = $stdin.read -doc = Kramdown::Document.new(markdown, input: 'GFM', hard_wrap: false) +markdown = STDIN.read() +doc = Kramdown::Document.new(markdown) tree = doc.to_hash_a_s_t puts JSON.pretty_generate(tree) diff --git a/bin/repo_check.py b/bin/repo_check.py index 9bf5c59..9097c0f 100755 --- a/bin/repo_check.py +++ b/bin/repo_check.py @@ -1,15 +1,17 @@ +#!/usr/bin/env python + """ Check repository settings. """ - +from __future__ import print_function import sys import os from subprocess import Popen, PIPE import re -from argparse import ArgumentParser +from optparse import OptionParser -from util import Reporter, require +from util import Reporter, load_yaml, require # Import this way to produce a more useful error message. try: @@ -20,7 +22,7 @@ # Pattern to match Git command-line output for remotes => (user name, project name). -P_GIT_REMOTE = re.compile(r'upstream\s+(?:https://|git@)github.com[:/]([^/]+)/([^.]+)(\.git)?\s+\(fetch\)') +P_GIT_REMOTE = re.compile(r'upstream\s+[^:]+:([^/]+)/([^.]+)\.git\s+\(fetch\)') # Repository URL format string. F_REPO_URL = 'https://github.com/{0}/{1}/' @@ -33,26 +35,15 @@ # Expected labels and colors. EXPECTED = { - 'help wanted': 'dcecc7', - 'status:in progress': '9bcc65', - 'status:changes requested': '679f38', - 'status:wait': 'fff2df', - 'status:refer to cac': 'ffdfb2', - 'status:need more info': 'ee6c00', - 'status:blocked': 'e55100', - 'status:out of scope': 'eeeeee', - 'status:duplicate': 'bdbdbd', - 'type:typo text': 'f8bad0', - 'type:bug': 'eb3f79', - 'type:formatting': 'ac1357', - 'type:template and tools': '7985cb', - 'type:instructor guide': '00887a', - 'type:discussion': 'b2e5fc', - 'type:enhancement': '7fdeea', - 'type:clarification': '00acc0', - 'type:teaching example': 'ced8dc', - 'good first issue': 'ffeb3a', - 'high priority': 'd22e2e' + 'bug' : 'bd2c00', + 'discussion' : 'fc8dc1', + 'enhancement' : '9cd6dc', + 'help-wanted' : 'f4fd9c', + 'instructor-training' : '6e5494', + 'newcomer-friendly' : 'eec275', + 'question' : '808040', + 'template-and-tools' : '2b3990', + 'work-in-progress' : '7ae78e' } @@ -63,7 +54,7 @@ def main(): args = parse_args() reporter = Reporter() - repo_url = get_repo_url(args.repo_url) + repo_url = get_repo_url(args.source_dir, args.repo_url) check_labels(reporter, repo_url) reporter.report() @@ -73,24 +64,24 @@ def parse_args(): Parse command-line arguments. """ - parser = ArgumentParser(description="""Check repository settings.""") - parser.add_argument('-r', '--repo', - default=None, - dest='repo_url', - help='repository URL') - parser.add_argument('-s', '--source', - default=os.curdir, - dest='source_dir', - help='source directory') - - args, extras = parser.parse_known_args() + parser = OptionParser() + parser.add_option('-r', '--repo', + default=None, + dest='repo_url', + help='repository URL') + parser.add_option('-s', '--source', + default=os.curdir, + dest='source_dir', + help='source directory') + + args, extras = parser.parse_args() require(not extras, 'Unexpected trailing command-line arguments "{0}"'.format(extras)) return args -def get_repo_url(repo_url): +def get_repo_url(source_dir, repo_url): """ Figure out which repository to query. """ @@ -101,8 +92,7 @@ def get_repo_url(repo_url): # Guess. cmd = 'git remote -v' - p = Popen(cmd, shell=True, stdin=PIPE, stdout=PIPE, - close_fds=True, universal_newlines=True, encoding='utf-8') + p = Popen(cmd, shell=True, stdin=PIPE, stdout=PIPE, close_fds=True, universal_newlines=True) stdout_data, stderr_data = p.communicate() stdout_data = stdout_data.split('\n') matches = [P_GIT_REMOTE.match(line) for line in stdout_data] @@ -111,12 +101,10 @@ def get_repo_url(repo_url): 'Unexpected output from git remote command: "{0}"'.format(matches)) username = matches[0].group(1) - require( - username, 'empty username in git remote output {0}'.format(matches[0])) + require(username, 'empty username in git remote output {0}'.format(matches[0])) project_name = matches[0].group(2) - require( - username, 'empty project name in git remote output {0}'.format(matches[0])) + require(username, 'empty project name in git remote output {0}'.format(matches[0])) url = F_REPO_URL.format(username, project_name) return url @@ -143,9 +131,9 @@ def check_labels(reporter, repo_url): overlap = set(EXPECTED.keys()).intersection(set(actual.keys())) for name in sorted(overlap): - reporter.check(EXPECTED[name].lower() == actual[name].lower(), + reporter.check(EXPECTED[name] == actual[name], None, - 'Color mis-match for label {0} in {1}: expected {2}, found {3}', + 'Color mismatch for label {0} in {1}: expected {2}, found {3}', name, repo_url, EXPECTED[name], actual[name]) @@ -155,15 +143,13 @@ def get_labels(repo_url): """ m = P_REPO_URL.match(repo_url) - require( - m, 'repository URL {0} does not match expected pattern'.format(repo_url)) + require(m, 'repository URL {0} does not match expected pattern'.format(repo_url)) username = m.group(1) require(username, 'empty username in repository URL {0}'.format(repo_url)) project_name = m.group(2) - require( - username, 'empty project name in repository URL {0}'.format(repo_url)) + require(username, 'empty project name in repository URL {0}'.format(repo_url)) url = F_API_URL.format(username, project_name) r = requests.get(url) diff --git a/bin/test_lesson_check.py b/bin/test_lesson_check.py index 960059e..743d0cf 100755 --- a/bin/test_lesson_check.py +++ b/bin/test_lesson_check.py @@ -1,14 +1,11 @@ -#!/usr/bin/env python3 - import unittest import lesson_check import util - class TestFileList(unittest.TestCase): def setUp(self): - self.reporter = util.Reporter() # TODO: refactor reporter class. + self.reporter = util.Reporter() ## TODO: refactor reporter class. def test_file_list_has_expected_entries(self): # For first pass, simply assume that all required files are present @@ -18,6 +15,5 @@ def test_file_list_has_expected_entries(self): lesson_check.check_fileset('', self.reporter, all_filenames) self.assertEqual(len(self.reporter.messages), 0) - if __name__ == "__main__": unittest.main() diff --git a/bin/workshop_check.py b/bin/workshop_check.py index 15d954a..d3051bf 100755 --- a/bin/workshop_check.py +++ b/bin/workshop_check.py @@ -1,8 +1,10 @@ +#!/usr/bin/env python + '''Check that a workshop's index.html metadata is valid. See the docstrings on the checking functions for a summary of the checks. ''' - +from __future__ import print_function import sys import os import re @@ -16,10 +18,10 @@ URL_PATTERN = r'https?://.+' # Defaults. -CARPENTRIES = ("dc", "swc", "lc", "cp") +CARPENTRIES = ("dc", "swc") DEFAULT_CONTACT_EMAIL = 'admin@software-carpentry.org' -USAGE = 'Usage: "workshop_check.py path/to/root/directory"' +USAGE = 'Usage: "check-workshop path/to/root/directory"' # Country and language codes. Note that codes mean different things: 'ar' # is 'Arabic' as a language but 'Argentina' as a country. @@ -89,7 +91,7 @@ def check_layout(layout): @look_for_fixme def check_carpentry(layout): - '''"carpentry" in YAML header must be "dc", "swc", "lc", or "cp".''' + '''"carpentry" in YAML header must be "dc" or "swc".''' return layout in CARPENTRIES @@ -115,7 +117,7 @@ def check_humandate(date): and 4-digit year. Examples include 'Feb 18-20, 2025' and 'Feb 18 and 20, 2025'. It may be in languages other than English, but the month name should be kept short to aid formatting of the main - Carpentries web site. + Software Carpentry web site. """ if ',' not in date: @@ -172,8 +174,8 @@ def check_latitude_longitude(latlng): try: lat, lng = latlng.split(',') lat = float(lat) - lng = float(lng) - return (-90.0 <= lat <= 90.0) and (-180.0 <= lng <= 180.0) + long = float(lng) + return (-90.0 <= lat <= 90.0) and (-180.0 <= long <= 180.0) except ValueError: return False @@ -201,22 +203,15 @@ def check_helpers(helpers): @look_for_fixme -def check_emails(emails): +def check_email(email): """ - 'emails' must be a comma-separated list of valid email addresses. - The list may be empty. A valid email address consists of characters, - an '@', and more characters. It should not contain the default contact + 'contact' must be a valid email address consisting of characters, + an '@', and more characters. It should not be the default contact + email address 'admin@software-carpentry.org'. """ - # YAML automatically loads list-like strings as lists. - if (isinstance(emails, list) and len(emails) >= 0): - for email in emails: - if ((not bool(re.match(EMAIL_PATTERN, email))) or (email == DEFAULT_CONTACT_EMAIL)): - return False - else: - return False - - return True + return bool(re.match(EMAIL_PATTERN, email)) and \ + (email != DEFAULT_CONTACT_EMAIL) def check_eventbrite(eventbrite): @@ -232,12 +227,12 @@ def check_eventbrite(eventbrite): @look_for_fixme -def check_collaborative_notes(collaborative_notes): +def check_etherpad(etherpad): """ - 'collaborative_notes' must be a valid URL. + 'etherpad' must be a valid URL. """ - return bool(re.match(URL_PATTERN, collaborative_notes)) + return bool(re.match(URL_PATTERN, etherpad)) @look_for_fixme @@ -291,14 +286,13 @@ def check_pass(value): 'helper list isn\'t a valid list of format ' + '["First helper", "Second helper",..]'), - 'email': (True, check_emails, - 'contact email list isn\'t a valid list of format ' + - '["me@example.org", "you@example.org",..] or contains incorrectly formatted email addresses or ' + - '"{0}".'.format(DEFAULT_CONTACT_EMAIL)), + 'contact': (True, check_email, + 'contact email invalid or still set to ' + + '"{0}".'.format(DEFAULT_CONTACT_EMAIL)), 'eventbrite': (False, check_eventbrite, 'Eventbrite key appears invalid'), - 'collaborative_notes': (False, check_collaborative_notes, 'Collaborative Notes URL appears invalid'), + 'etherpad': (False, check_etherpad, 'Etherpad URL appears invalid'), 'venue': (False, check_pass, 'venue name not specified'), @@ -306,10 +300,10 @@ def check_pass(value): } # REQUIRED is all required categories. -REQUIRED = {k for k in HANDLERS if HANDLERS[k][0]} +REQUIRED = set([k for k in HANDLERS if HANDLERS[k][0]]) # OPTIONAL is all optional categories. -OPTIONAL = {k for k in HANDLERS if not HANDLERS[k][0]} +OPTIONAL = set([k for k in HANDLERS if not HANDLERS[k][0]]) def check_blank_lines(reporter, raw): @@ -317,8 +311,7 @@ def check_blank_lines(reporter, raw): Blank lines are not allowed in category headers. """ - lines = [(i, x) for (i, x) in enumerate( - raw.strip().split('\n')) if not x.strip()] + lines = [(i, x) for (i, x) in enumerate(raw.strip().split('\n')) if not x.strip()] reporter.check(not lines, None, 'Blank line(s) in header: {0}', @@ -388,7 +381,7 @@ def check_config(reporter, filename): kind) carpentry = config.get('carpentry', None) - reporter.check(carpentry in ('swc', 'dc', 'lc', 'cp'), + reporter.check(carpentry in ('swc', 'dc'), filename, 'Missing or unknown carpentry: {0}', carpentry) @@ -408,7 +401,7 @@ def main(): reporter = Reporter() check_config(reporter, config_file) check_unwanted_files(root_dir, reporter) - with open(index_file, encoding='utf-8') as reader: + with open(index_file) as reader: data = reader.read() check_file(reporter, index_file, data) reporter.report() diff --git a/setup.md b/setup.md index ddb2972..2d9bb78 100644 --- a/setup.md +++ b/setup.md @@ -60,8 +60,8 @@ the Windows start menu. > Linux](https://docs.microsoft.com/en-us/windows/wsl/install-win10) > * Use the Windows [Powershell]( https://docs.microsoft.com/en-us/powershell/scripting/learn/remoting/ssh-remoting-in-powershell-core?view=powershell-7) -> * Read up on [Using a Unix/Linux emulator (Cygwin) or Secure Shell (SSH) client -> (Putty)](http://faculty.smu.edu/reynolds/unixtut/windows.html) +> * Read up on [Using a Unix/Linux emulator (Cygwin) or Secure Shell (SSH) +> client (Putty)](http://faculty.smu.edu/reynolds/unixtut/windows.html) > > > ## Warning > > From 7977b01e99e05d41a5a650e5340b6cbc64003583 Mon Sep 17 00:00:00 2001 From: Trevor Keller Date: Thu, 18 Mar 2021 01:03:11 -0400 Subject: [PATCH 2/4] create new snippets for extra long output, clear warnings --- .gitignore | 1 + _config.yml | 2 ++ _episodes/00-hpc-intro.md | 4 ++-- _episodes/04-wildcards-pipes.md | 9 +-------- _episodes/05-scripts.md | 11 ++--------- _includes/snippets/04/grep-act5c.snip | 9 +++++++++ _includes/snippets/05/ls-l.snip | 10 ++++++++++ 7 files changed, 27 insertions(+), 19 deletions(-) create mode 100644 _includes/snippets/04/grep-act5c.snip create mode 100644 _includes/snippets/05/ls-l.snip diff --git a/.gitignore b/.gitignore index b8b053f..5533282 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ *.pyc *~ .DS_Store +.bundle .ipynb_checkpoints .jekyll-cache .sass-cache diff --git a/_config.yml b/_config.yml index 09dca4b..7a65e74 100644 --- a/_config.yml +++ b/_config.yml @@ -2,6 +2,8 @@ # Values for this lesson. #------------------------------------------------------------ +snippets: "/snippets" + # Which carpentry is this ("swc", "dc", "lc", or "cp")? # swc: Software Carpentry # dc: Data Carpentry diff --git a/_episodes/00-hpc-intro.md b/_episodes/00-hpc-intro.md index b5e2bb7..e9fec4e 100644 --- a/_episodes/00-hpc-intro.md +++ b/_episodes/00-hpc-intro.md @@ -33,8 +33,8 @@ laptop computer where they started: their laptop will take over a month! * A genomics researcher has been using small datasets of sequence data, but soon will be receiving a new type of sequencing data that is 10 times as - large. It's already challenging to open the datasets on their computer — - analyzing these larger datasets will probably crash it. + large. It's already challenging to open the datasets on their computer + — analyzing these larger datasets will probably crash it. * An engineer is using a fluid dynamics package that has an option to run in parallel. So far, they haven't used this option on their desktop, but in going from 2D to 3D simulations, simulation time has more than tripled and it diff --git a/_episodes/04-wildcards-pipes.md b/_episodes/04-wildcards-pipes.md index 5d43918..f25a17b 100644 --- a/_episodes/04-wildcards-pipes.md +++ b/_episodes/04-wildcards-pipes.md @@ -318,14 +318,7 @@ The output is nearly unintelligible since there is so much of it. Let's send the output of that `grep` command to `head` so we can just take a peek at the first line. The `|` operator lets us send output from one command to the next: -``` -$ grep Act5C dmel-all-r6.19.gtf | head -n 1 -``` -{: .language-bash} -``` -X FlyBase gene 5900861 5905399 . + . gene_id "FBgn0000042"; gene_symbol "Act5C"; -``` -{: .output} +{% include {{ site.snippets }}/04/grep-act5c.snip %} Nice work, we sent the output of `grep` to `head`. Let's try counting the number of entries for Act5C with `wc -l`. We can do the same trick to send diff --git a/_episodes/05-scripts.md b/_episodes/05-scripts.md index 0eba39f..667b1cb 100644 --- a/_episodes/05-scripts.md +++ b/_episodes/05-scripts.md @@ -293,15 +293,8 @@ $ TEST=ls -l {: .error} What does work (we need to surround any command with `$(command)`): -``` -$ TEST=$(ls -l) -$ echo $TEST -``` -{: .language-bash} -``` -total 90372 -rw-rw-r-- 1 jeff jeff 12534006 Jan 16 18:50 bash-lesson.tar.gz -rwxrwxr-x. 1 jeff jeff 40 Jan 1619:41 demo.sh -rw-rw-r-- 1 jeff jeff 77426528 Jan 16 18:50 dmel-all-r6.19.gtf -rw-r--r-- 1 jeff jeff 721242 Jan 25 2016 dmel_unique_protein_isoforms_fb_2016_01.tsv drwxrwxr-x. 2 jeff jeff 4096 Jan 16 19:16 fastq -rw-r--r-- 1 jeff jeff 1830516 Jan 25 2016 gene_association.fb.gz -rw-rw-r-- 1 jeff jeff 15 Jan 16 19:17 test.txt -rw-rw-r-- 1 jeff jeff 245 Jan 16 19:24 word_counts.txt -``` -{: .output} + +{% include {{ site.snippets }}/05/ls-l.snip %} Note that everything got printed on the same line. This is a feature, not a bug, as it allows us to use `$(commands)` inside lines of script without diff --git a/_includes/snippets/04/grep-act5c.snip b/_includes/snippets/04/grep-act5c.snip new file mode 100644 index 0000000..37d245d --- /dev/null +++ b/_includes/snippets/04/grep-act5c.snip @@ -0,0 +1,9 @@ +``` +$ grep Act5C dmel-all-r6.19.gtf | head -n 1 +``` +{: .language-bash} + +``` +X FlyBase gene 5900861 5905399 . + . gene_id "FBgn0000042"; gene_symbol "Act5C"; +``` +{: .output} diff --git a/_includes/snippets/05/ls-l.snip b/_includes/snippets/05/ls-l.snip new file mode 100644 index 0000000..4115b39 --- /dev/null +++ b/_includes/snippets/05/ls-l.snip @@ -0,0 +1,10 @@ +``` +$ TEST=$(ls -l) +$ echo ${TEST} +``` +{: .language-bash} + +``` +total 90372 -rw-rw-r-- 1 jeff jeff 12534006 Jan 16 18:50 bash-lesson.tar.gz -rwxrwxr-x. 1 jeff jeff 40 Jan 1619:41 demo.sh -rw-rw-r-- 1 jeff jeff 77426528 Jan 16 18:50 dmel-all-r6.19.gtf -rw-r--r-- 1 jeff jeff 721242 Jan 25 2016 dmel_unique_protein_isoforms_fb_2016_01.tsv drwxrwxr-x. 2 jeff jeff 4096 Jan 16 19:16 fastq -rw-r--r-- 1 jeff jeff 1830516 Jan 25 2016 gene_association.fb.gz -rw-rw-r-- 1 jeff jeff 15 Jan 16 19:17 test.txt -rw-rw-r-- 1 jeff jeff 245 Jan 16 19:24 word_counts.txt +``` +{: .output} From 3f552b1e1f6d5b2d1aba58f3639fc62f2aa06c5f Mon Sep 17 00:00:00 2001 From: Trevor Keller Date: Thu, 18 Mar 2021 12:23:19 -0400 Subject: [PATCH 3/4] run markdownlint --- .github/workflows/test_and_build.yml | 3 +++ Gemfile | 1 + 2 files changed, 4 insertions(+) diff --git a/.github/workflows/test_and_build.yml b/.github/workflows/test_and_build.yml index 25f39ac..3c816e4 100644 --- a/.github/workflows/test_and_build.yml +++ b/.github/workflows/test_and_build.yml @@ -37,6 +37,9 @@ jobs: bundle config set path '.vendor/bundle' bundle config build.nokogiri --use-system-libraries bundle install + - name: "Lint Markdown" + run: | + find _episodes -name \*.md -exec bundle exec mdl -r MD001,MD003,MD005,MD006,MD007,MD008,MD009,MD010,MD011,MD012,MD015,MD016,MD017,MD018,MD019,MD020,MD021,MD022,MD023,MD025,MD035,MD036,MD037,MD038,MD039,MD046 {} \; - name: "Check lesson for warnings" run: | make lesson-check-all diff --git a/Gemfile b/Gemfile index 7e5e30d..5714bad 100644 --- a/Gemfile +++ b/Gemfile @@ -1,3 +1,4 @@ source "https://rubygems.org" gem "github-pages", group: :jekyll_plugins gem "kramdown-parser-gfm" +gem "mdl" From 6c45b86d7c7d37a3143d603475eac1880e7ed34a Mon Sep 17 00:00:00 2001 From: Trevor Keller Date: Thu, 18 Mar 2021 12:34:42 -0400 Subject: [PATCH 4/4] clear Markdown linter errors --- .github/workflows/test_and_build.yml | 5 +++- _episodes/01-connecting.md | 2 +- _episodes/02-navigation.md | 8 +++--- _episodes/03-files.md | 5 ---- _episodes/04-wildcards-pipes.md | 16 +++++------ _episodes/05-scripts.md | 38 +++++++++++++-------------- _includes/snippets/04/grep-act5c.snip | 2 +- 7 files changed, 36 insertions(+), 40 deletions(-) diff --git a/.github/workflows/test_and_build.yml b/.github/workflows/test_and_build.yml index 3c816e4..7779c10 100644 --- a/.github/workflows/test_and_build.yml +++ b/.github/workflows/test_and_build.yml @@ -37,9 +37,12 @@ jobs: bundle config set path '.vendor/bundle' bundle config build.nokogiri --use-system-libraries bundle install - - name: "Lint Markdown" + - name: "Lint episode markdown" run: | find _episodes -name \*.md -exec bundle exec mdl -r MD001,MD003,MD005,MD006,MD007,MD008,MD009,MD010,MD011,MD012,MD015,MD016,MD017,MD018,MD019,MD020,MD021,MD022,MD023,MD025,MD035,MD036,MD037,MD038,MD039,MD046 {} \; + - name: "Lint snippet markdown" + run: | + find _includes/snippets -name \*.md -exec bundle exec mdl -r MD001,MD003,MD005,MD006,MD007,MD008,MD009,MD010,MD011,MD012,MD015,MD016,MD017,MD018,MD019,MD020,MD021,MD022,MD023,MD025,MD035,MD036,MD037,MD038,MD039,MD046 {} \; - name: "Check lesson for warnings" run: | make lesson-check-all diff --git a/_episodes/01-connecting.md b/_episodes/01-connecting.md index 074464a..d71d5f4 100644 --- a/_episodes/01-connecting.md +++ b/_episodes/01-connecting.md @@ -1,6 +1,6 @@ --- title: "Connecting to the remote HPC system" -teaching: 20 +teaching: 20 exercises: 10 questions: - How do I open a terminal? diff --git a/_episodes/02-navigation.md b/_episodes/02-navigation.md index b530526..4294499 100644 --- a/_episodes/02-navigation.md +++ b/_episodes/02-navigation.md @@ -1,6 +1,6 @@ --- title: "Moving around and looking at things" -teaching: 15 +teaching: 15 exercises: 5 questions: - How do I navigate and look around the system? @@ -74,7 +74,7 @@ $ pwd ~~~ {: .output} -So, we know where we are. How do we look and see what's in our current +So, we know where we are. How do we look and see what's in our current directory? ``` $ ls @@ -382,7 +382,7 @@ Mandatory arguments to long options are mandatory for short options too. > [remote]$ ls -j > ~~~ > {: .language-bash} -> +> > ~~~ > ls: invalid option -- 'j' > Try 'ls --help' for more information. @@ -417,7 +417,7 @@ Mandatory arguments to long options are mandatory for short options too. > > 2. No: `/` stands for the root directory. > > 3. No: Amanda's home directory is `/Users/amanda`. > > 4. No: this goes up two levels, i.e. ends in `/Users`. -> > 5. Yes: `~` stands for the user's home directory, in this case +> > 5. Yes: `~` stands for the user's home directory, in this case > > `/Users/amanda`. > > 6. No: this would navigate into a directory `home` in the current directory > > if it exists. diff --git a/_episodes/03-files.md b/_episodes/03-files.md index efa29cb..fc38380 100644 --- a/_episodes/03-files.md +++ b/_episodes/03-files.md @@ -31,7 +31,6 @@ $ cd hpc-test ``` {: .language-bash} - ## Creating and Editing Text Files When working on an HPC system, we will frequently need to create or edit text @@ -72,7 +71,6 @@ editor. Here are the shortcut keys for a few common actions: * Ctrl+U — paste the cut text line (or lines). This command can be repeated to paste the same text elsewhere. - > ## Using `vim` as a text editor > > From time to time, you may encounter the `vim` text editor. Although `vim` @@ -104,7 +102,6 @@ draft.txt ``` {: .output} - ## Reading Files Let's read the file we just created now. There are a few different ways of @@ -146,7 +143,6 @@ it's "share and thrive". > print the contents of `draft.txt`, `chap1.txt`, and `chap2.txt`. {: .challenge} - ## Creating Directory We've successfully created a file. What about a directory? We've actually done @@ -162,7 +158,6 @@ draft.txt files ``` {: .output} - ## Moving, Renaming, Copying Files **Moving** — We will move `draft.txt` to the `files` directory with `mv` diff --git a/_episodes/04-wildcards-pipes.md b/_episodes/04-wildcards-pipes.md index f25a17b..6df3a11 100644 --- a/_episodes/04-wildcards-pipes.md +++ b/_episodes/04-wildcards-pipes.md @@ -22,7 +22,7 @@ keypoints: > > If you didn't get them in the last lesson, make sure to download the example > files used in the next few sections: -> +> > **Using wget**: `wget {{site.url}}{{site.baseurl}}/files/bash-lesson.tar.gz` > > **Using a web browser**: @@ -145,12 +145,12 @@ $ wc -l * > file with "fb" in it? > > > ## Solution -> > +> > > > ``` > > wc -l *fb* > > ``` > > {: .language-bash} -> > +> > > > i.e. *anything or nothing* then `fb` then *anything or nothing* > {: .solution} {: .challenge} @@ -161,7 +161,7 @@ $ wc -l * > "fastq" and move all of our .fastq files there in one `mv` command. > > > ## Solution -> > +> > > > ``` > > mkdir fastq > > mv *.fastq fastq/ @@ -343,12 +343,12 @@ the number of lines from that file. > shell to do this.) > > > ## Solution -> > +> > > > ``` > > ls fastq/ | wc -l > > ``` > > {: .language-bash} -> > +> > > > Output of `ls` is one line per item, when chaining commands together like > > this, so counting lines gives the number of files. > {: .solution} @@ -369,12 +369,12 @@ the number of lines from that file. > the file itself)? > > > ## Solution -> > +> > > > ``` > > zcat gene_association.fb.gz | head > > ``` > > {: .language-bash} -> > +> > > > The `head` command without any options shows the first 10 lines of a file. > {: .solution} {: .challenge} diff --git a/_episodes/05-scripts.md b/_episodes/05-scripts.md index 667b1cb..831bd0d 100644 --- a/_episodes/05-scripts.md +++ b/_episodes/05-scripts.md @@ -21,7 +21,6 @@ essentially a text file containing a list of UNIX commands to be executed in a sequential manner. These shell scripts can be run whenever we want, and are a great way to automate our work. - ## Writing a Script So how do we write a shell script, exactly? It turns out we can do this with a @@ -48,11 +47,11 @@ echo "Our script worked!" ``` {: .language-bash} -Ready to run our program? +Ready to run our program? Let's try running it: ``` -$ demo.sh +$ demo.sh ``` {: .language-bash} @@ -109,15 +108,15 @@ working left to right. `x` bits indicate permission to **R**ead, **W**rite, and e**X**ecute a file. There are three fields of `rwx` permissions following the spot for `d`. If a user is missing a permission to do something, it's indicated by a `-`. - - The first set of `rwx` are the permissions that the owner has (in this - case the owner is `yourUsername`). - - The second set of `rwx`s are permissions that other members of the owner's - group share (in this case, the group is named `tc001`). - - The third set of `rwx`s are permissions that anyone else with access to - this computer can do with a file. Though files are typically created with - read permissions for everyone, typically the permissions on your home - directory prevent others from being able to access the file in the first - place. + 1. The first set of `rwx` are the permissions that the owner has (in this + case the owner is `yourUsername`). + 1. The second set of `rwx`s are permissions that other members of the + owner's group share (in this case, the group is named `tc001`). + 1. The third set of `rwx`s are permissions that anyone else with access to + this computer can do with a file. Though files are typically created with + read permissions for everyone, typically the permissions on your home + directory prevent others from being able to access the file in the first + place. 2. **References:** This counts the number of references ([hard links](https://en.wikipedia.org/wiki/Hard_link)) to the item (file, folder, symbolic link or "shortcut"). @@ -154,7 +153,7 @@ drwxrwxr-x 2 yourUsername tc001 4096 Jan 16 19:16 fastq ``` {: .output} -Now that we have executable permissions for that file, we can run it. +Now that we have executable permissions for that file, we can run it. ``` $ ./demo.sh @@ -397,11 +396,11 @@ gene_association.fb.gz > Is there a way to only run the loop on fastq files ending in `_1.fastq`? > > > ## Solution -> > +> > > > Create the following script in a file called `head_all.sh` > > ``` > > #!/bin/bash -> > +> > > > for FILE in *.fastq > > do > > echo $FILE @@ -435,11 +434,11 @@ gene_association.fb.gz > with ".processed" added to it? > > > ## Solution -> > +> > > > Create the following script in a file called `process.sh` > > ``` > > #!/bin/bash -> > +> > > > for FILE in * > > do > > echo ${FILE}.processed @@ -453,7 +452,7 @@ gene_association.fb.gz > > > > ``` > > #!/bin/bash -> > +> > > > for FILE in $(find . -max-depth 1 -type f) > > do > > echo ${FILE}.processed @@ -465,8 +464,7 @@ gene_association.fb.gz > {: .solution} {: .challenge} - -> ## Special permissions +> ## Special permissions > > What if we want to give different sets of users different permissions. > `chmod` actually accepts special numeric codes instead of stuff like `chmod diff --git a/_includes/snippets/04/grep-act5c.snip b/_includes/snippets/04/grep-act5c.snip index 37d245d..8efca5c 100644 --- a/_includes/snippets/04/grep-act5c.snip +++ b/_includes/snippets/04/grep-act5c.snip @@ -4,6 +4,6 @@ $ grep Act5C dmel-all-r6.19.gtf | head -n 1 {: .language-bash} ``` -X FlyBase gene 5900861 5905399 . + . gene_id "FBgn0000042"; gene_symbol "Act5C"; +X FlyBase gene 5900861 5905399 . + . gene_id "FBgn0000042"; gene_symbol "Act5C"; ``` {: .output}