Better Docker containers for build automation (ros-infrastructure#83)

* Improves rosindex docker containerization. * Removes lunr-build-index progress bar, as it throws on Jenkins. * Drops testing configuration. * Provides default CMD when container runs. * Documents container environment vars Co-Authored-By: hidmic <[email protected]> * Updates README with proper usage instructions. * Fixes container environment vars.
gonzodepedro · Oct 23, 2018 · 73bdc3d · 73bdc3d
1 parent a9080d5
commit 73bdc3d
Show file tree

Hide file tree

Showing 19 changed files with 347 additions and 170 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -0,0 +1,4 @@
+_cache
+_sphinx
+_remotes
+_plugins_data
diff --git a/.gitignore b/.gitignore
@@ -2,3 +2,7 @@ _site
 .sass-cache
 _devel
 _roswiki
+_cache
+_sphinx
+_remotes
+_plugins_data
diff --git a/Gemfile b/Gemfile
@@ -5,11 +5,8 @@ gem 'colorator'
 #gem 'git'
 gem 'github-markup'
 gem 'jekyll'
-gem 'jekyll-optional-front-matter'
-gem 'jekyll-readme-index'
 gem 'jekyll-relative-links'
 gem 'jekyll-sitemap'
-gem 'jekyll-titles-from-headings'
 gem 'liquid'#, '~> 2.6.2'
 gem 'mercurial-ruby'
 gem 'nokogiri'

diff --git a/Gemfile.lock b/Gemfile.lock
@@ -0,0 +1,107 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    addressable (2.5.2)
+      public_suffix (>= 2.0.2, < 4.0)
+    awesome_print (1.8.0)
+    colorator (1.1.0)
+    concurrent-ruby (1.0.5)
+    em-websocket (0.5.1)
+      eventmachine (>= 0.12.9)
+      http_parser.rb (~> 0.6.0)
+    ethon (0.11.0)
+      ffi (>= 1.3.0)
+    eventmachine (1.2.7)
+    ffi (1.9.25)
+    forwardable-extended (2.6.0)
+    github-markup (2.0.1)
+    http_parser.rb (0.6.0)
+    i18n (0.9.5)
+      concurrent-ruby (~> 1.0)
+    jekyll (3.8.4)
+      addressable (~> 2.4)
+      colorator (~> 1.0)
+      em-websocket (~> 0.5)
+      i18n (~> 0.7)
+      jekyll-sass-converter (~> 1.0)
+      jekyll-watch (~> 2.0)
+      kramdown (~> 1.14)
+      liquid (~> 4.0)
+      mercenary (~> 0.3.3)
+      pathutil (~> 0.9)
+      rouge (>= 1.7, < 4)
+      safe_yaml (~> 1.0)
+    jekyll-optional-front-matter (0.3.0)
+      jekyll (~> 3.0)
+    jekyll-readme-index (0.2.0)
+      jekyll (~> 3.0)
+    jekyll-relative-links (0.5.3)
+      jekyll (~> 3.3)
+    jekyll-sass-converter (1.5.2)
+      sass (~> 3.4)
+    jekyll-sitemap (1.2.0)
+      jekyll (~> 3.3)
+    jekyll-titles-from-headings (0.5.1)
+      jekyll (~> 3.3)
+    jekyll-watch (2.1.1)
+      listen (~> 3.0)
+    kramdown (1.17.0)
+    liquid (4.0.1)
+    listen (3.1.5)
+      rb-fsevent (~> 0.9, >= 0.9.4)
+      rb-inotify (~> 0.9, >= 0.9.7)
+      ruby_dep (~> 1.2)
+    mercenary (0.3.6)
+    mercurial-ruby (0.7.12)
+      open4 (~> 1.3.0)
+    mini_portile2 (2.3.0)
+    multi_json (1.13.1)
+    nokogiri (1.8.5)
+      mini_portile2 (~> 2.3.0)
+    open4 (1.3.4)
+    pandoc-ruby (2.0.2)
+    pathutil (0.16.1)
+      forwardable-extended (~> 2.6)
+    public_suffix (3.0.3)
+    pygments.rb (1.2.1)
+      multi_json (>= 1.0.0)
+    rb-fsevent (0.10.3)
+    rb-inotify (0.9.10)
+      ffi (>= 0.5.0, < 2)
+    redcarpet (3.4.0)
+    rouge (3.3.0)
+    ruby_dep (1.5.0)
+    rugged (0.27.5)
+    safe_yaml (1.0.4)
+    sass (3.6.0)
+      sass-listen (~> 4.0.0)
+    sass-listen (4.0.0)
+      rb-fsevent (~> 0.9, >= 0.9.4)
+      rb-inotify (~> 0.9, >= 0.9.7)
+    typhoeus (1.3.0)
+      ethon (>= 0.9.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  awesome_print
+  colorator
+  github-markup
+  jekyll
+  jekyll-optional-front-matter
+  jekyll-readme-index
+  jekyll-relative-links
+  jekyll-sitemap
+  jekyll-titles-from-headings
+  liquid
+  mercurial-ruby
+  nokogiri
+  pandoc-ruby
+  pygments.rb
+  redcarpet
+  rugged
+  typhoeus
+
+BUNDLED WITH
+   1.16.2
diff --git a/Makefile b/Makefile
@@ -1,11 +1,12 @@
+site_path ?= _site
+
 devel_config_file=_config_devel.yml
 
 data_dir=_data
 cache_dir=_cache
 remotes_dir=_remotes
 plugins_data_dir=_plugins_data
 remotes_file=$(data_dir)/remotes.yml
-deploy_dir=$(remotes_dir)/deploy
 
 config_file=_config.yml
 index_file=index.yml
@@ -14,39 +15,30 @@ update_config=_config/update.yml
 scrape_config=_config/scrape.yml
 search_config=_config/search_index.yml
 
-# This target is invoked by a doc_independent job on the ROS buildfarm.
-html: build deploy
-
-build: download-previous discover update scrape search-index
+build: prepare-sources
+	bundle exec jekyll build --verbose --trace -d $(site_path) --config=$(config_file),$(index_file)
 
-download-previous:
+prepare-sources:
 	mkdir -p $(remotes_dir)
 	vcs import --input $(remotes_file) --force $(remotes_dir)
 
-discover:
-	bundle exec jekyll build --verbose --trace -d $(deploy_dir) --config=$(config_file),$(index_file),$(discover_config)
-
-update:
-	bundle exec jekyll build --verbose --trace -d $(deploy_dir) --config=$(config_file),$(index_file),$(update_config)
+discover: prepare-sources
+	bundle exec jekyll build --verbose --trace -d $(site_path) --config=$(config_file),$(index_file),$(discover_config)
 
-scrape:
-	bundle exec jekyll build --verbose --trace -d $(deploy_dir) --config=$(config_file),$(index_file),$(scrape_config)
+update: prepare-sources
+	bundle exec jekyll build --verbose --trace -d $(site_path) --config=$(config_file),$(index_file),$(update_config)
 
-search-index:
-	bundle exec jekyll build --verbose --trace -d $(deploy_dir) --config=$(config_file),$(index_file),$(search_config)
+scrape: prepare-sources
+	bundle exec jekyll build --verbose --trace -d $(site_path) --config=$(config_file),$(index_file),$(scrape_config)
 
-# deploy assumes build has run already
-deploy:
-	cd $(deploy_dir) && git add --all
-	cd $(deploy_dir) && git status
-	cd $(deploy_dir) && git commit -m "make deploy by `whoami` on `date`"
-	cd $(deploy_dir) && git push --verbose
+search-index: prepare-sources
+	bundle exec jekyll build --verbose --trace -d $(site_path) --config=$(config_file),$(index_file),$(search_config)
 
 serve:
-	bundle exec jekyll serve --host 0.0.0.0 --trace -d $(deploy_dir) --config=$(config_file),$(index_file) --skip-initial-build
+	bundle exec jekyll serve --host 0.0.0.0 --no-watch --trace -d $(site_path) --config=$(config_file),$(index_file) --skip-initial-build
 
 serve-devel:
-	bundle exec jekyll serve --host 0.0.0.0 --no-watch -d $(deploy_dir) --trace --config=$(config_file),$(devel_config_file),$(index_file) --skip-initial-build
+	bundle exec jekyll serve --host 0.0.0.0 --no-watch --trace -d $(site_path) --config=$(config_file),$(devel_config_file),$(index_file) --skip-initial-build
 
 clean-sources:
 	rm -rf $(plugins_data_dir)

diff --git a/README.md b/README.md
@@ -14,22 +14,24 @@ and uses client-side javascript for quick searching and visualization.
 
 [![Stories in Ready](https://badge.waffle.io/rosindex/rosindex.svg?label=ready&title=Ready)](http://waffle.io/rosindex/rosindex)
 
-## Building
+## Building the site
 
-### Pre-Requisites
+### On an Ubuntu 16.04 box
 
-#### Basic Ubuntu 16.04 Deps
+#### Pre-Requisites
 
-```
+##### Dependencies
+
+```bash
 sudo sh -c 'echo "deb http://packages.ros.org/ros/ubuntu $(lsb_release -sc) main" > /etc/apt/sources.list.d/ros-latest.list'
 apt-key adv --keyserver hkp://ha.pool.sks-keyservers.net:80 --recv-key 421C365BD9FF1F717815A3895523BAEEB01FA116
 sudo apt-get update
 sudo apt-get install curl git git-svn mercurial nodejs pandoc python3-vcstool
 ```
 
-#### Ruby 2.2 via RVM
+##### Ruby 2.2 via RVM
 
-```
+```bash
 curl -L https://get.rvm.io | bash -s stable
 # if this fails, add the PGP key and run again
 source ~/.rvm/scripts/rvm
@@ -38,51 +40,74 @@ rvm install ruby
 rvm rubygems current
 ```
 
-#### Node.js on Ubuntu 12.04
+##### Node.js on Ubuntu 12.04
 
-```
+```bash
 sudo apt-get install python-software-properties
 sudo apt-add-repository ppa:chris-lea/node.js
 sudo apt-get update
 sudo apt-get install nodejs
 ```
 
-#### Ruby Requirements
+##### Ruby Requirements
 
-```
+```bash
 gem install bundler
 ```
 
 #### Clone Source and Install Gems
 
-```
+```bash
 git clone [email protected]:ros2/rosindex.git --recursive
-cd rosindex.github.io
+cd rosindex
 bundle install
 ```
 
-### Clone repos that are part of rosdistro and build the index
+#### Clone repos that are part of rosdistro and build the index
 
-```
+Run:
+
+```bash
 make build
 ```
 
-### Serve the devel (tiny) version locally
+By default, site will be written to _site. This behavior can be
+overriden as follows:
 
+```bash
+make build site_path=/path/to/site
 ```
-make serve-devel
+
+### On the provided Ubuntu 16.04 container
+
+#### Pre-requisites
+
+##### Docker
+
+See https://docs.docker.com/install/linux/ for details on docker installation.
+
+#### Build docker image
+
+```bash
+docker/build.sh
 ```
 
-### Serve the full version locally
+#### Build the index inside the container
 
-**Note:** This requires a minimum of 30GB of
-free space for the `checkout` directory.
+Run:
 
+```bash
+docker/run.sh
+make build  # once inside the container
 ```
-make serve
+
+Or the following can be used as a shorthand:
+
+```bash
+docker/run.sh build_site
 ```
 
-### Skipping Parts of the Build
+## Skipping parts of the build
 
 The build process entails four long-running steps:
 
@@ -107,10 +132,71 @@ skip_scrape: false
 skip_search_index: false
 ```
 
-## Deployment
+Additionally, some make targets are provided for convenience:
+
+- To skip everything but repo discovering:
+
+  ```bash
+  make discover   #
+  ```
+
+- To skip everything but repo updates:
+
+  ```bash
+  make update
+  ```
 
-Deployment is done by calling the following make command:
+- To skip everything but repo scraping:
 
+  ```bash
+  make scrape
+  ```
+
+- To skip everything but a search index build:
+
+  ```bash
+  make search-index
+  ```
+
+Note that skipping parts of the rosindex build does not interfere with
+Jekyll's build process (e.g. generated files are still written to site).
+
+
+## Serving the site
+
+### Serve the devel (tiny) version locally
+
+Run:
+
+```bash
+make serve-devel
+```
+
+The following can be used as a shorthand if using
+docker containers:
+
+```bash
+docker/run.sh test_site
 ```
-make deploy
+
+### Serve the full version locally
+
+Run:
+
+```bash
+make serve
 ```
+
+**Note:** This requires a minimum of 30GB of free space for the
+`checkout` directory.
+
+## Deployment
+
+Deployment is not managed by these tools. It is to be managed
+externally e.g. using a local repository as site destination.
+
+## ROS buildfarm integration
+
+ROSIndex qualifies as independent documentation of **'external_site'**
+type. Therefore, it can readily be built and deployed as Github Pages by
+a *doc_independent* job.