diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..4a29b41c --- /dev/null +++ b/.dockerignore @@ -0,0 +1,4 @@ +_cache +_sphinx +_remotes +_plugins_data diff --git a/.gitignore b/.gitignore index a16653ad..1b2a37ac 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,7 @@ _site .sass-cache _devel _roswiki +_cache +_sphinx +_remotes +_plugins_data diff --git a/Gemfile b/Gemfile index fd1bb5e0..81ce7a9c 100644 --- a/Gemfile +++ b/Gemfile @@ -5,11 +5,8 @@ gem 'colorator' #gem 'git' gem 'github-markup' gem 'jekyll' -gem 'jekyll-optional-front-matter' -gem 'jekyll-readme-index' gem 'jekyll-relative-links' gem 'jekyll-sitemap' -gem 'jekyll-titles-from-headings' gem 'liquid'#, '~> 2.6.2' gem 'mercurial-ruby' gem 'nokogiri' diff --git a/Gemfile.lock b/Gemfile.lock new file mode 100644 index 00000000..78bb5e8c --- /dev/null +++ b/Gemfile.lock @@ -0,0 +1,107 @@ +GEM + remote: https://rubygems.org/ + specs: + addressable (2.5.2) + public_suffix (>= 2.0.2, < 4.0) + awesome_print (1.8.0) + colorator (1.1.0) + concurrent-ruby (1.0.5) + em-websocket (0.5.1) + eventmachine (>= 0.12.9) + http_parser.rb (~> 0.6.0) + ethon (0.11.0) + ffi (>= 1.3.0) + eventmachine (1.2.7) + ffi (1.9.25) + forwardable-extended (2.6.0) + github-markup (2.0.1) + http_parser.rb (0.6.0) + i18n (0.9.5) + concurrent-ruby (~> 1.0) + jekyll (3.8.4) + addressable (~> 2.4) + colorator (~> 1.0) + em-websocket (~> 0.5) + i18n (~> 0.7) + jekyll-sass-converter (~> 1.0) + jekyll-watch (~> 2.0) + kramdown (~> 1.14) + liquid (~> 4.0) + mercenary (~> 0.3.3) + pathutil (~> 0.9) + rouge (>= 1.7, < 4) + safe_yaml (~> 1.0) + jekyll-optional-front-matter (0.3.0) + jekyll (~> 3.0) + jekyll-readme-index (0.2.0) + jekyll (~> 3.0) + jekyll-relative-links (0.5.3) + jekyll (~> 3.3) + jekyll-sass-converter (1.5.2) + sass (~> 3.4) + jekyll-sitemap (1.2.0) + jekyll (~> 3.3) + jekyll-titles-from-headings (0.5.1) + jekyll (~> 3.3) + jekyll-watch (2.1.1) + listen (~> 3.0) + kramdown (1.17.0) + liquid (4.0.1) + listen (3.1.5) + rb-fsevent (~> 0.9, >= 0.9.4) + rb-inotify (~> 0.9, >= 0.9.7) + ruby_dep (~> 1.2) + mercenary (0.3.6) + mercurial-ruby (0.7.12) + open4 (~> 1.3.0) + mini_portile2 (2.3.0) + multi_json (1.13.1) + nokogiri (1.8.5) + mini_portile2 (~> 2.3.0) + open4 (1.3.4) + pandoc-ruby (2.0.2) + pathutil (0.16.1) + forwardable-extended (~> 2.6) + public_suffix (3.0.3) + pygments.rb (1.2.1) + multi_json (>= 1.0.0) + rb-fsevent (0.10.3) + rb-inotify (0.9.10) + ffi (>= 0.5.0, < 2) + redcarpet (3.4.0) + rouge (3.3.0) + ruby_dep (1.5.0) + rugged (0.27.5) + safe_yaml (1.0.4) + sass (3.6.0) + sass-listen (~> 4.0.0) + sass-listen (4.0.0) + rb-fsevent (~> 0.9, >= 0.9.4) + rb-inotify (~> 0.9, >= 0.9.7) + typhoeus (1.3.0) + ethon (>= 0.9.0) + +PLATFORMS + ruby + +DEPENDENCIES + awesome_print + colorator + github-markup + jekyll + jekyll-optional-front-matter + jekyll-readme-index + jekyll-relative-links + jekyll-sitemap + jekyll-titles-from-headings + liquid + mercurial-ruby + nokogiri + pandoc-ruby + pygments.rb + redcarpet + rugged + typhoeus + +BUNDLED WITH + 1.16.2 diff --git a/Makefile b/Makefile index 08348cfd..5d6df2e3 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,5 @@ +site_path ?= _site + devel_config_file=_config_devel.yml data_dir=_data @@ -5,7 +7,6 @@ cache_dir=_cache remotes_dir=_remotes plugins_data_dir=_plugins_data remotes_file=$(data_dir)/remotes.yml -deploy_dir=$(remotes_dir)/deploy config_file=_config.yml index_file=index.yml @@ -14,39 +15,30 @@ update_config=_config/update.yml scrape_config=_config/scrape.yml search_config=_config/search_index.yml -# This target is invoked by a doc_independent job on the ROS buildfarm. -html: build deploy - -build: download-previous discover update scrape search-index +build: prepare-sources + bundle exec jekyll build --verbose --trace -d $(site_path) --config=$(config_file),$(index_file) -download-previous: +prepare-sources: mkdir -p $(remotes_dir) vcs import --input $(remotes_file) --force $(remotes_dir) -discover: - bundle exec jekyll build --verbose --trace -d $(deploy_dir) --config=$(config_file),$(index_file),$(discover_config) - -update: - bundle exec jekyll build --verbose --trace -d $(deploy_dir) --config=$(config_file),$(index_file),$(update_config) +discover: prepare-sources + bundle exec jekyll build --verbose --trace -d $(site_path) --config=$(config_file),$(index_file),$(discover_config) -scrape: - bundle exec jekyll build --verbose --trace -d $(deploy_dir) --config=$(config_file),$(index_file),$(scrape_config) +update: prepare-sources + bundle exec jekyll build --verbose --trace -d $(site_path) --config=$(config_file),$(index_file),$(update_config) -search-index: - bundle exec jekyll build --verbose --trace -d $(deploy_dir) --config=$(config_file),$(index_file),$(search_config) +scrape: prepare-sources + bundle exec jekyll build --verbose --trace -d $(site_path) --config=$(config_file),$(index_file),$(scrape_config) -# deploy assumes build has run already -deploy: - cd $(deploy_dir) && git add --all - cd $(deploy_dir) && git status - cd $(deploy_dir) && git commit -m "make deploy by `whoami` on `date`" - cd $(deploy_dir) && git push --verbose +search-index: prepare-sources + bundle exec jekyll build --verbose --trace -d $(site_path) --config=$(config_file),$(index_file),$(search_config) serve: - bundle exec jekyll serve --host 0.0.0.0 --trace -d $(deploy_dir) --config=$(config_file),$(index_file) --skip-initial-build + bundle exec jekyll serve --host 0.0.0.0 --no-watch --trace -d $(site_path) --config=$(config_file),$(index_file) --skip-initial-build serve-devel: - bundle exec jekyll serve --host 0.0.0.0 --no-watch -d $(deploy_dir) --trace --config=$(config_file),$(devel_config_file),$(index_file) --skip-initial-build + bundle exec jekyll serve --host 0.0.0.0 --no-watch --trace -d $(site_path) --config=$(config_file),$(devel_config_file),$(index_file) --skip-initial-build clean-sources: rm -rf $(plugins_data_dir) diff --git a/README.md b/README.md index 9a5c40af..d6119955 100644 --- a/README.md +++ b/README.md @@ -14,22 +14,24 @@ and uses client-side javascript for quick searching and visualization. [![Stories in Ready](https://badge.waffle.io/rosindex/rosindex.svg?label=ready&title=Ready)](http://waffle.io/rosindex/rosindex) -## Building +## Building the site -### Pre-Requisites +### On an Ubuntu 16.04 box -#### Basic Ubuntu 16.04 Deps +#### Pre-Requisites -``` +##### Dependencies + +```bash sudo sh -c 'echo "deb http://packages.ros.org/ros/ubuntu $(lsb_release -sc) main" > /etc/apt/sources.list.d/ros-latest.list' apt-key adv --keyserver hkp://ha.pool.sks-keyservers.net:80 --recv-key 421C365BD9FF1F717815A3895523BAEEB01FA116 sudo apt-get update sudo apt-get install curl git git-svn mercurial nodejs pandoc python3-vcstool ``` -#### Ruby 2.2 via RVM +##### Ruby 2.2 via RVM -``` +```bash curl -L https://get.rvm.io | bash -s stable # if this fails, add the PGP key and run again source ~/.rvm/scripts/rvm @@ -38,51 +40,74 @@ rvm install ruby rvm rubygems current ``` -#### Node.js on Ubuntu 12.04 +##### Node.js on Ubuntu 12.04 -``` +```bash sudo apt-get install python-software-properties sudo apt-add-repository ppa:chris-lea/node.js sudo apt-get update sudo apt-get install nodejs ``` -#### Ruby Requirements +##### Ruby Requirements -``` +```bash gem install bundler ``` #### Clone Source and Install Gems -``` +```bash git clone git@github.com:ros2/rosindex.git --recursive -cd rosindex.github.io +cd rosindex bundle install ``` -### Clone repos that are part of rosdistro and build the index +#### Clone repos that are part of rosdistro and build the index -``` +Run: + +```bash make build ``` -### Serve the devel (tiny) version locally +By default, site will be written to _site. This behavior can be +overriden as follows: +```bash +make build site_path=/path/to/site ``` -make serve-devel + +### On the provided Ubuntu 16.04 container + +#### Pre-requisites + +##### Docker + +See https://docs.docker.com/install/linux/ for details on docker installation. + +#### Build docker image + +```bash +docker/build.sh ``` -### Serve the full version locally +#### Build the index inside the container -**Note:** This requires a minimum of 30GB of -free space for the `checkout` directory. +Run: +```bash +docker/run.sh +make build # once inside the container ``` -make serve + +Or the following can be used as a shorthand: + +```bash +docker/run.sh build_site ``` -### Skipping Parts of the Build +## Skipping parts of the build The build process entails four long-running steps: @@ -107,10 +132,71 @@ skip_scrape: false skip_search_index: false ``` -## Deployment +Additionally, some make targets are provided for convenience: + +- To skip everything but repo discovering: + + ```bash + make discover # + ``` + +- To skip everything but repo updates: + + ```bash + make update + ``` -Deployment is done by calling the following make command: +- To skip everything but repo scraping: + ```bash + make scrape + ``` + +- To skip everything but a search index build: + + ```bash + make search-index + ``` + +Note that skipping parts of the rosindex build does not interfere with +Jekyll's build process (e.g. generated files are still written to site). + + +## Serving the site + +### Serve the devel (tiny) version locally + +Run: + +```bash +make serve-devel +``` + +The following can be used as a shorthand if using +docker containers: + +```bash +docker/run.sh test_site ``` -make deploy + +### Serve the full version locally + +Run: + +```bash +make serve ``` + +**Note:** This requires a minimum of 30GB of free space for the +`checkout` directory. + +## Deployment + +Deployment is not managed by these tools. It is to be managed +externally e.g. using a local repository as site destination. + +## ROS buildfarm integration + +ROSIndex qualifies as independent documentation of **'external_site'** +type. Therefore, it can readily be built and deployed as Github Pages by +a *doc_independent* job. diff --git a/_config.yml b/_config.yml index 66f610c7..071fe1d1 100644 --- a/_config.yml +++ b/_config.yml @@ -34,6 +34,17 @@ report_diff_filename: _cache/_deploy_index_report_diff.yaml debbuild_release_path: _cache/debbuild_releases wiki_title_index_filename: _cache/ros_org_wiki_title_index.txt +# If true, this skips finding repos based on the repo sources +skip_discover: false +# If true, this skips updating the known repos +skip_update: false +# If true, this skips scraping the cloned repos +skip_scrape: false +# If true, this skips generating the search index +skip_search_index: false +# Shard count for search index partitioning. +search_index_shards: 20 + checkout_threads: 4 checkout_path: _plugins_data/checkout @@ -82,7 +93,7 @@ distros: old_distros: - 'jade' - 'hydro' - + # package list page packages_per_page: 50 repos_per_page: 50 @@ -95,7 +106,4 @@ readme_index: plugins: - jekyll-sitemap - - jekyll-optional-front-matter - - jekyll-titles-from-headings - - jekyll-readme-index - jekyll-relative-links diff --git a/_config/search_index.yml b/_config/search_index.yml index b97fe41b..047a3d9e 100644 --- a/_config/search_index.yml +++ b/_config/search_index.yml @@ -6,5 +6,3 @@ skip_update: true skip_scrape: true # If true, this skips generating the search index skip_search_index: false -# Shard count for search index partitioning. -search_index_shards: 20 diff --git a/_data/remotes.yml b/_data/remotes.yml index b5f6763a..c3091e35 100644 --- a/_data/remotes.yml +++ b/_data/remotes.yml @@ -2,26 +2,22 @@ repositories: rosdistro: type: git - url: git@github.com:ros/rosdistro.git + url: https://github.com/ros/rosdistro.git version: master ros2_rosdistro: type: git - url: git@github.com:ros2/rosdistro.git + url: https://github.com/ros2/rosdistro.git version: ros2 rosdep: type: git - url: git@github.com:ros/rosdistro.git + url: https://github.com/ros/rosdistro.git version: master rosforks: type: git - url: git@github.com:rosindex/rosforks.git + url: https://github.com/rosindex/rosforks.git version: master - deploy: - type: git - url: git@github.com:ros-infrastructure/index.ros.org.git - version: gh-pages # List of all non-package specific documentation repositories. ros2_overview: type: git - url: git@github.com:ros2/ros2_documentation.git + url: https://github.com/ros2/ros2_documentation.git version: master diff --git a/deploy.sh b/deploy.sh deleted file mode 100755 index a4bbcc01..00000000 --- a/deploy.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env sh - -git branch -D master -git checkout -b master -git add -f _site -git commit -m "deploying" -git filter-branch --subdirectory-filter _site/ -f -git push -f origin master -git checkout source - diff --git a/docker/Dockerfile b/docker/Dockerfile deleted file mode 100644 index 2e7fcae6..00000000 --- a/docker/Dockerfile +++ /dev/null @@ -1,71 +0,0 @@ -from ubuntu:xenial - -RUN echo "deb http://packages.ros.org/ros/ubuntu xenial main" > /etc/apt/sources.list.d/ros-latest.list -RUN apt-key adv --keyserver hkp://pool.sks-keyservers.net --recv-key 0xB01FA116 - -RUN apt-get update && \ - apt-get install --no-install-recommends -y \ - apt-utils \ - build-essential \ - bundler \ - cmake \ - curl \ - git-all \ - git-svn \ - libgit2-dev \ - libpthread-stubs0-dev \ - libssh-dev \ - libssh2-1 \ - libssh2-1-dev \ - libssl-dev \ - libsslcommon2-dev \ - libz-dev \ - mercurial \ - nodejs \ - openssl \ - pandoc \ - pkg-config \ - python-pip \ - python3-vcstool \ - ruby \ - ruby-dev \ - rubygems-integration \ - ssh \ - sudo \ - tmux && \ - rm -rf /var/lib/apt/lists/* - -RUN pip install --upgrade setuptools pip -RUN pip install \ - recommonmark==0.4 \ - sphinx==1.5.6 - -RUN ln -s `which nodejs` /usr/local/bin/node - -#RUN curl -sSL https://rvm.io/mpapis.asc | gpg --import - -#RUN curl -L https://get.rvm.io | bash -s stable - -#RUN . /usr/local/rvm/src/rvm/scripts/rvm -#RUN echo "source /usr/local/rvm/src/rvm/scripts/rvm" >> ~/.bashrc - -#RUN /usr/local/rvm/src/rvm/bin/rvm requirements -#RUN /usr/local/rvm/src/rvm/bin/rvm install ruby -#RUN /usr/local/rvm/src/rvm/bin/rvm rubygems current - -#RUN apt-get install -y python-software-properties -#RUN apt-add-repository -y ppa:chris-lea/node.js -#RUN apt-get update -#RUN apt-get install nodejs - -VOLUME /workdir - -RUN mkdir /root/.ssh && \ - touch /root/.ssh/known_hosts && \ - ssh-keyscan github.com >> /root/.ssh/known_hosts && \ - ssh-keyscan bitbucket.org >> /root/.ssh/known_hosts - -WORKDIR /workdir -COPY Gemfile /workdir -RUN bundle install - -WORKDIR /workdir/rosindex diff --git a/docker/build.sh b/docker/build.sh index 312cb5a9..5b9072f8 100755 --- a/docker/build.sh +++ b/docker/build.sh @@ -1,4 +1,6 @@ -#!/usr/bin/env bash +#!/bin/bash SCRIPT_DIR=`dirname $( readlink -m $( type -p $0 ))` -docker build -f $SCRIPT_DIR/Dockerfile -t rosindex/rosindex $SCRIPT_DIR/.. +docker build -f $SCRIPT_DIR/image/Dockerfile \ + --build-arg user=`whoami` --build-arg uid=`id -u` \ + -t rosindex/rosindex $SCRIPT_DIR/.. diff --git a/docker/image/Dockerfile b/docker/image/Dockerfile new file mode 100644 index 00000000..c8305fce --- /dev/null +++ b/docker/image/Dockerfile @@ -0,0 +1,66 @@ +FROM ubuntu:xenial + +ARG user=rosindex +ARG uid=1000 + +ENV DEBIAN_FRONTEND noninteractive + +RUN echo "deb http://packages.ros.org/ros/ubuntu xenial main" > \ + /etc/apt/sources.list.d/ros-latest.list +RUN apt-key adv --keyserver hkp://pool.sks-keyservers.net --recv-key 0xB01FA116 + +RUN apt-get update && \ + apt-get install --no-install-recommends -y \ + apt-utils \ + build-essential \ + cmake \ + curl \ + git-all \ + git-svn \ + libgit2-dev \ + libpthread-stubs0-dev \ + libssl-dev \ + libsslcommon2-dev \ + libz-dev \ + mercurial \ + nodejs \ + openssl \ + pandoc \ + pkg-config \ + python-pip \ + python3-vcstool \ + ruby \ + ruby-dev \ + rubygems-integration && \ + rm -rf /var/lib/apt/lists/* + +RUN gem install bundle +RUN pip install --upgrade setuptools pip +RUN pip install recommonmark==0.4 sphinx==1.5.6 + +RUN ln -s `which nodejs` /usr/local/bin/node + +COPY docker/image/build_site.sh /usr/local/bin/build_site +COPY docker/image/test_site.sh /usr/local/bin/test_site +COPY docker/image/update_site.sh /usr/local/bin/update_site + +RUN useradd -u $uid -m $user +ENV HOME=/home/$user +WORKDIR $HOME +COPY Gemfile* .bundle/ +RUN chown -R $user:$user . + +USER $user +ENV BUNDLE_PATH=$HOME/.bundle +ENV BUNDLE_GEMFILE=$HOME/.bundle/Gemfile +RUN bundle install + +RUN git config --global user.name "rosindex" +RUN git config --global user.email "rosindex@build.ros.org" + +# The environment variables REPO and SITE should be set when running this container. +# REPO: The path to this source repository in the container +# SITE: The path to the destination repository in the container +ENV REPO "" +ENV SITE "" +CMD update_site ${REPO} ${SITE} diff --git a/docker/image/build_site.sh b/docker/image/build_site.sh new file mode 100755 index 00000000..38a4ee80 --- /dev/null +++ b/docker/image/build_site.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +REPO_PATH=${1:-`pwd`} +SITE_PATH=${2:-${REPO_PATH}/_site} +make -C $REPO_PATH build site_path=$SITE_PATH diff --git a/docker/image/test_site.sh b/docker/image/test_site.sh new file mode 100755 index 00000000..59ef452f --- /dev/null +++ b/docker/image/test_site.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +REPO_PATH=${1:-`pwd`} +SITE_PATH=${2:-${REPO_PATH}/_site} +make -C $REPO_PATH serve-devel site_path=$SITE_PATH diff --git a/docker/image/update_site.sh b/docker/image/update_site.sh new file mode 100755 index 00000000..445b6f56 --- /dev/null +++ b/docker/image/update_site.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +REPO_PATH=${1:-`pwd`} +SITE_PATH=${2:-${REPO_PATH}/_site} +build_site $REPO_PATH $SITE_PATH +git -C $SITE_PATH commit -a -m "ROSIndex deployment by `whoami`" diff --git a/docker/run.sh b/docker/run.sh index cee833c6..d2b9a8b5 100755 --- a/docker/run.sh +++ b/docker/run.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/bin/bash SCRIPT_DIR=`dirname $( readlink -m $( type -p $0 ))` # Ensure your SSH private key has been added to the ssh-agent as follows: @@ -16,10 +16,6 @@ else fi docker run \ - --env SSH_AUTH_SOCK=/ssh-agent \ - -v $SSH_AUTH_SOCK:/ssh-agent \ - -v $SCRIPT_DIR/..:/workdir/rosindex:rw \ - --net=host \ - -p 4000:4000 \ - -ti rosindex/rosindex \ - $COMMAND + -w /home/`whoami`/rosindex \ + -v $SCRIPT_DIR/..:/home/`whoami`/rosindex:rw \ + --net=host -ti rosindex/rosindex $COMMAND diff --git a/docker/test_site.sh b/docker/test_site.sh deleted file mode 100755 index 4cff5de5..00000000 --- a/docker/test_site.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env bash -SCRIPT_DIR=`dirname $( readlink -m $( type -p $0 ))` - -$SCRIPT_DIR/run.sh make build serve-devel diff --git a/node_modules/lunr-index-build/bin/lunr-index-build b/node_modules/lunr-index-build/bin/lunr-index-build index 0548153d..a5cd20ab 100755 --- a/node_modules/lunr-index-build/bin/lunr-index-build +++ b/node_modules/lunr-index-build/bin/lunr-index-build @@ -2,7 +2,6 @@ var lunr = require('lunr'), optimist = require('optimist'), - ProgressBar = require('progress'), vuvuzela = require('vuvuzela') var argv = optimist @@ -37,14 +36,6 @@ process.stdin.on('end', function () { process.exit(1) } - var bar = new ProgressBar (' indexing [:bar] :percent :etas', { - total: documents.length, - width: 20, - complete: '=', - incomplete: ' ', - stream: process.stderr - }) - var idx = lunr(function () { if (argv.ref) this.ref(argv.ref) @@ -58,7 +49,6 @@ process.stdin.on('end', function () { documents.forEach(function (document) { this.add(document) - bar.tick(1) }, this) })