Merge branch 'master' into 8057_sklearn_pr

Author: Joan Massich

.travis.yml

@@ -38,7 +38,7 @@ matrix:
     # This environment tests the newest supported Anaconda release (4.4.0)
     # It also runs tests requiring Pandas.
     - env: DISTRIB="conda" PYTHON_VERSION="3.6.1" INSTALL_MKL="true"
-           NUMPY_VERSION="1.12.1" SCIPY_VERSION="0.19.0" PANDAS_VERSION="0.20.1"
+           NUMPY_VERSION="1.13" SCIPY_VERSION="0.19.0" PANDAS_VERSION="0.20.2"
            CYTHON_VERSION="0.25.2" COVERAGE=true
     # This environment use pytest to run the tests. It uses the newest
     # supported Anaconda release (4.4.0). It also runs tests requiring Pandas.
@@ -49,7 +49,7 @@ matrix:
     # flake8 linting on diff wrt common ancestor with upstream/master
     - env: RUN_FLAKE8="true" SKIP_TESTS="true"
            DISTRIB="conda" PYTHON_VERSION="3.5" INSTALL_MKL="true"
-           NUMPY_VERSION="1.12.1" SCIPY_VERSION="0.19.0" CYTHON_VERSION="0.23.5"
+           NUMPY_VERSION="1.13" SCIPY_VERSION="0.19.0" CYTHON_VERSION="0.23.5"
     # This environment tests scikit-learn against numpy and scipy master
     # installed from their CI wheels in a virtualenv with the Python
     # interpreter provided by travis.

build_tools/circle/build_doc.sh

@@ -109,7 +109,7 @@ conda update --yes --quiet conda
 conda create -n $CONDA_ENV_NAME --yes --quiet python numpy scipy \
   cython nose coverage matplotlib sphinx=1.6.2 pillow
 source activate testenv
-pip install numpydoc
+pip install sphinx-gallery numpydoc
 
 # Build and install scikit-learn in dev mode
 python setup.py develop

build_tools/travis/install.sh

@@ -39,22 +39,30 @@ if [[ "$DISTRIB" == "conda" ]]; then
 
     # Configure the conda environment and put it in the path using the
     # provided versions
+    if [[ "$USE_PYTEST" == "true" ]]; then
+        TEST_RUNNER_PACKAGE=pytest
+    else
+        TEST_RUNNER_PACKAGE=nose
+    fi
+
     if [[ "$INSTALL_MKL" == "true" ]]; then
-        conda create -n testenv --yes python=$PYTHON_VERSION pip nose pytest \
-            numpy=$NUMPY_VERSION scipy=$SCIPY_VERSION \
+        conda create -n testenv --yes python=$PYTHON_VERSION pip \
+            $TEST_RUNNER_PACKAGE numpy=$NUMPY_VERSION scipy=$SCIPY_VERSION \
             mkl cython=$CYTHON_VERSION \
             ${PANDAS_VERSION+pandas=$PANDAS_VERSION}
 
     else
-        conda create -n testenv --yes python=$PYTHON_VERSION pip nose pytest \
-            numpy=$NUMPY_VERSION scipy=$SCIPY_VERSION \
+        conda create -n testenv --yes python=$PYTHON_VERSION pip \
+            $TEST_RUNNER_PACKAGE numpy=$NUMPY_VERSION scipy=$SCIPY_VERSION \
             nomkl cython=$CYTHON_VERSION \
             ${PANDAS_VERSION+pandas=$PANDAS_VERSION}
     fi
     source activate testenv
 
-    # Install nose-timer via pip
-    pip install nose-timer
+    if [[ $USE_PYTEST != "true" ]]; then
+        # Install nose-timer via pip
+        pip install nose-timer
+    fi
 
 elif [[ "$DISTRIB" == "ubuntu" ]]; then
     # At the time of writing numpy 1.9.1 is included in the travis

build_tools/travis/test_script.sh

@@ -43,10 +43,13 @@ run_tests() {
     fi
     $TEST_CMD sklearn
 
-    # Test doc (only with nose until we switch completely to pytest)
-    if [[ "$USE_PYTEST" != "true" ]]; then
-        # Going back to git checkout folder needed for make test-doc
-        cd $OLDPWD
+    # Going back to git checkout folder needed to test documentation
+    cd $OLDPWD
+
+    if [[ "$USE_PYTEST" == "true" ]]; then
+        pytest $(find doc -name '*.rst' | sort)
+    else
+        # Makefile is using nose
         make test-doc
     fi
 }

conftest.py

@@ -1,8 +1,13 @@
 # Documentation for scikit-learn
 
 This section contains the full manual and web page as displayed in
-http://scikit-learn.org. To generate the full web page, including
-the example gallery (this might take a while):
+http://scikit-learn.org.
+Building the website requires the sphinx and sphinx-gallery packages:
+
+    pip install sphinx sphinx-gallery
+
+To generate the full web page, including the example gallery (this might take a
+while):
 
     make html
 
@@ -16,7 +21,6 @@ To build the PDF manual, run
 
     make latexpdf
 
-
 The website is hosted at github and can be updated manually (for releases)
 by pushing to the https://github.com/scikit-learn/scikit-learn.github.io repository.
 

doc/README.md

@@ -0,0 +1,75 @@
+from os.path import exists
+from os.path import join
+
+import numpy as np
+
+from sklearn.utils.testing import SkipTest
+from sklearn.utils.testing import check_skip_network
+from sklearn.datasets import get_data_home
+from sklearn.utils.testing import install_mldata_mock
+from sklearn.utils.testing import uninstall_mldata_mock
+
+
+def setup_labeled_faces():
+    data_home = get_data_home()
+    if not exists(join(data_home, 'lfw_home')):
+        raise SkipTest("Skipping dataset loading doctests")
+
+
+def setup_mldata():
+    # setup mock urllib2 module to avoid downloading from mldata.org
+    install_mldata_mock({
+        'mnist-original': {
+            'data': np.empty((70000, 784)),
+            'label': np.repeat(np.arange(10, dtype='d'), 7000),
+        },
+        'iris': {
+            'data': np.empty((150, 4)),
+        },
+        'datasets-uci-iris': {
+            'double0': np.empty((150, 4)),
+            'class': np.empty((150,)),
+        },
+    })
+
+
+def teardown_mldata():
+    uninstall_mldata_mock()
+
+
+def setup_rcv1():
+    check_skip_network()
+    # skip the test in rcv1.rst if the dataset is not already loaded
+    rcv1_dir = join(get_data_home(), "RCV1")
+    if not exists(rcv1_dir):
+        raise SkipTest("Download RCV1 dataset to run this test.")
+
+
+def setup_twenty_newsgroups():
+    data_home = get_data_home()
+    if not exists(join(data_home, '20news_home')):
+        raise SkipTest("Skipping dataset loading doctests")
+
+
+def setup_working_with_text_data():
+    check_skip_network()
+
+
+def pytest_runtest_setup(item):
+    fname = item.fspath.strpath
+    if fname.endswith('datasets/labeled_faces.rst'):
+        setup_labeled_faces()
+    elif fname.endswith('datasets/mldata.rst'):
+        setup_mldata()
+    elif fname.endswith('datasets/rcv1.rst'):
+        setup_rcv1()
+    elif fname.endswith('datasets/twenty_newsgroups.rst'):
+        setup_twenty_newsgroups()
+    elif fname.endswith('datasets/working_with_text_data.rst'):
+        setup_working_with_text_data()
+
+
+def pytest_runtest_teardown(item):
+    fname = item.fspath.strpath
+    if fname.endswith('datasets/mldata.rst'):
+        teardown_mldata()

doc/datasets/conftest.py

@@ -3,6 +3,11 @@
 
     >>> import numpy as np
     >>> import os
+    >>> import tempfile
+    >>> # Create a temporary folder for the data fetcher
+    >>> custom_data_home = tempfile.mkdtemp()
+    >>> os.makedirs(os.path.join(custom_data_home, 'mldata'))
+
 
 .. _mldata:
 
@@ -70,3 +75,8 @@ defaults to individual datasets:
     ...                      data_home=custom_data_home)
     >>> iris3 = fetch_mldata('datasets-UCI iris', target_name='class',
     ...                      data_name='double0', data_home=custom_data_home)
+
+
+..
+    >>> import shutil
+    >>> shutil.rmtree(custom_data_home)

doc/datasets/mldata.rst

@@ -3,26 +3,12 @@
 Mock urllib2 access to mldata.org and create a temporary data folder.
 """
 
-from os import makedirs
-from os.path import join
 import numpy as np
-import tempfile
-import shutil
 
-from sklearn import datasets
 from sklearn.utils.testing import install_mldata_mock
 from sklearn.utils.testing import uninstall_mldata_mock
 
 
-def globs(globs):
-    # Create a temporary folder for the data fetcher
-    global custom_data_home
-    custom_data_home = tempfile.mkdtemp()
-    makedirs(join(custom_data_home, 'mldata'))
-    globs['custom_data_home'] = custom_data_home
-    return globs
-
-
 def setup_module():
     # setup mock urllib2 module to avoid downloading from mldata.org
     install_mldata_mock({
@@ -42,4 +28,3 @@ def setup_module():
 
 def teardown_module():
     uninstall_mldata_mock()
-    shutil.rmtree(custom_data_home)

doc/datasets/mldata_fixture.py

@@ -207,13 +207,13 @@
                     <li><em>On-going development:</em>
                     <a href="/dev/whats_new.html"><em>What's new</em> (Changelog)</a>
                     </li>
-                    <li><em>September 2016.</em> scikit-learn 0.18.0 is available for download (<a href="whats_new.html#version-0-18">Changelog</a>).
+                    <li><em>September 2016.</em> scikit-learn 0.18.0 is available for download (<a href="whats_new/v0.18.html">Changelog</a>).
                     </li>
-                    <li><em>November 2015.</em> scikit-learn 0.17.0 is available for download (<a href="whats_new.html#version-0-17">Changelog</a>).
+                    <li><em>November 2015.</em> scikit-learn 0.17.0 is available for download (<a href="whats_new/v0.17.html">Changelog</a>).
                     </li>
-                    <li><em>March 2015.</em> scikit-learn 0.16.0 is available for download (<a href="whats_new.html#version-0-16">Changelog</a>).
+                    <li><em>March 2015.</em> scikit-learn 0.16.0 is available for download (<a href="whats_new/v0.16.html">Changelog</a>).
                     </li>
-                    <li><em>July 2014.</em> scikit-learn 0.15.0 is available for download (<a href="whats_new.html#version-0-15">Changelog</a>).
+                    <li><em>July 2014.</em> scikit-learn 0.15.0 is available for download (<a href="whats_new/v0.15.html">Changelog</a>).
                     </li>
                     <li><em>July 14-20th, 2014: international sprint.</em>
                     During this week-long sprint, we gathered 18 of the core
@@ -227,7 +227,7 @@
                     <a href="http://www.inria.fr/">Inria</a>,
                     and <a href="http://www.tinyclues.com/">tinyclues</a>.
                     </li>
-                    <li><em>August 2013.</em> scikit-learn 0.14 is available for download (<a href="whats_new.html#version-0-14">Changelog</a>).
+                    <li><em>August 2013.</em> scikit-learn 0.14 is available for download (<a href="whats_new/v0.14.html">Changelog</a>).
                     </li>
                     </ul>
                 </div>