COMPUTE-191 Drop python 2.7 from dxpy (#1270)

* Require Python 3.6 or higher

* Trim Python 2.7 requirements

.gitignore

@@ -71,6 +71,7 @@ dxR.Rcheck
 /bin/proot
 /bin/pbr
 /bin/xattr
+/bin/normalizer
 /bin/wsdump.py
 /bin/dx-su-contrib
 /src/python/test/*.traceability.*.csv

CHANGELOG.md

@@ -6,8 +6,15 @@ Categories for each release: Added, Changed, Deprecated, Removed, Fixed, Securit
 
 ## Unreleased
 
+### Changed
+
+* Python >= 3.6 dxpy
 * Updated Nextflow to 23.10.0 (staging only).
 
+### Removed
+
+* Python 2.7 support
+
 ## [364.0] - beta
 
 ### Added
@@ -17,7 +24,7 @@ Categories for each release: Added, Changed, Deprecated, Removed, Fixed, Securit
 
 ### Changed
 
-* Disallow Python versions <2.7 or <3.5 in setup.py for dxpy, the next release will only support Python >=3.5
+* Disallow Python versions <2.7 or <3.5 in setup.py for dxpy, the next release will only support Python >=3.6
 
 ### Fixed
 

Readme.md

@@ -42,7 +42,7 @@ Supported languages
 
 The Platform SDK contains API language bindings for the following platforms:
 
-* [Python](src/python/Readme.md) (requires Python 3.5 or higher)
+* [Python](src/python/Readme.md) (requires Python 3.6 or higher)
 * C++
 * [Java](src/java/Readme.md) (requires Java 7 or higher)
 

build/run_java_integration_tests.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 from __future__ import print_function, unicode_literals
 

build/run_python_integration_tests.py

@@ -1,8 +1,10 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 '''
 Runs Python integration tests
 
+Assumes  `make python src_libs && source build/py_env/bin/activate`
+
 If no arguments are given, all tests in src/python/test/ are run.
 
 To run a single test method, e.g. test_basic in class TestDXBashHelpers,
@@ -41,17 +43,9 @@
 os.environ['DNANEXUS_INSTALL_PYTHON_TEST_DEPS'] = 'yes'
 
 def run():
-    # src_libs is to ensure that dx-unpack is runnable. If we had "bash unit
-    # tests" that were broken out separately, that would obviate this though.
-    #
-    # Note that Macs must run the make command before running this script,
-    # as of b9d8487 (when virtualenv was added to the Mac dx-toolkit release).
-    if sys.platform != "darwin":
-        subprocess.check_call(["make", "python", "src_libs"], cwd=TOOLKIT_ROOT_DIR)
-
-    python_version = "python{}.{}".format(sys.version_info.major, sys.version_info.minor)
+    # Assumes `make python src_libs` has been run
 
-    cmd = ['python', '-m', 'unittest']
+    cmd = ['python3', '-m', 'unittest']
     if args.tests:
         cmd += ['-v'] + args.tests
     else:

build/run_traceability_integration_tests.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 '''
 Runs Python integration tests tagged for inclusion in the traceability matrix.
@@ -19,13 +19,6 @@
 os.environ['DX_USER_CONF_DIR'] = TOOLKIT_ROOT_DIR + "/dnanexus_config_relocated"
 
 def run():
-    # src_libs is to ensure that dx-unpack is runnable. If we had "bash unit
-    # tests" that were broken out separately, that would obviate this though.
-    #
-    # Note that Macs must run the make command before running this script,
-    # as of b9d8487 (when virtualenv was added to the Mac dx-toolkit release).
-    if sys.platform != "darwin":
-        subprocess.check_call(["make", "python", "src_libs"], cwd=TOOLKIT_ROOT_DIR)
 
     cmd = ['py.test', '-vv', '-s', '-m', 'TRACEABILITY_MATRIX', 'src/python/test/']
 

build/run_traceability_integration_tests.sh

@@ -8,11 +8,9 @@ export DNANEXUS_INSTALL_PYTHON_TEST_DEPS="yes"
 export DX_USER_CONF_DIR="${TOOLKIT_ROOT_DIR}/dnanexus_config_relocated"
 
 cd $TOOLKIT_ROOT_DIR
-make python
+make src_libs python
 
-source build/py_env2.7/bin/activate
-source environment
+source build/py_env/bin/activate
 
-export PYTHONPATH="${TOOLKIT_ROOT_DIR}/src/python/test:${TOOLKIT_ROOT_DIR}/share/dnanexus/lib/python2.7/site-packages"
 #py.test -m TRACEABILITY_MATRIX src/python/test/test_dx_bash_helpers.py::TestDXBashHelpers::test_basic -sv
 py.test -vv -s -m TRACEABILITY_MATRIX ${TOOLKIT_ROOT_DIR}/src/python/test/

contrib/perl/generatePerlAPIWrappers.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 #
 # Copyright (C) 2013-2016 DNAnexus, Inc.
 #

contrib/ruby/generateRubyAPIWrappers.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 #
 # Copyright (C) 2013-2016 DNAnexus, Inc.
 #

src/api_wrappers/generateCppAPICCWrappers.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 #
 # Copyright (C) 2013-2016 DNAnexus, Inc.
 #

src/api_wrappers/generateCppAPIHWrappers.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 #
 # Copyright (C) 2013-2016 DNAnexus, Inc.
 #

src/api_wrappers/generateJavaAPIWrappers.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 #
 # Copyright (C) 2013-2016 DNAnexus, Inc.
 #

src/api_wrappers/generatePythonAPIWrappers.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 #
 # Copyright (C) 2013-2016 DNAnexus, Inc.
 #

src/api_wrappers/generateRAPIWrappers.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 #
 # Copyright (C) 2013-2016 DNAnexus, Inc.
 #

src/mk/Makefile.linux

@@ -15,20 +15,14 @@
 #   under the License.
 
 $(DNANEXUS_HOME)/bin/dx: $(shell find python/{dxpy,scripts,requirements*,setup*} -not -name toolkit_version*)
-	python -c 'import sys; exit("dx-toolkit is not compatible with Python < 2.7" if sys.version_info < (2, 7) else 0)'
-	rm -rf "$(PYTHON_LIBDIR)" "$(DX_PY_ENV)" python/dist
-	mkdir -p "$$(dirname '$(PYTHON_LIBDIR)')"
+	python3 -c 'import sys; exit("dx-toolkit is not compatible with Python < 3.6" if sys.version_info < (3, 6) else 0)'
+	rm -rf "$(DX_PY_ENV)"
 	$(VIRTUAL_ENV) "$(DX_PY_ENV)"
 
-# Install setuptools and other fundamental packages
 	unset PYTHONPATH; source "$(DX_PY_ENV)/$(ACTIVATE)"; ${PIP} install --upgrade -r python/requirements_setuptools.txt
 
-# Build the dxpy wheel and move it into place
-	unset PYTHONPATH; source "$(DX_PY_ENV)/$(ACTIVATE)"; cd python; python setup.py bdist_wheel
-
-	export DXPY_WHEEL_FILENAME=$$(basename python/dist/dxpy-*.whl) ; \
-		unset PYTHONPATH; source "$(DX_PY_ENV)/$(ACTIVATE)"; ${PIP} install --ignore-installed --prefix="$(DNANEXUS_HOME)" python/dist/$${DXPY_WHEEL_FILENAME}[xattr]
-	mv "$(DNANEXUS_HOME)"/lib/python?.[0-9]*/site-packages "$(PYTHON_LIBDIR)"
+	# Build the dxpy wheel and move it into place
+	unset PYTHONPATH; source "$(DX_PY_ENV)/$(ACTIVATE)"; ${PIP} install python/
 
 
 # Installation

src/mk/Makefile.osx

@@ -15,18 +15,14 @@
 #   under the License.
 
 $(DNANEXUS_HOME)/bin/dx: $(shell find python/{dxpy,scripts,requirements*,setup*} -not -name toolkit_version*)
-	python -c 'import sys; exit("dx-toolkit is not compatible with Python < 2.7" if sys.version_info < (2, 7) else 0)'
-	rm -rf "$(PYTHON_LIBDIR)" "$(DX_PY_ENV)" python/dist
-	mkdir -p "$$(dirname '$(PYTHON_LIBDIR)')"
+	python3 -c 'import sys; exit("dx-toolkit is not compatible with Python < 3.6" if sys.version_info < (3, 6) else 0)'
+	rm -rf "$(DX_PY_ENV)" python/dist
 	$(VIRTUAL_ENV) "$(DX_PY_ENV)"
 
 	unset PYTHONPATH; source "$(DX_PY_ENV)/$(ACTIVATE)"; ${PIP} install --upgrade -r python/requirements_setuptools.txt
 
 	# Build the dxpy wheel and move it into place
-	unset PYTHONPATH; source "$(DX_PY_ENV)/$(ACTIVATE)"; cd python; python setup.py bdist_wheel
-	unset PYTHONPATH; source "$(DX_PY_ENV)/$(ACTIVATE)"; pip install --ignore-installed --prefix="$(DNANEXUS_HOME)" python/dist/*.whl
-	mv "$(DNANEXUS_HOME)"/lib/python?.?/site-packages "${PYTHON_LIBDIR}"
-	rm -f "$(DNANEXUS_HOME)/bin/xattr"
+	unset PYTHONPATH; source "$(DX_PY_ENV)/$(ACTIVATE)"; ls -l; ${PIP} install python/
 
 # System dependencies
 # ===================

src/mk/config.mk

@@ -43,8 +43,8 @@ endif
 
 # Extract the two most significant digits the python distribution
 #
-PYTHON_VERSION_NUMBER:=$(shell python -c 'import sys; print("{}.{}".format(sys.version_info[0], sys.version_info[1]))')
-PYTHON_MAJOR_VERSION:=$(shell python -c 'import sys; print(sys.version_info[0])')
+PYTHON_VERSION_NUMBER:=$(shell python3 -c 'import sys; print("{}.{}".format(sys.version_info[0], sys.version_info[1]))')
+PYTHON_MAJOR_VERSION:=$(shell python3 -c 'import sys; print(sys.version_info[0])')
 
 ifeq (${PYTHON_MAJOR_VERSION}, 2)
 	PIP=pip
@@ -69,7 +69,7 @@ endif
 
 export DNANEXUS_HOME := $(CURDIR)/..
 export PATH := $(DNANEXUS_HOME)/build/bin:$(PATH)
-export DX_PY_ENV := $(DNANEXUS_HOME)/build/py_env${PYTHON_VERSION_NUMBER}
+export DX_PY_ENV := $(DNANEXUS_HOME)/build/py_env
 export DNANEXUS_LIBDIR := $(DNANEXUS_HOME)/share/dnanexus/lib
 
 # Short-circuit sudo when running as root. In a chrooted environment we are
@@ -81,8 +81,6 @@ else
 	MAYBE_SUDO='sudo'
 endif
 
-PYTHON_LIBDIR = $(DNANEXUS_LIBDIR)/python${PYTHON_VERSION_NUMBER}/site-packages
-
 
 ifeq ($(PLATFORM), windows)
         ACTIVATE=Scripts/activate

src/python/Readme.md

@@ -56,7 +56,7 @@ Other useful resources:
 
 Python version compatibility
 ----------------------------
-dxpy is supported on Python 3 (3.5+)
+dxpy is supported on Python 3 (3.6+)
 
 Convention for Python scripts that are also modules
 ---------------------------------------------------

src/python/dxpy/cli/dataset_utilities.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 #
 # Copyright (C) 2022 DNAnexus, Inc.

src/python/dxpy/executable_builder.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 #
 # Copyright (C) 2013-2016 DNAnexus, Inc.
 #

src/python/dxpy/nextflow/ImageRefParser.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 import re
 

src/python/dxpy/nextflow/nextflow_builder.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 import os
 import dxpy
 import json

src/python/dxpy/nextflow/nextflow_templates.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 from .nextflow_utils import (get_template_dir, get_source_file_name, get_resources_subpath,
                                           get_importer_name, get_regional_options)

src/python/dxpy/nextflow/nextflow_utils.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 from os import path, makedirs, listdir
 import re
 import errno

src/python/dxpy/scripts/dx.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # coding: utf-8
 #
 # Copyright (C) 2013-2016 DNAnexus, Inc.

src/python/dxpy/scripts/dx_app_wizard.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 #
 # Copyright (C) 2013-2016 DNAnexus, Inc.
 #

src/python/dxpy/scripts/dx_build_app.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 #
 # Copyright (C) 2013-2016 DNAnexus, Inc.
 #

src/python/dxpy/scripts/dx_build_applet.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 #
 # Copyright (C) 2013-2016 DNAnexus, Inc.
 #

src/python/dxpy/templating/templates/python/basic/src/code.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # DX_APP_WIZARD_NAME DX_APP_WIZARD_VERSION
 # Generated by dx-app-wizard.
 #

src/python/dxpy/templating/templates/python/basic/test/test.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # DX_APP_WIZARD_NAME DX_APP_WIZARD_VERSION test suite
 # Generated by dx-app-wizard.
 

src/python/dxpy/templating/templates/python/parallelized/src/code.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # DX_APP_WIZARD_NAME DX_APP_WIZARD_VERSION
 # Generated by dx-app-wizard.
 #

src/python/dxpy/templating/templates/python/parallelized/test/test.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # DX_APP_WIZARD_NAME DX_APP_WIZARD_VERSION test suite
 # Generated by dx-app-wizard.
 

src/python/dxpy/templating/templates/python/scatter-process-gather/src/code.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # coding: utf-8
 # DX_APP_WIZARD_NAME DX_APP_WIZARD_VERSION
 # Generated by dx-app-wizard.

src/python/dxpy/templating/templates/python/scatter-process-gather/test/test.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # DX_APP_WIZARD_NAME DX_APP_WIZARD_VERSION test suite
 # Generated by dx-app-wizard.
 

src/python/dxpy/utils/file_handle.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 #
 # Copyright (C) 2022 DNAnexus, Inc.

src/python/dxpy/utils/pretty_print.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 # Copyright (C) 2013-2016 DNAnexus, Inc.
 #

src/python/requirements.txt

@@ -5,11 +5,7 @@ python-dateutil>=2.5
 psutil>=5.9.3
 requests>=2.8.0,<2.28; python_version < "3.7"
 requests>=2.8.0,<2.29; python_version >= "3.7"
-cryptography==3.3.2; python_version < "3"
-cryptography==3.2.1; python_version > "3" and python_version < "3.6"
 cryptography>=3.4.2,<41; python_version > "3.5"
-pyreadline==2.1; sys_platform == "win32" and python_version < "3.5"
 pyreadline3==3.4.1; sys_platform == "win32" and python_version >= "3.5"
-colorama>=0.4.4,<=0.4.6; sys_platform == "win32" and python_version == "2.7"
 colorama==0.4.4; sys_platform == "win32" and python_version >= "3.5" and python_version < "3.7"
-colorama>=0.4.4,<=0.4.6; sys_platform == "win32" and python_version >= "3.7"
+colorama>=0.4.4,<=0.4.6; sys_platform == "win32" and python_version >= "3.7"

src/python/requirements_setuptools.txt

@@ -1,3 +1,3 @@
-pip==20.3.4
-setuptools==41.4.0
-wheel==0.33.4
+pip==21.3.1
+setuptools==68.0.0
+wheel==0.41.3

src/python/requirements_test.txt

@@ -1,14 +1,8 @@
 pexpect==4.6
 mock==2.0.0
-pytest==7.4.2; python_version >= "3"
-pytest==4.6.9; python_version < "3"
-pytest-xdist==3.3.1; python_version >= "3"
-pytest-xdist==1.31.0; python_version < "3"
-pytest-timeout==2.1.0; python_version >= "3"
-pytest-timeout==1.3.4; python_version < "3"
-more-itertools<=6.0.0; python_version <= "3.5"
+pytest==7.4.2
+pytest-xdist==3.3.1
+pytest-timeout==2.1.0
 parameterized==0.8.1
 pandas==1.3.5; python_version>='3.7'
 pandas>=0.23.3,<=0.25.3; python_version>='3.5.3' and python_version<'3.7'
-pandas>=0.23.3,< 0.25.0; python_version<'3.5.3'
-

src/python/scripts/dx-clone-asset

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2.7
+#!/usr/bin/env python3
 
 from __future__ import print_function
 import argparse