diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..a634b5ed --- /dev/null +++ b/.gitmodules @@ -0,0 +1,9 @@ +[submodule "ThirdParty/pybind11"] + path = ThirdParty/pybind11 + url = https://github.com/pybind/pybind11.git +[submodule "ThirdParty/tbb"] + path = ThirdParty/tbb + url = https://github.com/wjakob/tbb.git +[submodule "ThirdParty/pagmo2"] + path = ThirdParty/pagmo2 + url = https://github.com/apc-llc/pagmo2.git diff --git a/CMakeLists.txt b/CMakeLists.txt index d0655aad..188cc098 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -150,14 +150,29 @@ endif() # Find the dependencies. +# TBB. +# TODO This should actually be a submodule of pagmo. +# The only reason to have it here is to avoid forking pagmo as well. +set(TBB_BUILD_TESTS OFF CACHE BOOL "" FORCE) +set(TBB_BUILD_SHARED ON CACHE BOOL "" FORCE) +set(TBB_BUILD_STATIC OFF CACHE BOOL "" FORCE) +add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/ThirdParty/tbb EXCLUDE_FROM_ALL) +set(TBB_FOUND TRUE) +set(TBB_LIBRARIES tbb tbb_interface) +set(TBB_INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/ThirdParty/tbb/include) +add_library(TBB::tbb ALIAS tbb) +add_library(TBB::tbb_interface ALIAS tbb_interface) +install(TARGETS ${TBB_LIBRARIES} EXPORT pagmo_export + LIBRARY DESTINATION ${CMAKE_CURRENT_BINARY_DIR} + ARCHIVE DESTINATION ${CMAKE_CURRENT_BINARY_DIR} + RUNTIME DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) + # pagmo. # NOTE: put the minimum version in a variable # so that we can re-use it below. -set (_PYGMO_MIN_PAGMO_VERSION 2.13.0) -find_package(pagmo REQUIRED) -if(${pagmo_VERSION} VERSION_LESS ${_PYGMO_MIN_PAGMO_VERSION}) - message(FATAL_ERROR "The minimum pagmo version required by pygmo is ${_PYGMO_MIN_PAGMO_VERSION}, but version ${pagmo_VERSION} was found instead.") -endif() +set(_PYGMO_MIN_PAGMO_VERSION 2.13.0) +option(PAGMO_BUILD_STATIC_LIBRARY "" TRUE) +add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/ThirdParty/pagmo2 EXCLUDE_FROM_ALL) # python. include(YACMAPythonSetup) @@ -171,7 +186,7 @@ if(${PYTHON_VERSION_MAJOR} LESS 3 OR (${PYTHON_VERSION_MAJOR} EQUAL 3 AND ${PYTH endif() # pybind11. -find_package(pybind11 REQUIRED) +add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/ThirdParty/pybind11) # Configure the sphinx config file. configure_file("${CMAKE_CURRENT_SOURCE_DIR}/doc/conf.py.in" "${CMAKE_CURRENT_SOURCE_DIR}/doc/conf.py" @ONLY) diff --git a/README.md b/README.md index 417397fb..c1dd80cf 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,4 @@ -pygmo -===== +# pygmo [![Build Status](https://img.shields.io/circleci/project/github/esa/pygmo2/master.svg?style=for-the-badge)](https://circleci.com/gh/esa/pygmo2) [![Build Status](https://img.shields.io/travis/esa/pygmo2/master.svg?logo=travis&style=for-the-badge)](https://travis-ci.org/esa/pygmo2) @@ -22,11 +21,23 @@ at [this link](https://doi.org/10.5281/zenodo.1045336). The full documentation can be found [here](https://esa.github.io/pygmo2/). -Upgrading from pygmo 1.x.x -========================== +## Building and installing from source + +``` +pip3 install git+https://github.com/esa/pygmo2.git +``` + +## Quick testing + +``` +python3 -c "import pygmo; pygmo.test.run_test_suite(1); pygmo.mp_island.shutdown_pool(); pygmo.mp_bfe.shutdown_pool()" +``` + +## Upgrading from pygmo 1.x.x If you were using the old pygmo, have a look here on some technical data on what and why a completely new API and code was developed: https://github.com/esa/pagmo2/wiki/From-1.x-to-2.x You will find many tutorials in the documentation, we suggest to skim through them to realize the differences. The new pygmo (version 2) should be considered (and is) as an entirely different code. + diff --git a/ThirdParty/pagmo2 b/ThirdParty/pagmo2 new file mode 160000 index 00000000..7c502abc --- /dev/null +++ b/ThirdParty/pagmo2 @@ -0,0 +1 @@ +Subproject commit 7c502abc707297b706d1805d85a2966bde32ce61 diff --git a/ThirdParty/pybind11 b/ThirdParty/pybind11 new file mode 160000 index 00000000..227170dc --- /dev/null +++ b/ThirdParty/pybind11 @@ -0,0 +1 @@ +Subproject commit 227170dc2f209795974fdeea2df96c699de981a0 diff --git a/ThirdParty/tbb b/ThirdParty/tbb new file mode 160000 index 00000000..806df70e --- /dev/null +++ b/ThirdParty/tbb @@ -0,0 +1 @@ +Subproject commit 806df70ee69fc7b332fcf90a48651f6dbf0663ba diff --git a/pygmo/CMakeLists.txt b/pygmo/CMakeLists.txt index f00756a9..47df10e8 100644 --- a/pygmo/CMakeLists.txt +++ b/pygmo/CMakeLists.txt @@ -52,7 +52,10 @@ foreach(PYGMO_PYTHON_FILE ${PYGMO_PYTHON_FILES}) endforeach() # Core module. -YACMA_PYTHON_MODULE(core +# Note that LTO causes link time errors with GCC. +# To avoid this, we disable LTO for pybind using NO_EXTRAS. +# For more details, see e.g. https://github.com/BlueBrain/nmodl/issues/266 +PYBIND11_ADD_MODULE(core MODULE NO_EXTRAS core.cpp common_utils.cpp common_base.cpp @@ -77,9 +80,9 @@ YACMA_PYTHON_MODULE(core handle_thread_py_exception.cpp ) -target_link_libraries(core PRIVATE Pagmo::pagmo Boost::boost Boost::serialization) -target_include_directories(core SYSTEM PRIVATE "${pybind11_INCLUDE_DIR}") -target_compile_definitions(core PRIVATE "${pybind11_DEFINITIONS}") +target_include_directories(core SYSTEM PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/../ThirdParty/pagmo2/include") +target_include_directories(core SYSTEM PRIVATE "${CMAKE_CURRENT_BINARY_DIR}/../ThirdParty/pagmo2/include") +target_link_libraries(core PRIVATE pagmo Boost::boost Boost::serialization pybind11::pybind11) target_compile_options(core PRIVATE "$<$:${PYGMO_CXX_FLAGS_DEBUG}>" "$<$:${PYGMO_CXX_FLAGS_RELEASE}>" @@ -115,7 +118,7 @@ if(${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.9.0") endif() # Setup the installation path. -set(PYGMO_INSTALL_PATH "${YACMA_PYTHON_MODULES_INSTALL_PATH}/pygmo") +set(PYGMO_INSTALL_PATH .) # Add submodules directories add_subdirectory(plotting) diff --git a/setup.py b/setup.py new file mode 100644 index 00000000..ab7805af --- /dev/null +++ b/setup.py @@ -0,0 +1,270 @@ +"""Setup for the ESA software components + Copyright (C) 2015 Ehsan Azar (dashesy@linux.com) + Copyright (C) 2019, 2020 Dmitry Mikushin (dmitry@kernelgen.org) + +This file basically just uses CMake to compile the given project. + +To build the package: + python3 setup.py build +To build and install: + python3 setup.py install +To package the wheel (after pip installing twine and wheel): + python3 setup.py bdist_wheel +To upload the binary wheel to PyPi + twine upload dist/*.whl +To upload the source distribution to PyPi + python3 setup.py sdist + twine upload /-*.tar.gz +To exclude certain options in the cmake config use --no: + for example: + --no USE_AVX_INSTRUCTIONS: will set -DUSE_AVX_INSTRUCTIONS=no +Additional options: + --compiler-flags: pass flags onto the compiler, e.g. --compiler-flags "-Os -Wall" passes -Os -Wall onto GCC. + -G: Set the CMake generator. E.g. -G "Visual Studio 14 2015" + --clean: delete any previous build folders and rebuild. You should do this if you change any build options + by setting --compiler-flags or --no since the last time you ran a build. This will + ensure the changes take effect. + --set: set arbitrary cmake options e.g. --set CUDA_HOST_COMPILER=/usr/bin/gcc-6.4.0 + passes -DCUDA_HOST_COMPILER=/usr/bin/gcc-6.4.0 to CMake. +""" + +project_name = "pygmo" + +import os +import re +import sys +import shutil +import platform +import subprocess +import multiprocessing +from distutils import log, dir_util +from math import ceil,floor + +import setuptools +from setuptools import setup, Extension +from setuptools.command.build_ext import build_ext +from setuptools.command.install_lib import install_lib +from distutils.version import LooseVersion + +from setuptools.command.test import test as TestCommand + +def get_extra_cmake_options(): + """read --clean, --no, --set, --compiler-flags, and -G options from the command line and add them as cmake switches. + """ + _cmake_extra_options = [] + _clean_build_folder = False + + opt_key = None + + has_generator = False + + argv = [arg for arg in sys.argv] # take a copy + # parse command line options and consume those we care about + for arg in argv: + if opt_key == 'compiler-flags': + _cmake_extra_options.append('-DCMAKE_CXX_FLAGS={arg}'.format(arg=arg.strip())) + elif opt_key == 'G': + has_generator = True + _cmake_extra_options += ['-G', arg.strip()] + elif opt_key == 'no': + _cmake_extra_options.append('-D{arg}=no'.format(arg=arg.strip())) + elif opt_key == 'set': + _cmake_extra_options.append('-D{arg}'.format(arg=arg.strip())) + + if opt_key: + sys.argv.remove(arg) + opt_key = None + continue + + if arg == '--clean': + _clean_build_folder = True + sys.argv.remove(arg) + continue + + if arg == '--yes': + print("The --yes options to setup.py don't do anything since all these options ") + print("are on by default. So --yes has been removed. Do not give it to setup.py.") + sys.exit(1) + if arg in ['--no', '--set', '--compiler-flags']: + opt_key = arg[2:].lower() + sys.argv.remove(arg) + continue + if arg in ['-G']: + opt_key = arg[1:] + sys.argv.remove(arg) + continue + + # If no explicit CMake Generator specification, + # prefer Ninja on Windows + if (not has_generator) and (platform.system() == "Windows") and shutil.which("ninja"): + _cmake_extra_options += ['-G', "Ninja"] + + return _cmake_extra_options, _clean_build_folder + +cmake_extra_options,clean_build_folder = get_extra_cmake_options() + + +class CMakeExtension(Extension): + def __init__(self, name, sourcedir=''): + Extension.__init__(self, name, sources=[]) + self.sourcedir = os.path.abspath(sourcedir) + +def rmtree(name): + """remove a directory and its subdirectories. + """ + def remove_read_only(func, path, exc): + excvalue = exc[1] + if func in (os.rmdir, os.remove) and excvalue.errno == errno.EACCES: + os.chmod(path, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + func(path) + else: + raise + + if os.path.exists(name): + log.info('Removing old directory {}'.format(name)) + shutil.rmtree(name, ignore_errors=False, onerror=remove_read_only) + + +class CMakeBuild(build_ext): + + def get_cmake_version(self): + try: + out = subprocess.check_output(['cmake', '--version']) + except OSError: + raise RuntimeError("\n*******************************************************************\n" + + " CMake must be installed to build the following extensions: " + + ", ".join(e.name for e in self.extensions) + + "\n*******************************************************************\n") + return re.search(r'version\s*([\d.]+)', out.decode()).group(1) + + def run(self): + cmake_version = self.get_cmake_version() + if platform.system() == "Windows": + if LooseVersion(cmake_version) < '3.1.0': + raise RuntimeError("CMake >= 3.1.0 is required on Windows") + + for ext in self.extensions: + self.build_extension(ext) + + def build_extension(self, ext): + cmake_args = [] + + cmake_args += cmake_extra_options + + cfg = 'Debug' if self.debug else 'Release' + build_args = ['--config', cfg] + + if platform.system() != "Windows": + cmake_args += ['-DCMAKE_BUILD_TYPE=' + cfg] + # Do a parallel build + build_args += ['--', '-j'+str(num_available_cpu_cores(2))] + + build_folder = os.path.abspath(self.build_temp) + + if clean_build_folder: + rmtree(build_folder) + if not os.path.exists(build_folder): + os.makedirs(build_folder) + + cmake_setup = ['cmake', ext.sourcedir] + cmake_args + cmake_build = ['cmake', '--build', '.'] + build_args + + print("Building extension for Python {}".format(sys.version.split('\n',1)[0])) + print("Invoking CMake setup: '{}'".format(' '.join(cmake_setup))) + sys.stdout.flush() + subprocess.check_call(cmake_setup, cwd=build_folder) + print("Invoking CMake build: '{}'".format(' '.join(cmake_build))) + sys.stdout.flush() + subprocess.check_call(cmake_build, cwd=build_folder) + +class CMakeInstall(install_lib): + + def install(self): + + build_cmd = self.get_finalized_command('build_ext') + build_files = build_cmd.get_outputs() + build_temp = getattr(build_cmd, 'build_temp') + + install_dir = os.path.join(os.path.abspath(self.install_dir), project_name) + + cmake_install_prefix = ['cmake', '-DCMAKE_INSTALL_PREFIX=' + install_dir, '-P', 'cmake_install.cmake' ] + + # Adjust install prefix as shown at LLVM and not widely known: + # https://llvm.org/docs/CMake.html#id6 + print("Adjusting CMake install prefix: '{}'".format(' '.join(cmake_install_prefix))) + sys.stdout.flush() + subprocess.check_call(cmake_install_prefix, cwd=build_temp) + +def num_available_cpu_cores(ram_per_build_process_in_gb): + if 'TRAVIS' in os.environ and os.environ['TRAVIS']=='true': + # When building on travis-ci, just use 2 cores since travis-ci limits + # you to that regardless of what the hardware might suggest. + return 2 + try: + mem_bytes = os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES') + mem_gib = mem_bytes/(1024.**3) + num_cores = multiprocessing.cpu_count() + # make sure we have enough ram for each build process. + mem_cores = int(floor(mem_gib/float(ram_per_build_process_in_gb)+0.5)); + # We are limited either by RAM or CPU cores. So pick the limiting amount + # and return that. + return max(min(num_cores, mem_cores), 1) + except ValueError: + return 2 # just assume 2 if we can't get the os to tell us the right answer. + +class PyTest(TestCommand): + user_options = [('pytest-args=', 'a', "Arguments to pass to pytest")] + + def initialize_options(self): + TestCommand.initialize_options(self) + self.pytest_args = '--ignore docs --ignore ' + os.path.join('ThirdParty', project_name) + + def run_tests(self): + import shlex + #import here, cause outside the eggs aren't loaded + import pytest + errno = pytest.main(shlex.split(self.pytest_args)) + sys.exit(errno) + +def read_version_from_cmakelists(cmake_file): + """Read version information + """ + major = re.findall("set\(CPACK_PACKAGE_VERSION_MAJOR.*\"(.*)\"", open(cmake_file).read())[0] + minor = re.findall("set\(CPACK_PACKAGE_VERSION_MINOR.*\"(.*)\"", open(cmake_file).read())[0] + patch = re.findall("set\(CPACK_PACKAGE_VERSION_PATCH.*\"(.*)\"", open(cmake_file).read())[0] + return major + '.' + minor + '.' + patch + +def read_entire_file(fname): + """Read text out of a file relative to setup.py. + """ + return open(os.path.join(fname)).read() + +with open("README.md", "r") as fh: + long_description = fh.read() + +setup( + name=project_name, + #version=read_version_from_cmakelists('CMakeLists.txt'), + version="2.15.0", + author="Dario Izzo", + author_email="dario.izzo@esa.int", + description="A platform to perform parallel computations of optimisation tasks (global and local) via the asynchronous generalized island model.", + long_description=long_description, + long_description_content_type="text/markdown", + url="https://github.com/esa/pygmo2", + packages=setuptools.find_packages(), + classifiers=[ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + ], + python_requires='>=3.6', + license='MPL 2.0', + ext_modules=[CMakeExtension(os.path.join('ThirdParty', project_name))], #'tools/python')], + cmdclass=dict(build_ext=CMakeBuild, install_lib=CMakeInstall), #, test=PyTest), + zip_safe=False, + tests_require=[], + # removed 'cmake' because the pip cmake package is busted, maybe someday it will be usable. + install_requires=['numpy', 'cloudpickle', 'networkx', 'dill', 'numba', 'numba', 'ipyparallel'], + keywords=['optimization', 'parallel computations', 'island model'], +)