diff --git a/julia/core.py b/julia/core.py index 62f03195..2f946910 100644 --- a/julia/core.py +++ b/julia/core.py @@ -140,7 +140,7 @@ def __setattr__(self, name, value): juliapath = remove_prefix(self.__name__, "julia.") setter = ''' PyCall.pyfunctionret( - (x) -> eval({}, :({} = $x)), + (x) -> Base.eval({}, :({} = $x)), Any, PyCall.PyAny) '''.format(juliapath, jl_name(name)) @@ -399,7 +399,14 @@ def is_compatible_exe(jlinfo, _debug=lambda *_: None): _separate_cache_error_statically_linked = """ Your Python interpreter "{sys.executable}" is statically linked to libpython. Currently, PyJulia does not support -such Python interpreter. For available workarounds, see: +such Python interpreter. One easy workaround is to run your Python +script with `python-jl` command bundled in PyJulia. You can simply do: + + python-jl PATH/TO/YOUR/SCRIPT.py + +See `python-jl --help` for more information. + +For other available workarounds, see: https://github.com/JuliaPy/pyjulia/issues/185 """ @@ -562,7 +569,7 @@ def __init__(self, init_julia=True, jl_init_path=None, runtime=None, else: # we're assuming here we're fully inside a running Julia process, # so we're fishing for symbols in our own process table - self.api = ctypes.PyDLL('') + self.api = ctypes.PyDLL(None) # Store the running interpreter reference so we can start using it via self.call self.api.jl_.argtypes = [void_p] @@ -679,6 +686,10 @@ def __init__(self, init_julia=True, jl_init_path=None, runtime=None, self.sprint = self.eval('sprint') self.showerror = self.eval('showerror') + if self.eval('VERSION >= v"0.7-"'): + self.eval("@eval Main import Base.MainInclude: eval, include") + # https://github.com/JuliaLang/julia/issues/28825 + def _debug(self, *msg): """ Print some debugging stuff, if enabled diff --git a/julia/pseudo_python_cli.py b/julia/pseudo_python_cli.py new file mode 100644 index 00000000..1258d9eb --- /dev/null +++ b/julia/pseudo_python_cli.py @@ -0,0 +1,318 @@ +""" +Pseudo Python command line interface. + +It tries to mimic a subset of Python CLI: +https://docs.python.org/3/using/cmdline.html +""" + +from __future__ import print_function, absolute_import + +from collections import namedtuple +import code +import copy +import runpy +import sys +import traceback + +try: + from types import SimpleNamespace +except ImportError: + from argparse import Namespace as SimpleNamespace + + +ARGUMENT_HELP = """ +positional arguments: + script path to file (default: None) + args arguments passed to program in sys.argv[1:] + +optional arguments: + -h, --help show this help message and exit + -i inspect interactively after running script. + --version, -V Print the Python version number and exit. + -VV is not supported. + -c COMMAND Execute the Python code in COMMAND. + -m MODULE Search sys.path for the named MODULE and execute its contents + as the __main__ module. +""" + + +def python(module, command, script, args, interactive): + if command: + sys.argv[0] = "-c" + + assert sys.argv + sys.argv[1:] = args + if script: + sys.argv[0] = script + + banner = "" + try: + if command: + scope = {} + exec(command, scope) + elif module: + scope = runpy.run_module( + module, + run_name="__main__", + alter_sys=True) + elif script == "-": + source = sys.stdin.read() + exec(compile(source, "", "exec"), scope) + elif script: + scope = runpy.run_path( + script, + run_name="__main__") + else: + interactive = True + scope = None + banner = None # show banner + except Exception: + if not interactive: + raise + traceback.print_exc() + + if interactive: + code.interact(banner=banner, local=scope) + +ArgDest = namedtuple("ArgDest", "dest names default") +Optional = namedtuple("Optional", "name is_long argdest nargs action terminal") +Result = namedtuple("Result", "option values") + + +class PyArgumentParser(object): + + """ + `ArgumentParser`-like parser with "terminal option" support. + + Major differences: + + * Formatted help has to be provided to `description`. + * Many options for `.add_argument` are not supported. + * Especially, there is no positional argument support: all positional + arguments go into `ns.args`. + * `.add_argument` can take boolean option `terminal` (default: `False`) + to stop parsing after consuming the given option. + """ + + def __init__(self, prog=None, usage="%(prog)s [options] [args]", + description=""): + self.prog = sys.argv[0] if prog is None else prog + self.usage = usage + self.description = description + + self._dests = ["args"] + self._argdests = [ArgDest("args", (), [])] + self._options = [] + + self.add_argument("--help", "-h", "-?", action="store_true") + + def format_usage(self): + return "usage: " + self.usage % {"prog": self.prog} + + # Once we drop Python 2, we can do: + """ + def add_argument(self, name, *alt, dest=None, nargs=None, action=None, + default=None, terminal=False): + """ + + def add_argument(self, name, *alt, **kwargs): + return self._add_argument_impl(name, alt, **kwargs) + + def _add_argument_impl(self, name, alt, dest=None, nargs=None, action=None, + default=None, terminal=False): + if dest is None: + if name.startswith("--"): + dest = name[2:] + elif not name.startswith("-"): + dest = name + else: + raise ValueError(name) + + if not name.startswith("-"): + raise NotImplementedError( + "Positional arguments are not supported." + " All positional arguments will be stored in `ns.args`.") + if terminal and action is not None: + raise NotImplementedError( + "Terminal option is assumed to have argument." + " Non-`None` action={} is not supported".format()) + + if nargs is not None and action is not None: + raise TypeError("`nargs` and `action` are mutually exclusive") + if action == "store_true": + nargs = 0 + assert nargs is None or isinstance(nargs, int) + assert action in (None, "store_true") + + assert dest not in self._dests + self._dests.append(dest) + + argdest = ArgDest( + dest=dest, + names=(name,) + alt, + default=default, + ) + self._argdests.append(argdest) + + for arg in (name,) + alt: + self._options.append(Optional( + name=arg, + is_long=arg.startswith("--"), + argdest=argdest, + nargs=nargs, + action=action, + terminal=terminal, + )) + + def parse_args(self, args): + ns = SimpleNamespace(**{ + argdest.dest: copy.copy(argdest.default) + for argdest in self._argdests + }) + args_iter = iter(args) + self._parse_until_terminal(ns, args_iter) + ns.args.extend(args_iter) + + if ns.help: + self.print_help() + self.exit() + del ns.help + + return ns + + def _parse_until_terminal(self, ns, args_iter): + seen = set() + for a in args_iter: + + results = self._find_matches(a) + if not results: + ns.args.append(a) + break + + for i, res in enumerate(results): + dest = res.option.argdest.dest + if dest in seen: + self._usage_and_error( + "{} provided more than twice" + .format(", ".join(res.option.argdest.names))) + seen.add(dest) + + num_args = res.option.nargs + if num_args is None: + num_args = 1 + while len(res.values) < num_args: + try: + res.values.append(next(args_iter)) + except StopIteration: + self.error(self.format_usage()) + + if res.option.action == "store_true": + setattr(ns, dest, True) + else: + value = res.values + if res.option.nargs is None: + value, = value + setattr(ns, dest, value) + + if res.option.terminal: + assert i == len(results) - 1 + return + + def _find_matches(self, arg): + """ + Return a list of `.Result`. + + If value presents in `arg` (i.e., ``--long-option=value``), it + becomes the element of `.Result.values` (a list). Otherwise, + this list has to be filled by the caller (`_parse_until_terminal`). + """ + for opt in self._options: + if arg == opt.name: + return [Result(opt, [])] + elif arg.startswith(opt.name): + # i.e., len(arg) > len(opt.name): + if opt.is_long and arg[len(opt.name)] == "=": + return [Result(opt, [arg[len(opt.name) + 1:]])] + elif not opt.is_long: + if opt.nargs != 0: + return [Result(opt, [arg[len(opt.name):]])] + else: + results = [Result(opt, [])] + rest = "-" + arg[len(opt.name):] + results.extend(self._find_matches(rest)) + return results + # arg="-ih" -> rest="-h" + return [] + + def print_usage(self, file=None): + print(self.format_usage(), file=file or sys.stdout) + + def print_help(self): + self.print_usage() + print() + print(self.description) + + def exit(self, status=0): + sys.exit(status) + + def _usage_and_error(self, message): + self.print_usage(sys.stderr) + print(file=sys.stderr) + self.error(message) + + def error(self, message): + print(message, file=sys.stderr) + self.exit(2) + + +def make_parser(description=__doc__ + ARGUMENT_HELP): + parser = PyArgumentParser( + prog=None if sys.argv[0] else "python", + usage="%(prog)s [option] ... [-c cmd | -m mod | script | -] [args]", + description=description) + + parser.add_argument("-i", dest="interactive", action="store_true") + parser.add_argument("--version", "-V", action="store_true") + parser.add_argument("-c", dest="command", terminal=True) + parser.add_argument("-m", dest="module", terminal=True) + + return parser + + +def parse_args_with(parser, args): + ns = parser.parse_args(args) + + if ns.command and ns.module: + parser.error("-c and -m are mutually exclusive") + if ns.version: + print("Python {0}.{1}.{2}".format(*sys.version_info)) + parser.exit() + del ns.version + + ns.script = None + if (not (ns.command or ns.module)) and ns.args: + ns.script = ns.args[0] + ns.args = ns.args[1:] + + return ns + + +def parse_args(args): + return parse_args_with(make_parser(), args) + + +def main(args=None): + if args is None: + args = sys.argv[1:] + try: + ns = parse_args(args) + python(**vars(ns)) + except SystemExit as err: + return err.code + except Exception: + traceback.print_exc() + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/julia/python_jl.py b/julia/python_jl.py new file mode 100644 index 00000000..d637eb26 --- /dev/null +++ b/julia/python_jl.py @@ -0,0 +1,123 @@ +""" +Python interpreter inside a Julia process. + +This command line interface mimics a basic subset of Python program so that +Python program involving calls to Julia functions can be run in a *Julia* +process. This avoids the known problem with pre-compilation cache in +Deiban-based distribution such as Ubuntu and Python executable installed by +Conda in Linux. + +In Windows and macOS, this CLI is not necessary because those platforms do +not have the pre-compilation issue mentioned above. In fact, this CLI is +known to not work on Windows at the moment. + +Although this script has -i option and it can do a basic REPL, contrl-c may +crash the whole process. Consider using IPython >= 7 which can be launched +by:: + + python-jl -m IPython + +.. NOTE:: + + For this command to work, Python environment with which PyCall.jl is + configured has to have PyJulia installed. +""" + +from __future__ import print_function, absolute_import + +import os +import sys + +from .pseudo_python_cli import make_parser, parse_args_with, ARGUMENT_HELP + +PYJL_ARGUMENT_HELP = ARGUMENT_HELP + """ + --julia JULIA Julia interpreter to be used. (default: julia) +""" + +script_jl = """ +import PyCall + +# Initialize julia.Julia once so that subsequent calls of julia.Julia() +# uses pre-configured DLL. +PyCall.pyimport("julia")[:Julia](init_julia=false) + +let code = PyCall.pyimport("julia.pseudo_python_cli")[:main](ARGS) + if code isa Integer + exit(code) + end +end +""" + + +def remove_julia_options(args): + """ + Remove options used in this Python process. + + >>> list(remove_julia_options(["a", "b", "c"])) + ['a', 'b', 'c'] + >>> list(remove_julia_options(["a", "--julia", "julia", "b", "c"])) + ['a', 'b', 'c'] + >>> list(remove_julia_options(["a", "b", "c", "--julia=julia"])) + ['a', 'b', 'c'] + """ + it = iter(args) + for a in it: + if a == "--julia": + try: + next(it) + except StopIteration: + return + continue + elif a.startswith("--julia="): + continue + yield a + + +def parse_pyjl_args(args): + """ + Return a pair of parsed result and "unused" arguments. + + Returns + ------- + ns : argparse.Namespace + Parsed result. Only `ns.julia` is relevant here. + unused_args : list + Arguments to be parsed (again) by `.pseudo_python_cli.main`. + + Examples + -------- + >>> ns, unused_args = parse_pyjl_args([]) + >>> ns.julia + 'julia' + >>> unused_args + [] + >>> ns, unused_args = parse_pyjl_args( + ... ["--julia", "julia-dev", "-i", "-c", "import julia"]) + >>> ns.julia + 'julia-dev' + >>> unused_args + ['-i', '-c', 'import julia'] + """ + # Mix the options we need in this Python process with the Python + # arguments to be parsed in the "subprocess". This way, we get a + # parse error right now without initiating Julia interpreter and + # importing PyCall.jl etc. to get an extra speedup for the + # abnormal case (including -h/--help and -V/--version). + parser = make_parser(description=__doc__ + PYJL_ARGUMENT_HELP) + parser.add_argument("--julia", default="julia") + + ns = parse_args_with(parser, args) + unused_args = list(remove_julia_options(args)) + return ns, unused_args + + +def main(args=None): + if args is None: + args = sys.argv[1:] + ns, unused_args = parse_pyjl_args(args) + julia = ns.julia + os.execvp(julia, [julia, "-e", script_jl, "--"] + unused_args) + + +if __name__ == "__main__": + main() diff --git a/setup.py b/setup.py index 3cbc0db5..66ab2a92 100644 --- a/setup.py +++ b/setup.py @@ -45,5 +45,10 @@ ], url='http://julialang.org', packages=['julia'], - package_data={'julia': ['fake-julia/*']} - ) + package_data={'julia': ['fake-julia/*']}, + entry_points={ + "console_scripts": [ + "python-jl = julia.python_jl:main", + ], + }, + ) diff --git a/test/test_pseudo_python_cli.py b/test/test_pseudo_python_cli.py new file mode 100644 index 00000000..238f510e --- /dev/null +++ b/test/test_pseudo_python_cli.py @@ -0,0 +1,93 @@ +import shlex + +import pytest + +from julia.pseudo_python_cli import parse_args + + +def make_dict(**kwargs): + ns = parse_args([]) + return dict(vars(ns), **kwargs) + + +@pytest.mark.parametrize("args, desired", [ + ("-m json.tool -h", make_dict(module="json.tool", args=["-h"])), + ("-mjson.tool -h", make_dict(module="json.tool", args=["-h"])), + ("-imjson.tool -h", + make_dict(interactive=True, module="json.tool", args=["-h"])), + ("-m ipykernel install --user --name NAME --display-name DISPLAY_NAME", + make_dict(module="ipykernel", + args=shlex.split("install --user --name NAME" + " --display-name DISPLAY_NAME"))), + ("-m ipykernel_launcher -f FILE", + make_dict(module="ipykernel_launcher", + args=shlex.split("-f FILE"))), + ("-", make_dict(script="-")), + ("- a", make_dict(script="-", args=["a"])), + ("script", make_dict(script="script")), + ("script a", make_dict(script="script", args=["a"])), + ("script -m", make_dict(script="script", args=["-m"])), + ("script -c 1", make_dict(script="script", args=["-c", "1"])), + ("script -h 1", make_dict(script="script", args=["-h", "1"])), +]) +def test_valid_args(args, desired): + ns = parse_args(shlex.split(args)) + actual = vars(ns) + assert actual == desired + + +@pytest.mark.parametrize("args", [ + "-m", + "-c", + "-i -m", + "-h -m", + "-V -m", +]) +def test_invalid_args(args, capsys): + with pytest.raises(SystemExit) as exc_info: + parse_args(shlex.split(args)) + assert exc_info.value.code != 0 + + captured = capsys.readouterr() + assert "usage:" in captured.err + assert not captured.out + + +@pytest.mark.parametrize("args", [ + "-h", + "-i --help", + "-h -i", + "-hi", + "-ih", + "-Vh", + "-hV", + "-h -m json.tool", + "-h -mjson.tool", +]) +def test_help_option(args, capsys): + with pytest.raises(SystemExit) as exc_info: + parse_args(shlex.split(args)) + assert exc_info.value.code == 0 + + captured = capsys.readouterr() + assert "usage:" in captured.out + assert not captured.err + + +@pytest.mark.parametrize("args", [ + "-V", + "--version", + "-V -i", + "-Vi", + "-iV", + "-V script", + "-V script -h", +]) +def test_version_option(args, capsys): + with pytest.raises(SystemExit) as exc_info: + parse_args(shlex.split(args)) + assert exc_info.value.code == 0 + + captured = capsys.readouterr() + assert "Python " in captured.out + assert not captured.err diff --git a/test/test_python_jl.py b/test/test_python_jl.py new file mode 100644 index 00000000..373f2690 --- /dev/null +++ b/test/test_python_jl.py @@ -0,0 +1,82 @@ +from textwrap import dedent +import os +import shlex +import subprocess + +import pytest + +from julia.core import which +from julia.python_jl import parse_pyjl_args + +is_windows = os.name == "nt" + +PYJULIA_TEST_REBUILD = os.environ.get("PYJULIA_TEST_REBUILD", "no") == "yes" +JULIA = os.environ.get("JULIA_EXE") + + +@pytest.mark.parametrize("args", [ + "-h", + "-i --help", + "--julia false -h", + "--julia false -i --help", +]) +def test_help_option(args, capsys): + with pytest.raises(SystemExit) as exc_info: + parse_pyjl_args(shlex.split(args)) + assert exc_info.value.code == 0 + + captured = capsys.readouterr() + assert "usage:" in captured.out + + +quick_pass_cli_args = [ + "-h", + "-i --help", + "-V", + "--version -c 1/0", +] + + +@pytest.mark.parametrize("args", quick_pass_cli_args) +def test_cli_quick_pass(args): + subprocess.check_output( + ["python-jl"] + shlex.split(args), + ) + + +@pytest.mark.skipif( + not which("false"), + reason="false command not found") +@pytest.mark.parametrize("args", quick_pass_cli_args) +def test_cli_quick_pass_no_julia(args): + subprocess.check_output( + ["python-jl", "--julia", "false"] + shlex.split(args), + ) + + +@pytest.mark.skipif( + is_windows, + reason="python-jl is not supported in Windows") +@pytest.mark.skipif( + not PYJULIA_TEST_REBUILD, + reason="PYJULIA_TEST_REBUILD=yes is not set") +def test_cli_import(): + args = ["-c", dedent(""" + from julia import Base + Base.banner() + from julia import Main + Main.x = 1 + assert Main.x == 1 + """)] + if JULIA: + args = ["--julia", JULIA] + args + output = subprocess.check_output( + ["python-jl"] + args, + universal_newlines=True) + assert "julialang.org" in output + +# Embedded julia does not have usual the Main.eval and Main.include. +# Main.eval is Core.eval. Let's test that we are not relying on this +# special behavior. +# +# See also: https://github.com/JuliaLang/julia/issues/28825