From 6d74d92eebfc7cec120f41bbd7ab5337cc3998f2 Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Fri, 12 Apr 2019 17:34:21 +0300 Subject: [PATCH 01/76] Re-do version range parsing for --expect Let's pass a parser function to argparse directly so we don't have to manually try/except etc. (I want this because I intend to add a few additional arguments that take version ranges.) --- check_python_versions.py | 27 ++++++++++++++++----------- tests.py | 35 ++++++++++++++++++++--------------- 2 files changed, 36 insertions(+), 26 deletions(-) diff --git a/check_python_versions.py b/check_python_versions.py index 5c76cd2..cae78ec 100755 --- a/check_python_versions.py +++ b/check_python_versions.py @@ -417,7 +417,15 @@ def important(versions): } -def parse_expect(v): +def parse_version(v): + try: + major, minor = map(int, v.split('.', 1)) + except ValueError: + raise argparse.ArgumentTypeError(f'bad version: {v}') + return (major, minor) + + +def parse_version_list(v): versions = set() for part in v.split(','): @@ -426,11 +434,12 @@ def parse_expect(v): else: lo = hi = part - lo_major, lo_minor = map(int, lo.split('.', 1)) - hi_major, hi_minor = map(int, hi.split('.', 1)) + lo_major, lo_minor = parse_version(lo) + hi_major, hi_minor = parse_version(hi) if lo_major != hi_major: - raise ValueError(f'bad range: {part} ({lo_major} != {hi_major})') + raise argparse.ArgumentTypeError( + f'bad range: {part} ({lo_major} != {hi_major})') for v in range(lo_minor, hi_minor + 1): versions.add(f'{lo_major}.{v}') @@ -496,8 +505,9 @@ def main(): parser.add_argument('--version', action='version', version="%(prog)s version " + __version__) parser.add_argument('--expect', metavar='VERSIONS', + type=parse_version_list, help='expect these versions to be supported, e.g.' - ' --expect 2.7,3.4-3.7') + ' --expect 2.7,3.5-3.7') parser.add_argument('--skip-non-packages', action='store_true', help='skip arguments that are not Python packages' ' without warning about them') @@ -506,11 +516,6 @@ def main(): ' and other files is located') args = parser.parse_args() - try: - expect = args.expect and parse_expect(args.expect) - except ValueError: - parser.error(f"bad value for --expect: {args.expect}") - where = args.where or ['.'] if args.skip_non_packages: where = [path for path in where if is_package(path)] @@ -522,7 +527,7 @@ def main(): if n: print("\n") print(f"{path}:\n") - if not check(path, expect=expect): + if not check(path, expect=args.expect): mismatches.append(path) if mismatches: diff --git a/tests.py b/tests.py index a43baed..d43d85d 100644 --- a/tests.py +++ b/tests.py @@ -1,3 +1,4 @@ +import argparse import ast import os import sys @@ -475,28 +476,31 @@ def test_important(monkeypatch): }) == {'2.7', '3.4'} -def test_parse_expect(): - assert cpv.parse_expect('2.7,3.4-3.6') == ['2.7', '3.4', '3.5', '3.6'] +def test_parse_version_list(): + assert cpv.parse_version_list( + '2.7,3.4-3.6' + ) == ['2.7', '3.4', '3.5', '3.6'] -def test_parse_expect_bad_range(): - with pytest.raises(ValueError, match=r'bad range: 2\.7-3\.4 \(2 != 3\)'): - cpv.parse_expect('2.7-3.4') +def test_parse_version_list_bad_range(): + with pytest.raises(argparse.ArgumentTypeError, + match=r'bad range: 2\.7-3\.4 \(2 != 3\)'): + cpv.parse_version_list('2.7-3.4') -def test_parse_expect_bad_number(): - with pytest.raises(ValueError): - cpv.parse_expect('2.x') +def test_parse_version_list_bad_number(): + with pytest.raises(argparse.ArgumentTypeError): + cpv.parse_version_list('2.x') -def test_parse_expect_too_few(): - with pytest.raises(ValueError): - cpv.parse_expect('2') +def test_parse_version_list_too_few(): + with pytest.raises(argparse.ArgumentTypeError): + cpv.parse_version_list('2') -def test_parse_expect_too_many_dots(): - with pytest.raises(ValueError): - cpv.parse_expect('2.7.1') +def test_parse_version_list_too_many_dots(): + with pytest.raises(argparse.ArgumentTypeError): + cpv.parse_version_list('2.7.1') def test_is_package(tmp_path): @@ -612,7 +616,8 @@ def test_main_expect_error_handling(monkeypatch, arg, capsys): ]) with pytest.raises(SystemExit): cpv.main() - assert f'bad value for --expect: {arg}' in capsys.readouterr().err + # the error is either 'bad version: ...' or 'bad range: ...' + assert f'--expect: bad' in capsys.readouterr().err def test_main_here(monkeypatch, capsys): From 825c9392222f6caec5ac40bea8c7011160db3b63 Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Fri, 12 Apr 2019 17:37:43 +0300 Subject: [PATCH 02/76] Refactoring: split check() into two functions --- check_python_versions.py | 17 +++++++++++++---- tests.py | 38 +++++++++++++++++++++++++++++++------- 2 files changed, 44 insertions(+), 11 deletions(-) diff --git a/check_python_versions.py b/check_python_versions.py index cae78ec..02b8289 100755 --- a/check_python_versions.py +++ b/check_python_versions.py @@ -452,16 +452,21 @@ def is_package(where='.'): return os.path.exists(setup_py) -def check(where='.', *, print=print, expect=None): +def check_package(where='.', *, print=print): if not os.path.isdir(where): print("not a directory") - return None + return False setup_py = os.path.join(where, 'setup.py') if not os.path.exists(setup_py): print("no setup.py -- not a Python package?") - return None + return False + + return True + + +def check_versions(where='.', *, print=print, expect=None): sources = [ ('setup.py', get_supported_python_versions, None), @@ -527,8 +532,12 @@ def main(): if n: print("\n") print(f"{path}:\n") - if not check(path, expect=args.expect): + if not check_package(path): + mismatches.append(path) + continue + if not check_versions(path, expect=args.expect): mismatches.append(path) + continue if mismatches: if multiple: diff --git a/tests.py b/tests.py index d43d85d..df8abf5 100644 --- a/tests.py +++ b/tests.py @@ -513,15 +513,26 @@ def test_is_package_no_setup_py(tmp_path): def test_check_not_a_directory(tmp_path, capsys): - assert cpv.check(tmp_path / "xyzzy") is None + assert not cpv.check_package(tmp_path / "xyzzy") assert capsys.readouterr().out == 'not a directory\n' def test_check_not_a_package(tmp_path, capsys): - assert cpv.check(tmp_path) is None + assert not cpv.check_package(tmp_path) assert capsys.readouterr().out == 'no setup.py -- not a Python package?\n' +def test_check_package(tmp_path): + setup_py = tmp_path / "setup.py" + setup_py.write_text(textwrap.dedent("""\ + from setuptools import setup + setup( + name='foo', + ) + """)) + assert cpv.check_package(tmp_path) is True + + def test_check_unknown(tmp_path, capsys): setup_py = tmp_path / "setup.py" setup_py.write_text(textwrap.dedent("""\ @@ -530,7 +541,7 @@ def test_check_unknown(tmp_path, capsys): name='foo', ) """)) - assert cpv.check(tmp_path) is True + assert cpv.check_versions(tmp_path) is True assert capsys.readouterr().out == textwrap.dedent("""\ setup.py says: (empty) """) @@ -548,7 +559,7 @@ def test_check_minimal(tmp_path, capsys): ], ) """)) - assert cpv.check(tmp_path) is True + assert cpv.check_versions(tmp_path) is True assert capsys.readouterr().out == textwrap.dedent("""\ setup.py says: 2.7, 3.6 """) @@ -571,7 +582,7 @@ def test_check_mismatch(tmp_path, capsys): [tox] envlist = py27 """)) - assert cpv.check(tmp_path) is False + assert cpv.check_versions(tmp_path) is False assert capsys.readouterr().out == textwrap.dedent("""\ setup.py says: 2.7, 3.6 tox.ini says: 2.7 @@ -590,7 +601,7 @@ def test_check_expectation(tmp_path, capsys): ], ) """)) - assert cpv.check(tmp_path, expect=['2.7', '3.6', '3.7']) is False + assert cpv.check_versions(tmp_path, expect=['2.7', '3.6', '3.7']) is False assert capsys.readouterr().out == textwrap.dedent("""\ setup.py says: 2.7, 3.6 expected: 2.7, 3.6, 3.7 @@ -657,7 +668,19 @@ def test_main_multiple(monkeypatch, capsys, tmp_path): 'check-python-versions', str(tmp_path / "a"), str(tmp_path / "b"), + '--expect', '3.6, 3.7' ]) + (tmp_path / "a").mkdir() + (tmp_path / "a" / "setup.py").write_text(textwrap.dedent("""\ + from setuptools import setup + setup( + name='foo', + classifiers=[ + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.6', + ], + ) + """)) with pytest.raises(SystemExit) as exc_info: cpv.main() assert ( @@ -665,7 +688,8 @@ def test_main_multiple(monkeypatch, capsys, tmp_path): ).replace(str(tmp_path) + os.path.sep, 'tmp/') == textwrap.dedent("""\ tmp/a: - not a directory + setup.py says: 2.7, 3.6 + expected: 3.6, 3.7 tmp/b: From 2321747052d8e72a86d84c71c88f9aceac61a7c3 Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Fri, 12 Apr 2019 18:57:45 +0300 Subject: [PATCH 03/76] Experimental update support check-python-versions {--add|--drop|--update} VERSIONS will attempt to update your setup.py to add/drop/set the supported version classifiers. The update script is interactive and shows you the diff and waits for confirmation before making any changes. --- check_python_versions.py | 153 ++++++++++++++++++++++++++++++++++++++- tests.py | 115 +++++++++++++++++++++++++++++ 2 files changed, 267 insertions(+), 1 deletion(-) diff --git a/check_python_versions.py b/check_python_versions.py index 02b8289..e112979 100755 --- a/check_python_versions.py +++ b/check_python_versions.py @@ -15,9 +15,11 @@ import argparse import ast import configparser +import difflib import logging import os import re +import string import subprocess import sys from functools import partial @@ -74,6 +76,11 @@ def get_supported_python_versions(repo_path='.'): return get_versions_from_classifiers(classifiers) +def is_version_classifier(s): + prefix = 'Programming Language :: Python :: ' + return s.startswith(prefix) and s[len(prefix):len(prefix) + 1].isdigit() + + def get_versions_from_classifiers(classifiers): # Based on # https://github.com/mgedmin/project-summary/blob/master/summary.py#L221-L234 @@ -83,7 +90,7 @@ def get_versions_from_classifiers(classifiers): versions = { s[len(prefix):].replace(' :: Only', '').rstrip() for s in classifiers - if s.startswith(prefix) and s[len(prefix):len(prefix) + 1].isdigit() + if is_version_classifier(s) } | { s[len(impl_prefix):].rstrip() for s in classifiers @@ -96,6 +103,33 @@ def get_versions_from_classifiers(classifiers): return sorted(versions) +def update_classifiers(classifiers, new_versions): + prefix = 'Programming Language :: Python :: ' + + for pos, s in enumerate(classifiers): + if is_version_classifier(s): + break + else: + pos = len(classifiers) + + classifiers = [ + s for s in classifiers if not is_version_classifier(s) + ] + new_classifiers = [ + f'{prefix}{version}' + for version in new_versions + ] + classifiers[pos:pos] = new_classifiers + return classifiers + + +def update_supported_python_versions(repo_path, new_versions): + setup_py = os.path.join(repo_path, 'setup.py') + classifiers = get_setup_py_keyword(setup_py, 'classifiers') + new_classifiers = update_classifiers(classifiers, new_versions) + update_setup_py_keyword(setup_py, 'classifiers', new_classifiers) + + def get_python_requires(setup_py='setup.py'): python_requires = get_setup_py_keyword(setup_py, 'python_requires') if python_requires is None: @@ -114,6 +148,85 @@ def get_setup_py_keyword(setup_py, keyword): return node and eval_ast_node(node, keyword) +def update_setup_py_keyword(setup_py, keyword, new_value): + with open(setup_py) as f: + lines = f.readlines() + new_lines = update_call_arg_in_source(lines, 'setup', keyword, new_value) + confirm_and_update_file(setup_py, lines, new_lines) + + +def confirm_and_update_file(filename, old_lines, new_lines): + print_diff(old_lines, new_lines, filename) + if confirm(f"Write changes to {filename}?"): + with open(filename + '.tmp', 'w') as f: + f.writelines(new_lines) + os.rename(filename + '.tmp', filename) + + +def print_diff(a, b, filename): + print(''.join(difflib.unified_diff( + a, b, + filename, filename, + "(original)", "(updated)", + ))) + + +def confirm(prompt): + while True: + answer = input(f'{prompt} [y/N] ').strip().lower() + if answer == 'y': + print() + return True + if answer == 'n' or not answer: + print() + return False + + +def to_literal(value, quote_style='"'): + safe_characters = string.ascii_letters + string.digits + ' .:,-=><()' + assert all( + c in safe_characters for c in value + ), f'{value!r} has unexpected characters' + assert quote_style not in value + return f'{quote_style}{value}{quote_style}' + + +def update_call_arg_in_source(source_lines, function, keyword, new_value): + lines = iter(enumerate(source_lines)) + for n, line in lines: + if line.startswith(f'{function}('): + break + else: + warn(f'Did not find {function}() call') + return + for n, line in lines: + if line.lstrip().startswith(f'{keyword}='): + break + else: + warn(f'Did not find {keyword}= argument in {function}() call') + return + + start = n + 1 + indent = 8 + quote_style = '"' + for n, line in lines: + stripped = line.lstrip() + if stripped.startswith('],'): + break + elif stripped: + indent = len(line) - len(stripped) + if stripped[0] in ('"', "'"): + quote_style = stripped[0] + else: + warn(f'Did not understand {keyword}= formatting in {function}() call') + end = n + + return source_lines[:start] + [ + f"{' ' * indent}{to_literal(value, quote_style)},\n" + for value in new_value + ] + source_lines[end:] + + def find_call_kwarg_in_ast(tree, funcname, keyword, filename='setup.py'): for node in ast.walk(tree): if (isinstance(node, ast.Call) @@ -447,6 +560,13 @@ def parse_version_list(v): return sorted(versions) +def update_version_list(versions, add=None, drop=None, update=None): + if update: + return sorted(update) + else: + return sorted(set(versions).union(add or ()).difference(drop or ())) + + def is_package(where='.'): setup_py = os.path.join(where, 'setup.py') return os.path.exists(setup_py) @@ -503,6 +623,18 @@ def check_versions(where='.', *, print=print, expect=None): ) +def update_versions(where='.', *, add=None, drop=None, update=None): + + versions = get_supported_python_versions(where) + if versions is None: + return + + new_versions = update_version_list( + versions, add=add, drop=drop, update=update) + if versions != new_versions: + update_supported_python_versions(where, new_versions) + + def main(): parser = argparse.ArgumentParser( description="verify that supported Python versions are the same" @@ -519,8 +651,24 @@ def main(): parser.add_argument('where', nargs='*', help='directory where a Python package with a setup.py' ' and other files is located') + group = parser.add_argument_group( + "updating supported version lists (EXPERIMENTAL)") + group.add_argument('--add', metavar='VERSIONS', type=parse_version_list, + help='add these versions to supported ones, e.g' + ' --add 3.8') + group.add_argument('--drop', metavar='VERSIONS', type=parse_version_list, + help='drop these versions from supported ones, e.g' + ' --drop 2.6,3.4') + group.add_argument('--update', metavar='VERSIONS', type=parse_version_list, + help='update the set of supported versions, e.g.' + ' --update 2.7,3.5-3.7') args = parser.parse_args() + if args.update and args.add: + parser.error("argument --add: not allowed with argument --update") + if args.update and args.drop: + parser.error("argument --drop: not allowed with argument --update") + where = args.where or ['.'] if args.skip_non_packages: where = [path for path in where if is_package(path)] @@ -535,6 +683,9 @@ def main(): if not check_package(path): mismatches.append(path) continue + if args.add or args.drop or args.update: + update_versions(path, add=args.add, drop=args.drop, + update=args.update) if not check_versions(path, expect=args.expect): mismatches.append(path) continue diff --git a/tests.py b/tests.py index df8abf5..8790562 100644 --- a/tests.py +++ b/tests.py @@ -81,6 +81,53 @@ def test_get_versions_from_classifiers_with_trailing_whitespace(): ]) == ['3.6'] +def test_update_classifiers(): + assert cpv.update_classifiers([ + 'Development Status :: 4 - Beta', + 'Environment :: Console', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: GNU General Public License (GPL)', + 'Operating System :: OS Independent', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: Implementation :: CPython', + 'Programming Language :: Python :: Implementation :: PyPy', + 'Typing :: Typed', + ], ['2.7', '3.7']) == [ + 'Development Status :: 4 - Beta', + 'Environment :: Console', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: GNU General Public License (GPL)', + 'Operating System :: OS Independent', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: Implementation :: CPython', + 'Programming Language :: Python :: Implementation :: PyPy', + 'Typing :: Typed', + ] + + +def test_update_classifiers_none_were_present(): + assert cpv.update_classifiers([ + 'Development Status :: 4 - Beta', + 'Environment :: Console', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: GNU General Public License (GPL)', + 'Operating System :: OS Independent', + ], ['2.7', '3.7']) == [ + 'Development Status :: 4 - Beta', + 'Environment :: Console', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: GNU General Public License (GPL)', + 'Operating System :: OS Independent', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.7', + ] + + def test_get_python_requires(tmp_path, monkeypatch): setup_py = tmp_path / "setup.py" setup_py.write_text(textwrap.dedent("""\ @@ -156,6 +203,54 @@ def test_eval_ast_node(code, expected): assert cpv.eval_ast_node(node, 'bar') == expected +def test_update_call_arg_in_source(): + source_lines = textwrap.dedent("""\ + setup( + foo=1, + bar=[ + "a", + "b", + "c", + ], + baz=2, + ) + """).splitlines(True) + result = cpv.update_call_arg_in_source(source_lines, "setup", "bar", + ["x", "y"]) + assert "".join(result) == textwrap.dedent("""\ + setup( + foo=1, + bar=[ + "x", + "y", + ], + baz=2, + ) + """) + + +def test_update_call_arg_in_source_preserves_indent_and_quote_style(): + source_lines = textwrap.dedent("""\ + setup(foo=1, + bar=[ + 'a', + 'b', + 'c', + ], + ) + """).splitlines(True) + result = cpv.update_call_arg_in_source(source_lines, "setup", "bar", + ["x", "y"]) + assert "".join(result) == textwrap.dedent("""\ + setup(foo=1, + bar=[ + 'x', + 'y', + ], + ) + """) + + @pytest.mark.parametrize('code', [ '[2 * 2]', '"".join([2 * 2])', @@ -503,6 +598,26 @@ def test_parse_version_list_too_many_dots(): cpv.parse_version_list('2.7.1') +def test_update_version_list(): + assert cpv.update_version_list(['2.7', '3.4']) == ['2.7', '3.4'] + assert cpv.update_version_list(['2.7', '3.4'], add=['3.4', '3.5']) == [ + '2.7', '3.4', '3.5', + ] + assert cpv.update_version_list(['2.7', '3.4'], drop=['3.4', '3.5']) == [ + '2.7', + ] + assert cpv.update_version_list(['2.7', '3.4'], add=['3.5'], + drop=['2.7']) == [ + '3.4', '3.5', + ] + assert cpv.update_version_list(['2.7', '3.4'], drop=['3.4', '3.5']) == [ + '2.7', + ] + assert cpv.update_version_list(['2.7', '3.4'], update=['3.4', '3.5']) == [ + '3.4', '3.5', + ] + + def test_is_package(tmp_path): (tmp_path / "setup.py").write_text("") assert cpv.is_package(tmp_path) From fe708f54bd0dc1523315168dc1aa32186e14d463 Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Fri, 12 Apr 2019 19:02:44 +0300 Subject: [PATCH 04/76] Preserve file mode please --- check_python_versions.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/check_python_versions.py b/check_python_versions.py index e112979..18bc4a1 100755 --- a/check_python_versions.py +++ b/check_python_versions.py @@ -19,6 +19,7 @@ import logging import os import re +import stat import string import subprocess import sys @@ -158,9 +159,12 @@ def update_setup_py_keyword(setup_py, keyword, new_value): def confirm_and_update_file(filename, old_lines, new_lines): print_diff(old_lines, new_lines, filename) if confirm(f"Write changes to {filename}?"): - with open(filename + '.tmp', 'w') as f: + mode = stat.S_IMODE(os.stat(filename).st_mode) + tempfile = filename + '.tmp' + with open(tempfile, 'w') as f: + os.fchmod(f.fileno(), mode) f.writelines(new_lines) - os.rename(filename + '.tmp', filename) + os.rename(tempfile, filename) def print_diff(a, b, filename): From 3eae4b1ca78c6f6661b4bd33c1e48ea2aaf306a5 Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Fri, 12 Apr 2019 19:07:32 +0300 Subject: [PATCH 05/76] Testing on ~/projects/* and fixes --- check_python_versions.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/check_python_versions.py b/check_python_versions.py index 18bc4a1..e25ccee 100755 --- a/check_python_versions.py +++ b/check_python_versions.py @@ -127,6 +127,8 @@ def update_classifiers(classifiers, new_versions): def update_supported_python_versions(repo_path, new_versions): setup_py = os.path.join(repo_path, 'setup.py') classifiers = get_setup_py_keyword(setup_py, 'classifiers') + if classifiers is None: + return new_classifiers = update_classifiers(classifiers, new_versions) update_setup_py_keyword(setup_py, 'classifiers', new_classifiers) @@ -187,7 +189,7 @@ def confirm(prompt): def to_literal(value, quote_style='"'): - safe_characters = string.ascii_letters + string.digits + ' .:,-=><()' + safe_characters = string.ascii_letters + string.digits + ' .:,-=><()/+' assert all( c in safe_characters for c in value ), f'{value!r} has unexpected characters' @@ -633,6 +635,7 @@ def update_versions(where='.', *, add=None, drop=None, update=None): if versions is None: return + versions = sorted(important(versions)) new_versions = update_version_list( versions, add=add, drop=drop, update=update) if versions != new_versions: From 5200c40362991a7774b3cb04a87905c23e01450f Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Fri, 12 Apr 2019 23:39:47 +0300 Subject: [PATCH 06/76] Make sure update_call_arg_in_source() can handle all classifiers --- CLASSIFIERS | 680 +++++++++++++++++++++++++++++++++++++++ check_python_versions.py | 7 +- tests.py | 22 ++ 3 files changed, 708 insertions(+), 1 deletion(-) create mode 100644 CLASSIFIERS diff --git a/CLASSIFIERS b/CLASSIFIERS new file mode 100644 index 0000000..70f7118 --- /dev/null +++ b/CLASSIFIERS @@ -0,0 +1,680 @@ +Development Status :: 1 - Planning +Development Status :: 2 - Pre-Alpha +Development Status :: 3 - Alpha +Development Status :: 4 - Beta +Development Status :: 5 - Production/Stable +Development Status :: 6 - Mature +Development Status :: 7 - Inactive +Environment :: Console +Environment :: Console :: Curses +Environment :: Console :: Framebuffer +Environment :: Console :: Newt +Environment :: Console :: svgalib +Environment :: Handhelds/PDA's +Environment :: MacOS X +Environment :: MacOS X :: Aqua +Environment :: MacOS X :: Carbon +Environment :: MacOS X :: Cocoa +Environment :: No Input/Output (Daemon) +Environment :: OpenStack +Environment :: Other Environment +Environment :: Plugins +Environment :: Web Environment +Environment :: Web Environment :: Buffet +Environment :: Web Environment :: Mozilla +Environment :: Web Environment :: ToscaWidgets +Environment :: Win32 (MS Windows) +Environment :: X11 Applications +Environment :: X11 Applications :: Gnome +Environment :: X11 Applications :: GTK +Environment :: X11 Applications :: KDE +Environment :: X11 Applications :: Qt +Framework :: AiiDA +Framework :: AsyncIO +Framework :: BEAT +Framework :: BFG +Framework :: Bob +Framework :: Bottle +Framework :: Buildout +Framework :: Buildout :: Extension +Framework :: Buildout :: Recipe +Framework :: CastleCMS +Framework :: CastleCMS :: Theme +Framework :: Chandler +Framework :: CherryPy +Framework :: CubicWeb +Framework :: Django +Framework :: Django :: 1.10 +Framework :: Django :: 1.11 +Framework :: Django :: 1.4 +Framework :: Django :: 1.5 +Framework :: Django :: 1.6 +Framework :: Django :: 1.7 +Framework :: Django :: 1.8 +Framework :: Django :: 1.9 +Framework :: Django :: 2.0 +Framework :: Django :: 2.1 +Framework :: Django :: 2.2 +Framework :: Flake8 +Framework :: Flask +Framework :: Hypothesis +Framework :: IDLE +Framework :: IPython +Framework :: Jupyter +Framework :: Lektor +Framework :: Masonite +Framework :: Nengo +Framework :: Odoo +Framework :: Opps +Framework :: Paste +Framework :: Pelican +Framework :: Pelican :: Plugins +Framework :: Pelican :: Themes +Framework :: Plone +Framework :: Plone :: 3.2 +Framework :: Plone :: 3.3 +Framework :: Plone :: 4.0 +Framework :: Plone :: 4.1 +Framework :: Plone :: 4.2 +Framework :: Plone :: 4.3 +Framework :: Plone :: 5.0 +Framework :: Plone :: 5.1 +Framework :: Plone :: 5.2 +Framework :: Plone :: 5.3 +Framework :: Plone :: Addon +Framework :: Plone :: Core +Framework :: Plone :: Theme +Framework :: Pylons +Framework :: Pyramid +Framework :: Pytest +Framework :: Review Board +Framework :: Robot Framework +Framework :: Robot Framework :: Library +Framework :: Robot Framework :: Tool +Framework :: Scrapy +Framework :: Setuptools Plugin +Framework :: Sphinx +Framework :: Sphinx :: Extension +Framework :: Sphinx :: Theme +Framework :: tox +Framework :: Trac +Framework :: Trio +Framework :: Tryton +Framework :: TurboGears +Framework :: TurboGears :: Applications +Framework :: TurboGears :: Widgets +Framework :: Twisted +Framework :: Wagtail +Framework :: Wagtail :: 1 +Framework :: Wagtail :: 2 +Framework :: ZODB +Framework :: Zope +Framework :: Zope2 +Framework :: Zope :: 2 +Framework :: Zope3 +Framework :: Zope :: 3 +Framework :: Zope :: 4 +Intended Audience :: Customer Service +Intended Audience :: Developers +Intended Audience :: Education +Intended Audience :: End Users/Desktop +Intended Audience :: Financial and Insurance Industry +Intended Audience :: Healthcare Industry +Intended Audience :: Information Technology +Intended Audience :: Legal Industry +Intended Audience :: Manufacturing +Intended Audience :: Other Audience +Intended Audience :: Religion +Intended Audience :: Science/Research +Intended Audience :: System Administrators +Intended Audience :: Telecommunications Industry +License :: Aladdin Free Public License (AFPL) +License :: CC0 1.0 Universal (CC0 1.0) Public Domain Dedication +License :: CeCILL-B Free Software License Agreement (CECILL-B) +License :: CeCILL-C Free Software License Agreement (CECILL-C) +License :: DFSG approved +License :: Eiffel Forum License (EFL) +License :: Free For Educational Use +License :: Free For Home Use +License :: Free for non-commercial use +License :: Freely Distributable +License :: Free To Use But Restricted +License :: Freeware +License :: GUST Font License 1.0 +License :: GUST Font License 2006-09-30 +License :: Netscape Public License (NPL) +License :: Nokia Open Source License (NOKOS) +License :: OSI Approved +License :: OSI Approved :: Academic Free License (AFL) +License :: OSI Approved :: Apache Software License +License :: OSI Approved :: Apple Public Source License +License :: OSI Approved :: Artistic License +License :: OSI Approved :: Attribution Assurance License +License :: OSI Approved :: Boost Software License 1.0 (BSL-1.0) +License :: OSI Approved :: BSD License +License :: OSI Approved :: CEA CNRS Inria Logiciel Libre License, version 2.1 (CeCILL-2.1) +License :: OSI Approved :: Common Development and Distribution License 1.0 (CDDL-1.0) +License :: OSI Approved :: Common Public License +License :: OSI Approved :: Eclipse Public License 1.0 (EPL-1.0) +License :: OSI Approved :: Eclipse Public License 2.0 (EPL-2.0) +License :: OSI Approved :: Eiffel Forum License +License :: OSI Approved :: European Union Public Licence 1.0 (EUPL 1.0) +License :: OSI Approved :: European Union Public Licence 1.1 (EUPL 1.1) +License :: OSI Approved :: European Union Public Licence 1.2 (EUPL 1.2) +License :: OSI Approved :: GNU Affero General Public License v3 +License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+) +License :: OSI Approved :: GNU Free Documentation License (FDL) +License :: OSI Approved :: GNU General Public License (GPL) +License :: OSI Approved :: GNU General Public License v2 (GPLv2) +License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+) +License :: OSI Approved :: GNU General Public License v3 (GPLv3) +License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+) +License :: OSI Approved :: GNU Lesser General Public License v2 (LGPLv2) +License :: OSI Approved :: GNU Lesser General Public License v2 or later (LGPLv2+) +License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3) +License :: OSI Approved :: GNU Lesser General Public License v3 or later (LGPLv3+) +License :: OSI Approved :: GNU Library or Lesser General Public License (LGPL) +License :: OSI Approved :: Historical Permission Notice and Disclaimer (HPND) +License :: OSI Approved :: IBM Public License +License :: OSI Approved :: Intel Open Source License +License :: OSI Approved :: ISC License (ISCL) +License :: OSI Approved :: Jabber Open Source License +License :: OSI Approved :: MirOS License (MirOS) +License :: OSI Approved :: MIT License +License :: OSI Approved :: MITRE Collaborative Virtual Workspace License (CVW) +License :: OSI Approved :: Motosoto License +License :: OSI Approved :: Mozilla Public License 1.0 (MPL) +License :: OSI Approved :: Mozilla Public License 1.1 (MPL 1.1) +License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0) +License :: OSI Approved :: Nethack General Public License +License :: OSI Approved :: Nokia Open Source License +License :: OSI Approved :: Open Group Test Suite License +License :: OSI Approved :: PostgreSQL License +License :: OSI Approved :: Python License (CNRI Python License) +License :: OSI Approved :: Python Software Foundation License +License :: OSI Approved :: Qt Public License (QPL) +License :: OSI Approved :: Ricoh Source Code Public License +License :: OSI Approved :: SIL Open Font License 1.1 (OFL-1.1) +License :: OSI Approved :: Sleepycat License +License :: OSI Approved :: Sun Industry Standards Source License (SISSL) +License :: OSI Approved :: Sun Public License +License :: OSI Approved :: Universal Permissive License (UPL) +License :: OSI Approved :: University of Illinois/NCSA Open Source License +License :: OSI Approved :: Vovida Software License 1.0 +License :: OSI Approved :: W3C License +License :: OSI Approved :: X.Net License +License :: OSI Approved :: zlib/libpng License +License :: OSI Approved :: Zope Public License +License :: Other/Proprietary License +License :: Public Domain +License :: Repoze Public License +Natural Language :: Afrikaans +Natural Language :: Arabic +Natural Language :: Bengali +Natural Language :: Bosnian +Natural Language :: Bulgarian +Natural Language :: Cantonese +Natural Language :: Catalan +Natural Language :: Chinese (Simplified) +Natural Language :: Chinese (Traditional) +Natural Language :: Croatian +Natural Language :: Czech +Natural Language :: Danish +Natural Language :: Dutch +Natural Language :: English +Natural Language :: Esperanto +Natural Language :: Finnish +Natural Language :: French +Natural Language :: Galician +Natural Language :: German +Natural Language :: Greek +Natural Language :: Hebrew +Natural Language :: Hindi +Natural Language :: Hungarian +Natural Language :: Icelandic +Natural Language :: Indonesian +Natural Language :: Italian +Natural Language :: Japanese +Natural Language :: Javanese +Natural Language :: Korean +Natural Language :: Latin +Natural Language :: Latvian +Natural Language :: Macedonian +Natural Language :: Malay +Natural Language :: Marathi +Natural Language :: Norwegian +Natural Language :: Panjabi +Natural Language :: Persian +Natural Language :: Polish +Natural Language :: Portuguese +Natural Language :: Portuguese (Brazilian) +Natural Language :: Romanian +Natural Language :: Russian +Natural Language :: Serbian +Natural Language :: Slovak +Natural Language :: Slovenian +Natural Language :: Spanish +Natural Language :: Swedish +Natural Language :: Tamil +Natural Language :: Telugu +Natural Language :: Thai +Natural Language :: Tibetan +Natural Language :: Turkish +Natural Language :: Ukrainian +Natural Language :: Urdu +Natural Language :: Vietnamese +Operating System :: Android +Operating System :: BeOS +Operating System :: iOS +Operating System :: MacOS +Operating System :: MacOS :: MacOS 9 +Operating System :: MacOS :: MacOS X +Operating System :: Microsoft +Operating System :: Microsoft :: MS-DOS +Operating System :: Microsoft :: Windows +Operating System :: Microsoft :: Windows :: Windows 10 +Operating System :: Microsoft :: Windows :: Windows 3.1 or Earlier +Operating System :: Microsoft :: Windows :: Windows 7 +Operating System :: Microsoft :: Windows :: Windows 8 +Operating System :: Microsoft :: Windows :: Windows 8.1 +Operating System :: Microsoft :: Windows :: Windows 95/98/2000 +Operating System :: Microsoft :: Windows :: Windows CE +Operating System :: Microsoft :: Windows :: Windows NT/2000 +Operating System :: Microsoft :: Windows :: Windows Server 2003 +Operating System :: Microsoft :: Windows :: Windows Server 2008 +Operating System :: Microsoft :: Windows :: Windows Vista +Operating System :: Microsoft :: Windows :: Windows XP +Operating System :: OS/2 +Operating System :: OS Independent +Operating System :: Other OS +Operating System :: PalmOS +Operating System :: PDA Systems +Operating System :: POSIX +Operating System :: POSIX :: AIX +Operating System :: POSIX :: BSD +Operating System :: POSIX :: BSD :: BSD/OS +Operating System :: POSIX :: BSD :: FreeBSD +Operating System :: POSIX :: BSD :: NetBSD +Operating System :: POSIX :: BSD :: OpenBSD +Operating System :: POSIX :: GNU Hurd +Operating System :: POSIX :: HP-UX +Operating System :: POSIX :: IRIX +Operating System :: POSIX :: Linux +Operating System :: POSIX :: Other +Operating System :: POSIX :: SCO +Operating System :: POSIX :: SunOS/Solaris +Operating System :: Unix +Programming Language :: Ada +Programming Language :: APL +Programming Language :: ASP +Programming Language :: Assembly +Programming Language :: Awk +Programming Language :: Basic +Programming Language :: C +Programming Language :: C# +Programming Language :: C++ +Programming Language :: Cold Fusion +Programming Language :: Cython +Programming Language :: Delphi/Kylix +Programming Language :: Dylan +Programming Language :: Eiffel +Programming Language :: Emacs-Lisp +Programming Language :: Erlang +Programming Language :: Euler +Programming Language :: Euphoria +Programming Language :: Forth +Programming Language :: Fortran +Programming Language :: Haskell +Programming Language :: Java +Programming Language :: JavaScript +Programming Language :: Lisp +Programming Language :: Logo +Programming Language :: ML +Programming Language :: Modula +Programming Language :: Objective C +Programming Language :: Object Pascal +Programming Language :: OCaml +Programming Language :: Other +Programming Language :: Other Scripting Engines +Programming Language :: Pascal +Programming Language :: Perl +Programming Language :: PHP +Programming Language :: Pike +Programming Language :: Pliant +Programming Language :: PL/SQL +Programming Language :: PROGRESS +Programming Language :: Prolog +Programming Language :: Python +Programming Language :: Python :: 2 +Programming Language :: Python :: 2.3 +Programming Language :: Python :: 2.4 +Programming Language :: Python :: 2.5 +Programming Language :: Python :: 2.6 +Programming Language :: Python :: 2.7 +Programming Language :: Python :: 2 :: Only +Programming Language :: Python :: 3 +Programming Language :: Python :: 3.0 +Programming Language :: Python :: 3.1 +Programming Language :: Python :: 3.2 +Programming Language :: Python :: 3.3 +Programming Language :: Python :: 3.4 +Programming Language :: Python :: 3.5 +Programming Language :: Python :: 3.6 +Programming Language :: Python :: 3.7 +Programming Language :: Python :: 3.8 +Programming Language :: Python :: 3 :: Only +Programming Language :: Python :: Implementation +Programming Language :: Python :: Implementation :: CPython +Programming Language :: Python :: Implementation :: IronPython +Programming Language :: Python :: Implementation :: Jython +Programming Language :: Python :: Implementation :: MicroPython +Programming Language :: Python :: Implementation :: PyPy +Programming Language :: Python :: Implementation :: Stackless +Programming Language :: R +Programming Language :: REBOL +Programming Language :: Rexx +Programming Language :: Ruby +Programming Language :: Rust +Programming Language :: Scheme +Programming Language :: Simula +Programming Language :: Smalltalk +Programming Language :: SQL +Programming Language :: Tcl +Programming Language :: Unix Shell +Programming Language :: Visual Basic +Programming Language :: XBasic +Programming Language :: YACC +Programming Language :: Zope +Topic :: Adaptive Technologies +Topic :: Artistic Software +Topic :: Communications +Topic :: Communications :: BBS +Topic :: Communications :: Chat +Topic :: Communications :: Chat :: ICQ +Topic :: Communications :: Chat :: Internet Relay Chat +Topic :: Communications :: Chat :: Unix Talk +Topic :: Communications :: Conferencing +Topic :: Communications :: Email +Topic :: Communications :: Email :: Address Book +Topic :: Communications :: Email :: Email Clients (MUA) +Topic :: Communications :: Email :: Filters +Topic :: Communications :: Email :: Mailing List Servers +Topic :: Communications :: Email :: Mail Transport Agents +Topic :: Communications :: Email :: Post-Office +Topic :: Communications :: Email :: Post-Office :: IMAP +Topic :: Communications :: Email :: Post-Office :: POP3 +Topic :: Communications :: Fax +Topic :: Communications :: FIDO +Topic :: Communications :: File Sharing +Topic :: Communications :: File Sharing :: Gnutella +Topic :: Communications :: File Sharing :: Napster +Topic :: Communications :: Ham Radio +Topic :: Communications :: Internet Phone +Topic :: Communications :: Telephony +Topic :: Communications :: Usenet News +Topic :: Database +Topic :: Database :: Database Engines/Servers +Topic :: Database :: Front-Ends +Topic :: Desktop Environment +Topic :: Desktop Environment :: File Managers +Topic :: Desktop Environment :: Gnome +Topic :: Desktop Environment :: GNUstep +Topic :: Desktop Environment :: K Desktop Environment (KDE) +Topic :: Desktop Environment :: K Desktop Environment (KDE) :: Themes +Topic :: Desktop Environment :: PicoGUI +Topic :: Desktop Environment :: PicoGUI :: Applications +Topic :: Desktop Environment :: PicoGUI :: Themes +Topic :: Desktop Environment :: Screen Savers +Topic :: Desktop Environment :: Window Managers +Topic :: Desktop Environment :: Window Managers :: Afterstep +Topic :: Desktop Environment :: Window Managers :: Afterstep :: Themes +Topic :: Desktop Environment :: Window Managers :: Applets +Topic :: Desktop Environment :: Window Managers :: Blackbox +Topic :: Desktop Environment :: Window Managers :: Blackbox :: Themes +Topic :: Desktop Environment :: Window Managers :: CTWM +Topic :: Desktop Environment :: Window Managers :: CTWM :: Themes +Topic :: Desktop Environment :: Window Managers :: Enlightenment +Topic :: Desktop Environment :: Window Managers :: Enlightenment :: Epplets +Topic :: Desktop Environment :: Window Managers :: Enlightenment :: Themes DR15 +Topic :: Desktop Environment :: Window Managers :: Enlightenment :: Themes DR16 +Topic :: Desktop Environment :: Window Managers :: Enlightenment :: Themes DR17 +Topic :: Desktop Environment :: Window Managers :: Fluxbox +Topic :: Desktop Environment :: Window Managers :: Fluxbox :: Themes +Topic :: Desktop Environment :: Window Managers :: FVWM +Topic :: Desktop Environment :: Window Managers :: FVWM :: Themes +Topic :: Desktop Environment :: Window Managers :: IceWM +Topic :: Desktop Environment :: Window Managers :: IceWM :: Themes +Topic :: Desktop Environment :: Window Managers :: MetaCity +Topic :: Desktop Environment :: Window Managers :: MetaCity :: Themes +Topic :: Desktop Environment :: Window Managers :: Oroborus +Topic :: Desktop Environment :: Window Managers :: Oroborus :: Themes +Topic :: Desktop Environment :: Window Managers :: Sawfish +Topic :: Desktop Environment :: Window Managers :: Sawfish :: Themes 0.30 +Topic :: Desktop Environment :: Window Managers :: Sawfish :: Themes pre-0.30 +Topic :: Desktop Environment :: Window Managers :: Waimea +Topic :: Desktop Environment :: Window Managers :: Waimea :: Themes +Topic :: Desktop Environment :: Window Managers :: Window Maker +Topic :: Desktop Environment :: Window Managers :: Window Maker :: Applets +Topic :: Desktop Environment :: Window Managers :: Window Maker :: Themes +Topic :: Desktop Environment :: Window Managers :: XFCE +Topic :: Desktop Environment :: Window Managers :: XFCE :: Themes +Topic :: Documentation +Topic :: Documentation :: Sphinx +Topic :: Education +Topic :: Education :: Computer Aided Instruction (CAI) +Topic :: Education :: Testing +Topic :: Games/Entertainment +Topic :: Games/Entertainment :: Arcade +Topic :: Games/Entertainment :: Board Games +Topic :: Games/Entertainment :: First Person Shooters +Topic :: Games/Entertainment :: Fortune Cookies +Topic :: Games/Entertainment :: Multi-User Dungeons (MUD) +Topic :: Games/Entertainment :: Puzzle Games +Topic :: Games/Entertainment :: Real Time Strategy +Topic :: Games/Entertainment :: Role-Playing +Topic :: Games/Entertainment :: Side-Scrolling/Arcade Games +Topic :: Games/Entertainment :: Simulation +Topic :: Games/Entertainment :: Turn Based Strategy +Topic :: Home Automation +Topic :: Internet +Topic :: Internet :: File Transfer Protocol (FTP) +Topic :: Internet :: Finger +Topic :: Internet :: Log Analysis +Topic :: Internet :: Name Service (DNS) +Topic :: Internet :: Proxy Servers +Topic :: Internet :: WAP +Topic :: Internet :: WWW/HTTP +Topic :: Internet :: WWW/HTTP :: Browsers +Topic :: Internet :: WWW/HTTP :: Dynamic Content +Topic :: Internet :: WWW/HTTP :: Dynamic Content :: CGI Tools/Libraries +Topic :: Internet :: WWW/HTTP :: Dynamic Content :: Content Management System +Topic :: Internet :: WWW/HTTP :: Dynamic Content :: Message Boards +Topic :: Internet :: WWW/HTTP :: Dynamic Content :: News/Diary +Topic :: Internet :: WWW/HTTP :: Dynamic Content :: Page Counters +Topic :: Internet :: WWW/HTTP :: Dynamic Content :: Wiki +Topic :: Internet :: WWW/HTTP :: HTTP Servers +Topic :: Internet :: WWW/HTTP :: Indexing/Search +Topic :: Internet :: WWW/HTTP :: Session +Topic :: Internet :: WWW/HTTP :: Site Management +Topic :: Internet :: WWW/HTTP :: Site Management :: Link Checking +Topic :: Internet :: WWW/HTTP :: WSGI +Topic :: Internet :: WWW/HTTP :: WSGI :: Application +Topic :: Internet :: WWW/HTTP :: WSGI :: Middleware +Topic :: Internet :: WWW/HTTP :: WSGI :: Server +Topic :: Internet :: XMPP +Topic :: Internet :: Z39.50 +Topic :: Multimedia +Topic :: Multimedia :: Graphics +Topic :: Multimedia :: Graphics :: 3D Modeling +Topic :: Multimedia :: Graphics :: 3D Rendering +Topic :: Multimedia :: Graphics :: Capture +Topic :: Multimedia :: Graphics :: Capture :: Digital Camera +Topic :: Multimedia :: Graphics :: Capture :: Scanners +Topic :: Multimedia :: Graphics :: Capture :: Screen Capture +Topic :: Multimedia :: Graphics :: Editors +Topic :: Multimedia :: Graphics :: Editors :: Raster-Based +Topic :: Multimedia :: Graphics :: Editors :: Vector-Based +Topic :: Multimedia :: Graphics :: Graphics Conversion +Topic :: Multimedia :: Graphics :: Presentation +Topic :: Multimedia :: Graphics :: Viewers +Topic :: Multimedia :: Sound/Audio +Topic :: Multimedia :: Sound/Audio :: Analysis +Topic :: Multimedia :: Sound/Audio :: Capture/Recording +Topic :: Multimedia :: Sound/Audio :: CD Audio +Topic :: Multimedia :: Sound/Audio :: CD Audio :: CD Playing +Topic :: Multimedia :: Sound/Audio :: CD Audio :: CD Ripping +Topic :: Multimedia :: Sound/Audio :: CD Audio :: CD Writing +Topic :: Multimedia :: Sound/Audio :: Conversion +Topic :: Multimedia :: Sound/Audio :: Editors +Topic :: Multimedia :: Sound/Audio :: MIDI +Topic :: Multimedia :: Sound/Audio :: Mixers +Topic :: Multimedia :: Sound/Audio :: Players +Topic :: Multimedia :: Sound/Audio :: Players :: MP3 +Topic :: Multimedia :: Sound/Audio :: Sound Synthesis +Topic :: Multimedia :: Sound/Audio :: Speech +Topic :: Multimedia :: Video +Topic :: Multimedia :: Video :: Capture +Topic :: Multimedia :: Video :: Conversion +Topic :: Multimedia :: Video :: Display +Topic :: Multimedia :: Video :: Non-Linear Editor +Topic :: Office/Business +Topic :: Office/Business :: Financial +Topic :: Office/Business :: Financial :: Accounting +Topic :: Office/Business :: Financial :: Investment +Topic :: Office/Business :: Financial :: Point-Of-Sale +Topic :: Office/Business :: Financial :: Spreadsheet +Topic :: Office/Business :: Groupware +Topic :: Office/Business :: News/Diary +Topic :: Office/Business :: Office Suites +Topic :: Office/Business :: Scheduling +Topic :: Other/Nonlisted Topic +Topic :: Printing +Topic :: Religion +Topic :: Scientific/Engineering +Topic :: Scientific/Engineering :: Artificial Intelligence +Topic :: Scientific/Engineering :: Artificial Life +Topic :: Scientific/Engineering :: Astronomy +Topic :: Scientific/Engineering :: Atmospheric Science +Topic :: Scientific/Engineering :: Bio-Informatics +Topic :: Scientific/Engineering :: Chemistry +Topic :: Scientific/Engineering :: Electronic Design Automation (EDA) +Topic :: Scientific/Engineering :: GIS +Topic :: Scientific/Engineering :: Human Machine Interfaces +Topic :: Scientific/Engineering :: Image Recognition +Topic :: Scientific/Engineering :: Information Analysis +Topic :: Scientific/Engineering :: Interface Engine/Protocol Translator +Topic :: Scientific/Engineering :: Mathematics +Topic :: Scientific/Engineering :: Medical Science Apps. +Topic :: Scientific/Engineering :: Physics +Topic :: Scientific/Engineering :: Visualization +Topic :: Security +Topic :: Security :: Cryptography +Topic :: Sociology +Topic :: Sociology :: Genealogy +Topic :: Sociology :: History +Topic :: Software Development +Topic :: Software Development :: Assemblers +Topic :: Software Development :: Bug Tracking +Topic :: Software Development :: Build Tools +Topic :: Software Development :: Code Generators +Topic :: Software Development :: Compilers +Topic :: Software Development :: Debuggers +Topic :: Software Development :: Disassemblers +Topic :: Software Development :: Documentation +Topic :: Software Development :: Embedded Systems +Topic :: Software Development :: Internationalization +Topic :: Software Development :: Interpreters +Topic :: Software Development :: Libraries +Topic :: Software Development :: Libraries :: Application Frameworks +Topic :: Software Development :: Libraries :: Java Libraries +Topic :: Software Development :: Libraries :: Perl Modules +Topic :: Software Development :: Libraries :: PHP Classes +Topic :: Software Development :: Libraries :: Pike Modules +Topic :: Software Development :: Libraries :: pygame +Topic :: Software Development :: Libraries :: Python Modules +Topic :: Software Development :: Libraries :: Ruby Modules +Topic :: Software Development :: Libraries :: Tcl Extensions +Topic :: Software Development :: Localization +Topic :: Software Development :: Object Brokering +Topic :: Software Development :: Object Brokering :: CORBA +Topic :: Software Development :: Pre-processors +Topic :: Software Development :: Quality Assurance +Topic :: Software Development :: Testing +Topic :: Software Development :: Testing :: Acceptance +Topic :: Software Development :: Testing :: BDD +Topic :: Software Development :: Testing :: Mocking +Topic :: Software Development :: Testing :: Traffic Generation +Topic :: Software Development :: Testing :: Unit +Topic :: Software Development :: User Interfaces +Topic :: Software Development :: Version Control +Topic :: Software Development :: Version Control :: Bazaar +Topic :: Software Development :: Version Control :: CVS +Topic :: Software Development :: Version Control :: Git +Topic :: Software Development :: Version Control :: Mercurial +Topic :: Software Development :: Version Control :: RCS +Topic :: Software Development :: Version Control :: SCCS +Topic :: Software Development :: Widget Sets +Topic :: System +Topic :: System :: Archiving +Topic :: System :: Archiving :: Backup +Topic :: System :: Archiving :: Compression +Topic :: System :: Archiving :: Mirroring +Topic :: System :: Archiving :: Packaging +Topic :: System :: Benchmark +Topic :: System :: Boot +Topic :: System :: Boot :: Init +Topic :: System :: Clustering +Topic :: System :: Console Fonts +Topic :: System :: Distributed Computing +Topic :: System :: Emulators +Topic :: System :: Filesystems +Topic :: System :: Hardware +Topic :: System :: Hardware :: Hardware Drivers +Topic :: System :: Hardware :: Mainframes +Topic :: System :: Hardware :: Symmetric Multi-processing +Topic :: System :: Installation/Setup +Topic :: System :: Logging +Topic :: System :: Monitoring +Topic :: System :: Networking +Topic :: System :: Networking :: Firewalls +Topic :: System :: Networking :: Monitoring +Topic :: System :: Networking :: Monitoring :: Hardware Watchdog +Topic :: System :: Networking :: Time Synchronization +Topic :: System :: Operating System +Topic :: System :: Operating System Kernels +Topic :: System :: Operating System Kernels :: BSD +Topic :: System :: Operating System Kernels :: GNU Hurd +Topic :: System :: Operating System Kernels :: Linux +Topic :: System :: Power (UPS) +Topic :: System :: Recovery Tools +Topic :: System :: Shells +Topic :: System :: Software Distribution +Topic :: System :: Systems Administration +Topic :: System :: Systems Administration :: Authentication/Directory +Topic :: System :: Systems Administration :: Authentication/Directory :: LDAP +Topic :: System :: Systems Administration :: Authentication/Directory :: NIS +Topic :: System :: System Shells +Topic :: Terminals +Topic :: Terminals :: Serial +Topic :: Terminals :: Telnet +Topic :: Terminals :: Terminal Emulators/X Terminals +Topic :: Text Editors +Topic :: Text Editors :: Documentation +Topic :: Text Editors :: Emacs +Topic :: Text Editors :: Integrated Development Environments (IDE) +Topic :: Text Editors :: Text Processing +Topic :: Text Editors :: Word Processors +Topic :: Text Processing +Topic :: Text Processing :: Filters +Topic :: Text Processing :: Fonts +Topic :: Text Processing :: General +Topic :: Text Processing :: Indexing +Topic :: Text Processing :: Linguistic +Topic :: Text Processing :: Markup +Topic :: Text Processing :: Markup :: HTML +Topic :: Text Processing :: Markup :: LaTeX +Topic :: Text Processing :: Markup :: SGML +Topic :: Text Processing :: Markup :: VRML +Topic :: Text Processing :: Markup :: XML +Topic :: Utilities +Typing :: Typed \ No newline at end of file diff --git a/check_python_versions.py b/check_python_versions.py index e25ccee..e3b1461 100755 --- a/check_python_versions.py +++ b/check_python_versions.py @@ -189,10 +189,15 @@ def confirm(prompt): def to_literal(value, quote_style='"'): - safe_characters = string.ascii_letters + string.digits + ' .:,-=><()/+' + # Because I don't want to deal with quoting, I'll require all values + # to contain only safe characters (i.e. no ' or " or \). Except some + # PyPI classifiers do include ' so I need to handle that at least. + safe_characters = string.ascii_letters + string.digits + " .:,-=><()/+'#" assert all( c in safe_characters for c in value ), f'{value!r} has unexpected characters' + if quote_style == "'" and quote_style in value: + quote_style = '"' assert quote_style not in value return f'{quote_style}{value}{quote_style}' diff --git a/tests.py b/tests.py index 8790562..95db4c0 100644 --- a/tests.py +++ b/tests.py @@ -203,6 +203,28 @@ def test_eval_ast_node(code, expected): assert cpv.eval_ast_node(node, 'bar') == expected +def test_to_literal(): + assert cpv.to_literal("blah") == '"blah"' + assert cpv.to_literal("blah", "'") == "'blah'" + + +def test_to_literal_embedded_quote(): + assert cpv.to_literal( + "Environment :: Handhelds/PDA's" + ) == '"Environment :: Handhelds/PDA\'s"' + assert cpv.to_literal( + "Environment :: Handhelds/PDA's", "'" + ) == '"Environment :: Handhelds/PDA\'s"' + + +def test_to_literal_all_the_classifiers(): + with open('CLASSIFIERS') as f: + for line in f: + classifier = line.strip() + literal = cpv.to_literal(classifier) + assert ast.literal_eval(literal) == classifier + + def test_update_call_arg_in_source(): source_lines = textwrap.dedent("""\ setup( From 8cb79832b26a51da37e86fdeb995818196070a1b Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Sat, 13 Apr 2019 00:08:23 +0300 Subject: [PATCH 07/76] Allow half-open ranges like --expect 3.5- --- CHANGES.rst | 2 ++ check_python_versions.py | 17 ++++++++++++++++- tests.py | 13 +++++++++++++ 3 files changed, 31 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index b00ddf0..8020531 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -6,6 +6,8 @@ Changelog - Ignore unreleased Python versions (3.8 at the moment). +- Allow half-open ranges like ``--expect 3.5-``. + 0.11.0 (2019-02-13) ------------------- diff --git a/check_python_versions.py b/check_python_versions.py index e3b1461..fc2e391 100755 --- a/check_python_versions.py +++ b/check_python_versions.py @@ -53,6 +53,12 @@ MAX_PYTHON_2_VERSION = 7 # i.e. 2.7 CURRENT_PYTHON_3_VERSION = 7 # i.e. 3.7 +MAX_MINOR_FOR_MAJOR = { + 1: MAX_PYTHON_1_VERSION, + 2: MAX_PYTHON_2_VERSION, + 3: CURRENT_PYTHON_3_VERSION, +} + def warn(msg): print(msg, file=sys.stderr) @@ -559,7 +565,16 @@ def parse_version_list(v): lo = hi = part lo_major, lo_minor = parse_version(lo) - hi_major, hi_minor = parse_version(hi) + + if hi: + hi_major, hi_minor = parse_version(hi) + else: + hi_major = lo_major + try: + hi_minor = MAX_MINOR_FOR_MAJOR[hi_major] + except KeyError: + raise argparse.ArgumentTypeError( + f'bad range: {part}') if lo_major != hi_major: raise argparse.ArgumentTypeError( diff --git a/tests.py b/tests.py index 95db4c0..3e28e07 100644 --- a/tests.py +++ b/tests.py @@ -599,6 +599,19 @@ def test_parse_version_list(): ) == ['2.7', '3.4', '3.5', '3.6'] +def test_parse_version_list_magic_range(monkeypatch): + monkeypatch.setattr(cpv, 'CURRENT_PYTHON_3_VERSION', 7) + assert cpv.parse_version_list( + '2.7,3.4-' + ) == ['2.7', '3.4', '3.5', '3.6', '3.7'] + + +def test_parse_version_list_bad_magic_range(): + with pytest.raises(argparse.ArgumentTypeError, + match=r'bad range: 4\.1-'): + cpv.parse_version_list('4.1-') + + def test_parse_version_list_bad_range(): with pytest.raises(argparse.ArgumentTypeError, match=r'bad range: 2\.7-3\.4 \(2 != 3\)'): From ec3228f114bb6ed0018836f5a50e2d6fd85a7688 Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Sat, 13 Apr 2019 00:22:19 +0300 Subject: [PATCH 08/76] Preserve major Python version classifiers --- check_python_versions.py | 15 ++++++++++ tests.py | 59 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+) diff --git a/check_python_versions.py b/check_python_versions.py index fc2e391..0995d6a 100755 --- a/check_python_versions.py +++ b/check_python_versions.py @@ -88,6 +88,14 @@ def is_version_classifier(s): return s.startswith(prefix) and s[len(prefix):len(prefix) + 1].isdigit() +def is_major_version_classifier(s): + prefix = 'Programming Language :: Python :: ' + return ( + s.startswith(prefix) + and s[len(prefix):].replace(' :: Only', '').isdigit() + ) + + def get_versions_from_classifiers(classifiers): # Based on # https://github.com/mgedmin/project-summary/blob/master/summary.py#L221-L234 @@ -119,6 +127,13 @@ def update_classifiers(classifiers, new_versions): else: pos = len(classifiers) + if any(map(is_major_version_classifier, classifiers)): + new_versions = sorted( + set(new_versions).union( + v.partition('.')[0] for v in new_versions + ) + ) + classifiers = [ s for s in classifiers if not is_version_classifier(s) ] diff --git a/tests.py b/tests.py index 3e28e07..cc7a187 100644 --- a/tests.py +++ b/tests.py @@ -96,6 +96,65 @@ def test_update_classifiers(): 'Programming Language :: Python :: Implementation :: CPython', 'Programming Language :: Python :: Implementation :: PyPy', 'Typing :: Typed', + ], ['2.7', '3.7']) == [ + 'Development Status :: 4 - Beta', + 'Environment :: Console', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: GNU General Public License (GPL)', + 'Operating System :: OS Independent', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: Implementation :: CPython', + 'Programming Language :: Python :: Implementation :: PyPy', + 'Typing :: Typed', + ] + + +def test_update_classifiers_drop_major(): + assert cpv.update_classifiers([ + 'Development Status :: 4 - Beta', + 'Environment :: Console', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: GNU General Public License (GPL)', + 'Operating System :: OS Independent', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: Implementation :: CPython', + 'Programming Language :: Python :: Implementation :: PyPy', + 'Typing :: Typed', + ], ['3.6', '3.7']) == [ + 'Development Status :: 4 - Beta', + 'Environment :: Console', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: GNU General Public License (GPL)', + 'Operating System :: OS Independent', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: Implementation :: CPython', + 'Programming Language :: Python :: Implementation :: PyPy', + 'Typing :: Typed', + ] + + +def test_update_classifiers_no_major(): + assert cpv.update_classifiers([ + 'Development Status :: 4 - Beta', + 'Environment :: Console', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: GNU General Public License (GPL)', + 'Operating System :: OS Independent', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: Implementation :: CPython', + 'Programming Language :: Python :: Implementation :: PyPy', + 'Typing :: Typed', ], ['2.7', '3.7']) == [ 'Development Status :: 4 - Beta', 'Environment :: Console', From ffc8e51d7d0269a2560bf05eb491ae096038aeb8 Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Sat, 13 Apr 2019 00:37:44 +0300 Subject: [PATCH 09/76] Fix update when classifiers cannot be parsed Fixes TypeError in difflib because new_lines happened to be None because I forgot to return a value when update_call_arg_in_source fails. --- check_python_versions.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/check_python_versions.py b/check_python_versions.py index 0995d6a..b23aef5 100755 --- a/check_python_versions.py +++ b/check_python_versions.py @@ -181,7 +181,7 @@ def update_setup_py_keyword(setup_py, keyword, new_value): def confirm_and_update_file(filename, old_lines, new_lines): print_diff(old_lines, new_lines, filename) - if confirm(f"Write changes to {filename}?"): + if new_lines != old_lines and confirm(f"Write changes to {filename}?"): mode = stat.S_IMODE(os.stat(filename).st_mode) tempfile = filename + '.tmp' with open(tempfile, 'w') as f: @@ -230,13 +230,13 @@ def update_call_arg_in_source(source_lines, function, keyword, new_value): break else: warn(f'Did not find {function}() call') - return + return source_lines for n, line in lines: if line.lstrip().startswith(f'{keyword}='): break else: warn(f'Did not find {keyword}= argument in {function}() call') - return + return source_lines start = n + 1 indent = 8 @@ -251,6 +251,7 @@ def update_call_arg_in_source(source_lines, function, keyword, new_value): quote_style = stripped[0] else: warn(f'Did not understand {keyword}= formatting in {function}() call') + return source_lines end = n return source_lines[:start] + [ From 85392f3a1ea76d5228fe482267ba9a65fcbc3625 Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Sat, 13 Apr 2019 00:39:38 +0300 Subject: [PATCH 10/76] Treat EOF as a blank answer in confirm() When the stdin is redirected from /dev/null, this will apply the default action (no update). When the stdin is a terminal, ^D will be the same as hitting Enter and accepting the default action (no update) once. --- check_python_versions.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/check_python_versions.py b/check_python_versions.py index b23aef5..5df0913 100755 --- a/check_python_versions.py +++ b/check_python_versions.py @@ -200,7 +200,10 @@ def print_diff(a, b, filename): def confirm(prompt): while True: - answer = input(f'{prompt} [y/N] ').strip().lower() + try: + answer = input(f'{prompt} [y/N] ').strip().lower() + except EOFError: + answer = "" if answer == 'y': print() return True From 004d02a7a186a6178c0dc4866a6ef2ad61546ea9 Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Sat, 13 Apr 2019 00:42:46 +0300 Subject: [PATCH 11/76] Suppress the traceback on ^C --- check_python_versions.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/check_python_versions.py b/check_python_versions.py index 5df0913..2d76478 100755 --- a/check_python_versions.py +++ b/check_python_versions.py @@ -681,7 +681,7 @@ def update_versions(where='.', *, add=None, drop=None, update=None): update_supported_python_versions(where, new_versions) -def main(): +def _main(): parser = argparse.ArgumentParser( description="verify that supported Python versions are the same" " in setup.py, tox.ini, .travis.yml and appveyor.yml") @@ -745,5 +745,12 @@ def main(): print("\n\nall ok!") +def main(): + try: + _main() + except KeyboardInterrupt: + sys.exit(2) + + if __name__ == '__main__': main() From 8146097926a3438b1a4c3982c11f93f7ae2d500f Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Sat, 13 Apr 2019 01:04:55 +0300 Subject: [PATCH 12/76] Allow half-open ranges like --drop -3.4 --- check_python_versions.py | 17 +++++++++++------ tests.py | 20 ++++++++++++-------- 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/check_python_versions.py b/check_python_versions.py index 2d76478..be22912 100755 --- a/check_python_versions.py +++ b/check_python_versions.py @@ -583,17 +583,22 @@ def parse_version_list(v): else: lo = hi = part - lo_major, lo_minor = parse_version(lo) - - if hi: + if lo and hi: + lo_major, lo_minor = parse_version(lo) hi_major, hi_minor = parse_version(hi) - else: - hi_major = lo_major + elif hi and not lo: + hi_major, hi_minor = parse_version(hi) + lo_major, lo_minor = hi_major, 0 + elif lo and not hi: + lo_major, lo_minor = parse_version(lo) try: - hi_minor = MAX_MINOR_FOR_MAJOR[hi_major] + hi_major, hi_minor = lo_major, MAX_MINOR_FOR_MAJOR[lo_major] except KeyError: raise argparse.ArgumentTypeError( f'bad range: {part}') + else: + raise argparse.ArgumentTypeError( + f'bad range: {part}') if lo_major != hi_major: raise argparse.ArgumentTypeError( diff --git a/tests.py b/tests.py index cc7a187..4017fae 100644 --- a/tests.py +++ b/tests.py @@ -1,6 +1,7 @@ import argparse import ast import os +import re import sys import textwrap @@ -663,19 +664,22 @@ def test_parse_version_list_magic_range(monkeypatch): assert cpv.parse_version_list( '2.7,3.4-' ) == ['2.7', '3.4', '3.5', '3.6', '3.7'] + assert cpv.parse_version_list( + '2.6,-3.4' + ) == ['2.6', '3.0', '3.1', '3.2', '3.3', '3.4'] -def test_parse_version_list_bad_magic_range(): +@pytest.mark.parametrize('v', [ + '4.1-', # unknown major version + '-', # both endpoints missing + '2.7-3.4', # major versions differ +]) +def test_parse_version_list_bad_range(v): with pytest.raises(argparse.ArgumentTypeError, - match=r'bad range: 4\.1-'): - cpv.parse_version_list('4.1-') + match=re.escape(f'bad range: {v}')): + cpv.parse_version_list(v) -def test_parse_version_list_bad_range(): - with pytest.raises(argparse.ArgumentTypeError, - match=r'bad range: 2\.7-3\.4 \(2 != 3\)'): - cpv.parse_version_list('2.7-3.4') - def test_parse_version_list_bad_number(): with pytest.raises(argparse.ArgumentTypeError): From 4a2593ffff56ec87024d5b0524ba2668def40eca Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Sat, 13 Apr 2019 01:13:37 +0300 Subject: [PATCH 13/76] Fix updating classifiers when ] is on the same line --- check_python_versions.py | 13 +++++++++++-- tests.py | 23 +++++++++++++++++++++++ 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/check_python_versions.py b/check_python_versions.py index be22912..0d5c225 100755 --- a/check_python_versions.py +++ b/check_python_versions.py @@ -244,23 +244,32 @@ def update_call_arg_in_source(source_lines, function, keyword, new_value): start = n + 1 indent = 8 quote_style = '"' + need_closing_brace = False for n, line in lines: stripped = line.lstrip() if stripped.startswith('],'): + end = n break elif stripped: indent = len(line) - len(stripped) if stripped[0] in ('"', "'"): quote_style = stripped[0] + if line.rstrip().endswith('],'): + need_closing_brace = True + end = n + 1 + break else: warn(f'Did not understand {keyword}= formatting in {function}() call') return source_lines - end = n + + extra = [] + if need_closing_brace: + extra = [f"{' ' * (indent - 4)}],\n"] return source_lines[:start] + [ f"{' ' * indent}{to_literal(value, quote_style)},\n" for value in new_value - ] + source_lines[end:] + ] + extra + source_lines[end:] def find_call_kwarg_in_ast(tree, funcname, keyword, filename='setup.py'): diff --git a/tests.py b/tests.py index 4017fae..8af9425 100644 --- a/tests.py +++ b/tests.py @@ -333,6 +333,29 @@ def test_update_call_arg_in_source_preserves_indent_and_quote_style(): """) +def test_update_call_arg_in_source_fixes_closing_bracket(): + source_lines = textwrap.dedent("""\ + setup(foo=1, + bar=[ + 'a', + 'b', + 'c'], + baz=2, + ) + """).splitlines(True) + result = cpv.update_call_arg_in_source(source_lines, "setup", "bar", + ["x", "y"]) + assert "".join(result) == textwrap.dedent("""\ + setup(foo=1, + bar=[ + 'x', + 'y', + ], + baz=2, + ) + """) + + @pytest.mark.parametrize('code', [ '[2 * 2]', '"".join([2 * 2])', From 71f1f9c359dfaab423ee6cac589306334f0f96eb Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Sat, 13 Apr 2019 01:32:53 +0300 Subject: [PATCH 14/76] Disable coverage enforcement for now --- tox.ini | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 25c3b02..d651019 100644 --- a/tox.ini +++ b/tox.ini @@ -16,7 +16,8 @@ deps = coverage commands = coverage run -m pytest tests.py {posargs} - coverage report -m --fail-under=100 +## coverage report -m --fail-under=100 + coverage report -m [testenv:flake8] basepython = python3.6 From fa3981aea17675d9bba0914450da7d4c525875ea Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Sat, 13 Apr 2019 01:33:03 +0300 Subject: [PATCH 15/76] Handle another list literal style --- check_python_versions.py | 26 ++++++++++++++------------ tests.py | 22 ++++++++++++++++++++++ 2 files changed, 36 insertions(+), 12 deletions(-) diff --git a/check_python_versions.py b/check_python_versions.py index 0d5c225..e0dda39 100755 --- a/check_python_versions.py +++ b/check_python_versions.py @@ -235,41 +235,43 @@ def update_call_arg_in_source(source_lines, function, keyword, new_value): warn(f'Did not find {function}() call') return source_lines for n, line in lines: - if line.lstrip().startswith(f'{keyword}='): + stripped = line.lstrip() + if stripped.startswith(f'{keyword}='): + first_indent = len(line) - len(stripped) + must_fix_indents = not line.rstrip().endswith('=[') break else: warn(f'Did not find {keyword}= argument in {function}() call') return source_lines - start = n + 1 - indent = 8 + start = n + indent = first_indent + 4 quote_style = '"' - need_closing_brace = False for n, line in lines: stripped = line.lstrip() if stripped.startswith('],'): - end = n + end = n + 1 break elif stripped: - indent = len(line) - len(stripped) + if not must_fix_indents: + indent = len(line) - len(stripped) if stripped[0] in ('"', "'"): quote_style = stripped[0] if line.rstrip().endswith('],'): - need_closing_brace = True end = n + 1 break else: warn(f'Did not understand {keyword}= formatting in {function}() call') return source_lines - extra = [] - if need_closing_brace: - extra = [f"{' ' * (indent - 4)}],\n"] - return source_lines[:start] + [ + f"{' ' * first_indent}{keyword}=[\n" + ] + [ f"{' ' * indent}{to_literal(value, quote_style)},\n" for value in new_value - ] + extra + source_lines[end:] + ] + [ + f"{' ' * first_indent}],\n" + ] + source_lines[end:] def find_call_kwarg_in_ast(tree, funcname, keyword, filename='setup.py'): diff --git a/tests.py b/tests.py index 8af9425..6ab68a0 100644 --- a/tests.py +++ b/tests.py @@ -356,6 +356,28 @@ def test_update_call_arg_in_source_fixes_closing_bracket(): """) +def test_update_call_arg_in_source_fixes_opening_bracket(): + source_lines = textwrap.dedent("""\ + setup(foo=1, + bar=['a', + 'b', + 'c'], + baz=2, + ) + """).splitlines(True) + result = cpv.update_call_arg_in_source(source_lines, "setup", "bar", + ["x", "y"]) + assert "".join(result) == textwrap.dedent("""\ + setup(foo=1, + bar=[ + 'x', + 'y', + ], + baz=2, + ) + """) + + @pytest.mark.parametrize('code', [ '[2 * 2]', '"".join([2 * 2])', From c39b71fba395e532992269df749b795a280f0ad5 Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Sat, 13 Apr 2019 20:32:10 +0300 Subject: [PATCH 16/76] Refactoring: move the code into a package --- MANIFEST.in | 4 ++++ check-python-versions | 14 ++++++++++++++ setup.py | 9 ++++++--- .../check_python_versions/__init__.py | 5 ----- 4 files changed, 24 insertions(+), 8 deletions(-) create mode 100755 check-python-versions rename check_python_versions.py => src/check_python_versions/__init__.py (99%) diff --git a/MANIFEST.in b/MANIFEST.in index b196f4e..655dd9d 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -10,3 +10,7 @@ include tox.ini # added by check_manifest.py include *.yml + +# added by check_manifest.py +include CLASSIFIERS +include check-python-versions diff --git a/check-python-versions b/check-python-versions new file mode 100755 index 0000000..ad70c47 --- /dev/null +++ b/check-python-versions @@ -0,0 +1,14 @@ +#!/usr/bin/env python3 +""" +Wrapper script for running ./check-python-versions directly from a source +checkout without installing. (You'll need pyyaml in your system Python.) +""" + +import sys +import os + +here = os.path.dirname(__file__) +sys.path.insert(0, os.path.join(here, 'src')) + +from check_python_versions import main # noqa: E402 +main() diff --git a/setup.py b/setup.py index 9ee799c..ce8a1fb 100755 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ import os import re -from setuptools import setup +from setuptools import setup, find_packages here = os.path.dirname(__file__) @@ -11,8 +11,10 @@ with open(os.path.join(here, 'README.rst')) as f: long_description = f.read() +source_dir = os.path.join(here, 'src', 'check_python_versions') + metadata = {} -with open(os.path.join(here, 'check_python_versions.py')) as f: +with open(os.path.join(source_dir, '__init__.py')) as f: rx = re.compile('(__version__|__author__|__url__|__licence__) = (.*)') for line in f: m = rx.match(line) @@ -45,7 +47,8 @@ ], license='GPL', python_requires=">=3.6", - py_modules=['check_python_versions'], + packages=find_packages('src'), + package_dir={'': 'src/'}, entry_points={ 'console_scripts': [ 'check-python-versions = check_python_versions:main', diff --git a/check_python_versions.py b/src/check_python_versions/__init__.py similarity index 99% rename from check_python_versions.py rename to src/check_python_versions/__init__.py index e0dda39..f8624be 100755 --- a/check_python_versions.py +++ b/src/check_python_versions/__init__.py @@ -1,4 +1,3 @@ -#!/usr/bin/python3 """ Check supported Python versions in a Python package. @@ -766,7 +765,3 @@ def main(): _main() except KeyboardInterrupt: sys.exit(2) - - -if __name__ == '__main__': - main() From 18022c720ff3825fbf9cfa29b40e45a61410bbb0 Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Sat, 13 Apr 2019 20:39:52 +0300 Subject: [PATCH 17/76] Move the code to cli.py --- setup.py | 2 +- src/check_python_versions/__init__.py | 753 -------------------------- src/check_python_versions/cli.py | 752 +++++++++++++++++++++++++ tests.py | 2 +- 4 files changed, 754 insertions(+), 755 deletions(-) mode change 100755 => 100644 src/check_python_versions/__init__.py create mode 100755 src/check_python_versions/cli.py diff --git a/setup.py b/setup.py index ce8a1fb..619aa3d 100755 --- a/setup.py +++ b/setup.py @@ -51,7 +51,7 @@ package_dir={'': 'src/'}, entry_points={ 'console_scripts': [ - 'check-python-versions = check_python_versions:main', + 'check-python-versions = check_python_versions.cli:main', ], }, install_requires=['pyyaml'], diff --git a/src/check_python_versions/__init__.py b/src/check_python_versions/__init__.py old mode 100755 new mode 100644 index f8624be..c58c0d6 --- a/src/check_python_versions/__init__.py +++ b/src/check_python_versions/__init__.py @@ -10,758 +10,5 @@ - (optionally) .manylinux-install.sh as used by various ZopeFoundation projects """ - -import argparse -import ast -import configparser -import difflib -import logging -import os -import re -import stat -import string -import subprocess -import sys -from functools import partial - - -try: - import yaml -except ImportError: # pragma: nocover - # Shouldn't happen, we install_requires=['PyYAML'], but maybe someone is - # running ./check_python_versions.py directly from a git checkout. - yaml = None - print("PyYAML is needed for Travis CI/Appveyor support" - " (apt install python3-yaml)") - - __author__ = 'Marius Gedminas ' __version__ = '0.12.0.dev0' - - -log = logging.getLogger('check-python-versions') - - -TOX_INI = 'tox.ini' -TRAVIS_YML = '.travis.yml' -APPVEYOR_YML = 'appveyor.yml' -MANYLINUX_INSTALL_SH = '.manylinux-install.sh' - - -MAX_PYTHON_1_VERSION = 6 # i.e. 1.6 -MAX_PYTHON_2_VERSION = 7 # i.e. 2.7 -CURRENT_PYTHON_3_VERSION = 7 # i.e. 3.7 - -MAX_MINOR_FOR_MAJOR = { - 1: MAX_PYTHON_1_VERSION, - 2: MAX_PYTHON_2_VERSION, - 3: CURRENT_PYTHON_3_VERSION, -} - - -def warn(msg): - print(msg, file=sys.stderr) - - -def pipe(*cmd, **kwargs): - if 'cwd' in kwargs: - log.debug('EXEC cd %s && %s', kwargs['cwd'], ' '.join(cmd)) - else: - log.debug('EXEC %s', ' '.join(cmd)) - p = subprocess.Popen(cmd, stdout=subprocess.PIPE, **kwargs) - return p.communicate()[0].decode('UTF-8', 'replace') - - -def get_supported_python_versions(repo_path='.'): - setup_py = os.path.join(repo_path, 'setup.py') - classifiers = get_setup_py_keyword(setup_py, 'classifiers') - if classifiers is None: - # AST parsing is complicated - classifiers = pipe("python", "setup.py", "-q", "--classifiers", - cwd=repo_path).splitlines() - return get_versions_from_classifiers(classifiers) - - -def is_version_classifier(s): - prefix = 'Programming Language :: Python :: ' - return s.startswith(prefix) and s[len(prefix):len(prefix) + 1].isdigit() - - -def is_major_version_classifier(s): - prefix = 'Programming Language :: Python :: ' - return ( - s.startswith(prefix) - and s[len(prefix):].replace(' :: Only', '').isdigit() - ) - - -def get_versions_from_classifiers(classifiers): - # Based on - # https://github.com/mgedmin/project-summary/blob/master/summary.py#L221-L234 - prefix = 'Programming Language :: Python :: ' - impl_prefix = 'Programming Language :: Python :: Implementation :: ' - cpython = impl_prefix + 'CPython' - versions = { - s[len(prefix):].replace(' :: Only', '').rstrip() - for s in classifiers - if is_version_classifier(s) - } | { - s[len(impl_prefix):].rstrip() - for s in classifiers - if s.startswith(impl_prefix) and s != cpython - } - for major in '2', '3': - if major in versions and any( - v.startswith(f'{major}.') for v in versions): - versions.remove(major) - return sorted(versions) - - -def update_classifiers(classifiers, new_versions): - prefix = 'Programming Language :: Python :: ' - - for pos, s in enumerate(classifiers): - if is_version_classifier(s): - break - else: - pos = len(classifiers) - - if any(map(is_major_version_classifier, classifiers)): - new_versions = sorted( - set(new_versions).union( - v.partition('.')[0] for v in new_versions - ) - ) - - classifiers = [ - s for s in classifiers if not is_version_classifier(s) - ] - new_classifiers = [ - f'{prefix}{version}' - for version in new_versions - ] - classifiers[pos:pos] = new_classifiers - return classifiers - - -def update_supported_python_versions(repo_path, new_versions): - setup_py = os.path.join(repo_path, 'setup.py') - classifiers = get_setup_py_keyword(setup_py, 'classifiers') - if classifiers is None: - return - new_classifiers = update_classifiers(classifiers, new_versions) - update_setup_py_keyword(setup_py, 'classifiers', new_classifiers) - - -def get_python_requires(setup_py='setup.py'): - python_requires = get_setup_py_keyword(setup_py, 'python_requires') - if python_requires is None: - return None - return parse_python_requires(python_requires) - - -def get_setup_py_keyword(setup_py, keyword): - with open(setup_py) as f: - try: - tree = ast.parse(f.read(), setup_py) - except SyntaxError as error: - warn(f'Could not parse {setup_py}: {error}') - return None - node = find_call_kwarg_in_ast(tree, 'setup', keyword) - return node and eval_ast_node(node, keyword) - - -def update_setup_py_keyword(setup_py, keyword, new_value): - with open(setup_py) as f: - lines = f.readlines() - new_lines = update_call_arg_in_source(lines, 'setup', keyword, new_value) - confirm_and_update_file(setup_py, lines, new_lines) - - -def confirm_and_update_file(filename, old_lines, new_lines): - print_diff(old_lines, new_lines, filename) - if new_lines != old_lines and confirm(f"Write changes to {filename}?"): - mode = stat.S_IMODE(os.stat(filename).st_mode) - tempfile = filename + '.tmp' - with open(tempfile, 'w') as f: - os.fchmod(f.fileno(), mode) - f.writelines(new_lines) - os.rename(tempfile, filename) - - -def print_diff(a, b, filename): - print(''.join(difflib.unified_diff( - a, b, - filename, filename, - "(original)", "(updated)", - ))) - - -def confirm(prompt): - while True: - try: - answer = input(f'{prompt} [y/N] ').strip().lower() - except EOFError: - answer = "" - if answer == 'y': - print() - return True - if answer == 'n' or not answer: - print() - return False - - -def to_literal(value, quote_style='"'): - # Because I don't want to deal with quoting, I'll require all values - # to contain only safe characters (i.e. no ' or " or \). Except some - # PyPI classifiers do include ' so I need to handle that at least. - safe_characters = string.ascii_letters + string.digits + " .:,-=><()/+'#" - assert all( - c in safe_characters for c in value - ), f'{value!r} has unexpected characters' - if quote_style == "'" and quote_style in value: - quote_style = '"' - assert quote_style not in value - return f'{quote_style}{value}{quote_style}' - - -def update_call_arg_in_source(source_lines, function, keyword, new_value): - lines = iter(enumerate(source_lines)) - for n, line in lines: - if line.startswith(f'{function}('): - break - else: - warn(f'Did not find {function}() call') - return source_lines - for n, line in lines: - stripped = line.lstrip() - if stripped.startswith(f'{keyword}='): - first_indent = len(line) - len(stripped) - must_fix_indents = not line.rstrip().endswith('=[') - break - else: - warn(f'Did not find {keyword}= argument in {function}() call') - return source_lines - - start = n - indent = first_indent + 4 - quote_style = '"' - for n, line in lines: - stripped = line.lstrip() - if stripped.startswith('],'): - end = n + 1 - break - elif stripped: - if not must_fix_indents: - indent = len(line) - len(stripped) - if stripped[0] in ('"', "'"): - quote_style = stripped[0] - if line.rstrip().endswith('],'): - end = n + 1 - break - else: - warn(f'Did not understand {keyword}= formatting in {function}() call') - return source_lines - - return source_lines[:start] + [ - f"{' ' * first_indent}{keyword}=[\n" - ] + [ - f"{' ' * indent}{to_literal(value, quote_style)},\n" - for value in new_value - ] + [ - f"{' ' * first_indent}],\n" - ] + source_lines[end:] - - -def find_call_kwarg_in_ast(tree, funcname, keyword, filename='setup.py'): - for node in ast.walk(tree): - if (isinstance(node, ast.Call) - and isinstance(node.func, ast.Name) - and node.func.id == funcname): - for kwarg in node.keywords: - if kwarg.arg == keyword: - return kwarg.value - else: - return None - else: - warn(f'Could not find {funcname}() call in {filename}') - return None - - -def eval_ast_node(node, keyword): - if isinstance(node, ast.Str): - return node.s - if isinstance(node, (ast.List, ast.Tuple)): - try: - return ast.literal_eval(node) - except ValueError: - pass - if (isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute) - and isinstance(node.func.value, ast.Str) - and node.func.attr == 'join'): - try: - return node.func.value.s.join(ast.literal_eval(node.args[0])) - except ValueError: - pass - warn(f'Non-literal {keyword}= passed to setup()') - return None - - -def parse_python_requires(s): - # https://www.python.org/dev/peps/pep-0440/#version-specifiers - rx = re.compile(r'^(~=|==|!=|<=|>=|<|>|===)\s*(\d+(?:\.\d+)*(?:\.\*)?)$') - - class BadConstraint(Exception): - pass - - handlers = {} - handler = partial(partial, handlers.__setitem__) - - # - # We are not doing a strict PEP-440 implementation here because if - # python_reqiures allows, say, Python 2.7.16, then we want to report that - # as Python 2.7. In each handler ``canditate`` is a two-tuple (X, Y) - # that represents any Python version between X.Y.0 and X.Y.. - # - - @handler('~=') - def compatible_version(constraint): - if len(constraint) < 2: - raise BadConstraint('~= requires a version with at least one dot') - if constraint[-1] == '*': - raise BadConstraint('~= does not allow a .*') - return lambda candidate: candidate == constraint[:2] - - @handler('==') - def matching_version(constraint): - # we know len(candidate) == 2 - if len(constraint) == 2 and constraint[-1] == '*': - return lambda candidate: candidate[0] == constraint[0] - elif len(constraint) == 1: - # == X should imply Python X.0 - return lambda candidate: candidate == constraint + (0,) - else: - # == X.Y.* and == X.Y.Z both imply Python X.Y - return lambda candidate: candidate == constraint[:2] - - @handler('!=') - def excluded_version(constraint): - # we know len(candidate) == 2 - if constraint[-1] != '*': - # != X or != X.Y or != X.Y.Z all are meaningless for us, because - # there exists some W != Z where we allow X.Y.W and thus allow - # Python X.Y. - return lambda candidate: True - elif len(constraint) == 2: - # != X.* excludes the entirety of a major version - return lambda candidate: candidate[0] != constraint[0] - else: - # != X.Y.* excludes one particular minor version X.Y, - # != X.Y.Z.* does not exclude anything, but it's fine, - # len(candidate) != len(constraint[:-1] so it'll be equivalent to - # True anyway. - return lambda candidate: candidate != constraint[:-1] - - @handler('>=') - def greater_or_equal_version(constraint): - if constraint[-1] == '*': - raise BadConstraint('>= does not allow a .*') - # >= X, >= X.Y, >= X.Y.Z all work out nicely because in Python - # (3, 0) >= (3,) - return lambda candidate: candidate >= constraint[:2] - - @handler('<=') - def lesser_or_equal_version(constraint): - if constraint[-1] == '*': - raise BadConstraint('<= does not allow a .*') - if len(constraint) == 1: - # <= X allows up to X.0 - return lambda candidate: candidate <= constraint + (0,) - else: - # <= X.Y[.Z] allows up to X.Y - return lambda candidate: candidate <= constraint - - @handler('>') - def greater_version(constraint): - if constraint[-1] == '*': - raise BadConstraint('> does not allow a .*') - if len(constraint) == 1: - # > X allows X+1.0 etc - return lambda candidate: candidate[0] > constraint[0] - elif len(constraint) == 2: - # > X.Y allows X.Y+1 etc - return lambda candidate: candidate > constraint - else: - # > X.Y.Z allows X.Y - return lambda candidate: candidate >= constraint[:2] - - @handler('<') - def lesser_version(constraint): - if constraint[-1] == '*': - raise BadConstraint('< does not allow a .*') - # < X, < X.Y, < X.Y.Z all work out nicely because in Python - # (3, 0) > (3,), (3, 0) == (3, 0) and (3, 0) < (3, 0, 1) - return lambda candidate: candidate < constraint - - @handler('===') - def arbitrary_version(constraint): - if constraint[-1] == '*': - raise BadConstraint('=== does not allow a .*') - # === X does not allow anything - # === X.Y throws me into confusion; will pip compare Python's X.Y.Z === - # X.Y and reject all possible values of Z? - # === X.Y.Z allows X.Y - return lambda candidate: candidate == constraint[:2] - - constraints = [] - for specifier in map(str.strip, s.split(',')): - m = rx.match(specifier) - if not m: - warn(f'Bad python_requires specifier: {specifier}') - continue - op, ver = m.groups() - ver = tuple( - int(segment) if segment != '*' else segment - for segment in ver.split('.') - ) - try: - constraints.append(handlers[op](ver)) - except BadConstraint as error: - warn(f'Bad python_requires specifier: {specifier} ({error})') - - if not constraints: - return None - - versions = [] - for major, max_minor in [ - (1, MAX_PYTHON_1_VERSION), - (2, MAX_PYTHON_2_VERSION), - (3, CURRENT_PYTHON_3_VERSION)]: - for minor in range(0, max_minor + 1): - if all(constraint((major, minor)) for constraint in constraints): - versions.append(f'{major}.{minor}') - return versions - - -def get_tox_ini_python_versions(filename=TOX_INI): - conf = configparser.ConfigParser() - try: - conf.read(filename) - envlist = conf.get('tox', 'envlist') - except configparser.Error: - return [] - envlist = parse_envlist(envlist) - return sorted(set( - tox_env_to_py_version(e) for e in envlist if e.startswith('py'))) - - -def parse_envlist(envlist): - envs = [] - for part in re.split(r'((?:[{][^}]*[}]|[^,{\s])+)|,|\s+', envlist): - # NB: part can be None - part = (part or '').strip() - if not part: - continue - envs += brace_expand(part) - return envs - - -def brace_expand(s): - m = re.match('^([^{]*)[{]([^}]*)[}](.*)$', s) - if not m: - return [s] - left = m.group(1) - right = m.group(3) - res = [] - for alt in m.group(2).split(','): - res += brace_expand(left + alt + right) - return res - - -def tox_env_to_py_version(env): - if '-' in env: - # e.g. py34-coverage, pypy-subunit - env = env.partition('-')[0] - if env.startswith('pypy'): - return 'PyPy' + env[4:] - elif env.startswith('py') and len(env) >= 4: - return f'{env[2]}.{env[3:]}' - else: - return env - - -def get_travis_yml_python_versions(filename=TRAVIS_YML): - with open(filename) as fp: - conf = yaml.safe_load(fp) - versions = [] - if 'python' in conf: - versions += map(travis_normalize_py_version, conf['python']) - if 'matrix' in conf and 'include' in conf['matrix']: - for job in conf['matrix']['include']: - if 'python' in job: - versions.append(travis_normalize_py_version(job['python'])) - if 'jobs' in conf and 'include' in conf['jobs']: - for job in conf['jobs']['include']: - if 'python' in job: - versions.append(travis_normalize_py_version(job['python'])) - if 'env' in conf: - toxenvs = [] - for env in conf['env']: - if env.startswith('TOXENV='): - toxenvs.extend(parse_envlist(env.partition('=')[-1])) - versions.extend( - tox_env_to_py_version(e) for e in toxenvs if e.startswith('py')) - return sorted(set(versions)) - - -def travis_normalize_py_version(v): - v = str(v) - if v.startswith('pypy3'): - # could be pypy3, pypy3.5, pypy3.5-5.10.0 - return 'PyPy3' - elif v.startswith('pypy'): - # could be pypy, pypy2, pypy2.7, pypy2.7-5.10.0 - return 'PyPy' - else: - return v - - -def get_appveyor_yml_python_versions(filename=APPVEYOR_YML): - with open(filename) as fp: - conf = yaml.safe_load(fp) - # There's more than one way of doing this, I'm setting %PYTHON% to - # the directory that has a Python interpreter (C:\PythonXY) - versions = [] - for env in conf['environment']['matrix']: - for var, value in env.items(): - if var.lower() == 'python': - versions.append(appveyor_normalize_py_version(value)) - elif var == 'TOXENV': - toxenvs = parse_envlist(value) - versions.extend( - tox_env_to_py_version(e) - for e in toxenvs if e.startswith('py')) - return sorted(set(versions)) - - -def appveyor_normalize_py_version(ver): - ver = str(ver).lower() - if ver.startswith('c:\\python'): - ver = ver[len('c:\\python'):] - if ver.endswith('\\'): - ver = ver[:-1] - if ver.endswith('-x64'): - ver = ver[:-len('-x64')] - assert len(ver) >= 2 and ver[:2].isdigit() - return f'{ver[0]}.{ver[1:]}' - - -def get_manylinux_python_versions(filename=MANYLINUX_INSTALL_SH): - magic = re.compile(r'.*\[\[ "\$\{PYBIN\}" == \*"cp(\d)(\d)"\* \]\]') - versions = [] - with open(filename) as fp: - for line in fp: - m = magic.match(line) - if m: - versions.append('{}.{}'.format(*m.groups())) - return sorted(set(versions)) - - -def important(versions): - upcoming_release = f'3.{CURRENT_PYTHON_3_VERSION + 1}' - return { - v for v in versions - if not v.startswith(('PyPy', 'Jython')) and v != 'nightly' - and not v.endswith('-dev') and v != upcoming_release - } - - -def parse_version(v): - try: - major, minor = map(int, v.split('.', 1)) - except ValueError: - raise argparse.ArgumentTypeError(f'bad version: {v}') - return (major, minor) - - -def parse_version_list(v): - versions = set() - - for part in v.split(','): - if '-' in part: - lo, hi = part.split('-', 1) - else: - lo = hi = part - - if lo and hi: - lo_major, lo_minor = parse_version(lo) - hi_major, hi_minor = parse_version(hi) - elif hi and not lo: - hi_major, hi_minor = parse_version(hi) - lo_major, lo_minor = hi_major, 0 - elif lo and not hi: - lo_major, lo_minor = parse_version(lo) - try: - hi_major, hi_minor = lo_major, MAX_MINOR_FOR_MAJOR[lo_major] - except KeyError: - raise argparse.ArgumentTypeError( - f'bad range: {part}') - else: - raise argparse.ArgumentTypeError( - f'bad range: {part}') - - if lo_major != hi_major: - raise argparse.ArgumentTypeError( - f'bad range: {part} ({lo_major} != {hi_major})') - - for v in range(lo_minor, hi_minor + 1): - versions.add(f'{lo_major}.{v}') - - return sorted(versions) - - -def update_version_list(versions, add=None, drop=None, update=None): - if update: - return sorted(update) - else: - return sorted(set(versions).union(add or ()).difference(drop or ())) - - -def is_package(where='.'): - setup_py = os.path.join(where, 'setup.py') - return os.path.exists(setup_py) - - -def check_package(where='.', *, print=print): - - if not os.path.isdir(where): - print("not a directory") - return False - - setup_py = os.path.join(where, 'setup.py') - if not os.path.exists(setup_py): - print("no setup.py -- not a Python package?") - return False - - return True - - -def check_versions(where='.', *, print=print, expect=None): - - sources = [ - ('setup.py', get_supported_python_versions, None), - ('- python_requires', get_python_requires, 'setup.py'), - (TOX_INI, get_tox_ini_python_versions, TOX_INI), - (TRAVIS_YML, get_travis_yml_python_versions, TRAVIS_YML), - (APPVEYOR_YML, get_appveyor_yml_python_versions, APPVEYOR_YML), - (MANYLINUX_INSTALL_SH, get_manylinux_python_versions, - MANYLINUX_INSTALL_SH), - ] - - width = max(len(title) for title, *etc in sources) + len(" says:") - - version_sets = [] - - for (title, extractor, filename) in sources: - arg = os.path.join(where, filename) if filename else where - if not os.path.exists(arg): - continue - versions = extractor(arg) - if versions is None: - continue - print(f"{title} says:".ljust(width), ", ".join(versions) or "(empty)") - version_sets.append(important(versions)) - - if not expect: - expect = version_sets[0] - else: - print("expected:".ljust(width), ', '.join(expect)) - - expect = important(expect) - return all( - expect == v for v in version_sets - ) - - -def update_versions(where='.', *, add=None, drop=None, update=None): - - versions = get_supported_python_versions(where) - if versions is None: - return - - versions = sorted(important(versions)) - new_versions = update_version_list( - versions, add=add, drop=drop, update=update) - if versions != new_versions: - update_supported_python_versions(where, new_versions) - - -def _main(): - parser = argparse.ArgumentParser( - description="verify that supported Python versions are the same" - " in setup.py, tox.ini, .travis.yml and appveyor.yml") - parser.add_argument('--version', action='version', - version="%(prog)s version " + __version__) - parser.add_argument('--expect', metavar='VERSIONS', - type=parse_version_list, - help='expect these versions to be supported, e.g.' - ' --expect 2.7,3.5-3.7') - parser.add_argument('--skip-non-packages', action='store_true', - help='skip arguments that are not Python packages' - ' without warning about them') - parser.add_argument('where', nargs='*', - help='directory where a Python package with a setup.py' - ' and other files is located') - group = parser.add_argument_group( - "updating supported version lists (EXPERIMENTAL)") - group.add_argument('--add', metavar='VERSIONS', type=parse_version_list, - help='add these versions to supported ones, e.g' - ' --add 3.8') - group.add_argument('--drop', metavar='VERSIONS', type=parse_version_list, - help='drop these versions from supported ones, e.g' - ' --drop 2.6,3.4') - group.add_argument('--update', metavar='VERSIONS', type=parse_version_list, - help='update the set of supported versions, e.g.' - ' --update 2.7,3.5-3.7') - args = parser.parse_args() - - if args.update and args.add: - parser.error("argument --add: not allowed with argument --update") - if args.update and args.drop: - parser.error("argument --drop: not allowed with argument --update") - - where = args.where or ['.'] - if args.skip_non_packages: - where = [path for path in where if is_package(path)] - - multiple = len(where) > 1 - mismatches = [] - for n, path in enumerate(where): - if multiple: - if n: - print("\n") - print(f"{path}:\n") - if not check_package(path): - mismatches.append(path) - continue - if args.add or args.drop or args.update: - update_versions(path, add=args.add, drop=args.drop, - update=args.update) - if not check_versions(path, expect=args.expect): - mismatches.append(path) - continue - - if mismatches: - if multiple: - sys.exit(f"\n\nmismatch in {' '.join(mismatches)}!") - else: - sys.exit("\nmismatch!") - elif multiple: - print("\n\nall ok!") - - -def main(): - try: - _main() - except KeyboardInterrupt: - sys.exit(2) diff --git a/src/check_python_versions/cli.py b/src/check_python_versions/cli.py new file mode 100755 index 0000000..3ba4df6 --- /dev/null +++ b/src/check_python_versions/cli.py @@ -0,0 +1,752 @@ +import argparse +import ast +import configparser +import difflib +import logging +import os +import re +import stat +import string +import subprocess +import sys +from functools import partial + +from . import __version__ + + +try: + import yaml +except ImportError: # pragma: nocover + # Shouldn't happen, we install_requires=['PyYAML'], but maybe someone is + # running ./check-python-versions directly from a git checkout. + yaml = None + print("PyYAML is needed for Travis CI/Appveyor support" + " (apt install python3-yaml)") + + +log = logging.getLogger('check-python-versions') + + +TOX_INI = 'tox.ini' +TRAVIS_YML = '.travis.yml' +APPVEYOR_YML = 'appveyor.yml' +MANYLINUX_INSTALL_SH = '.manylinux-install.sh' + + +MAX_PYTHON_1_VERSION = 6 # i.e. 1.6 +MAX_PYTHON_2_VERSION = 7 # i.e. 2.7 +CURRENT_PYTHON_3_VERSION = 7 # i.e. 3.7 + +MAX_MINOR_FOR_MAJOR = { + 1: MAX_PYTHON_1_VERSION, + 2: MAX_PYTHON_2_VERSION, + 3: CURRENT_PYTHON_3_VERSION, +} + + +def warn(msg): + print(msg, file=sys.stderr) + + +def pipe(*cmd, **kwargs): + if 'cwd' in kwargs: + log.debug('EXEC cd %s && %s', kwargs['cwd'], ' '.join(cmd)) + else: + log.debug('EXEC %s', ' '.join(cmd)) + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, **kwargs) + return p.communicate()[0].decode('UTF-8', 'replace') + + +def get_supported_python_versions(repo_path='.'): + setup_py = os.path.join(repo_path, 'setup.py') + classifiers = get_setup_py_keyword(setup_py, 'classifiers') + if classifiers is None: + # AST parsing is complicated + classifiers = pipe("python", "setup.py", "-q", "--classifiers", + cwd=repo_path).splitlines() + return get_versions_from_classifiers(classifiers) + + +def is_version_classifier(s): + prefix = 'Programming Language :: Python :: ' + return s.startswith(prefix) and s[len(prefix):len(prefix) + 1].isdigit() + + +def is_major_version_classifier(s): + prefix = 'Programming Language :: Python :: ' + return ( + s.startswith(prefix) + and s[len(prefix):].replace(' :: Only', '').isdigit() + ) + + +def get_versions_from_classifiers(classifiers): + # Based on + # https://github.com/mgedmin/project-summary/blob/master/summary.py#L221-L234 + prefix = 'Programming Language :: Python :: ' + impl_prefix = 'Programming Language :: Python :: Implementation :: ' + cpython = impl_prefix + 'CPython' + versions = { + s[len(prefix):].replace(' :: Only', '').rstrip() + for s in classifiers + if is_version_classifier(s) + } | { + s[len(impl_prefix):].rstrip() + for s in classifiers + if s.startswith(impl_prefix) and s != cpython + } + for major in '2', '3': + if major in versions and any( + v.startswith(f'{major}.') for v in versions): + versions.remove(major) + return sorted(versions) + + +def update_classifiers(classifiers, new_versions): + prefix = 'Programming Language :: Python :: ' + + for pos, s in enumerate(classifiers): + if is_version_classifier(s): + break + else: + pos = len(classifiers) + + if any(map(is_major_version_classifier, classifiers)): + new_versions = sorted( + set(new_versions).union( + v.partition('.')[0] for v in new_versions + ) + ) + + classifiers = [ + s for s in classifiers if not is_version_classifier(s) + ] + new_classifiers = [ + f'{prefix}{version}' + for version in new_versions + ] + classifiers[pos:pos] = new_classifiers + return classifiers + + +def update_supported_python_versions(repo_path, new_versions): + setup_py = os.path.join(repo_path, 'setup.py') + classifiers = get_setup_py_keyword(setup_py, 'classifiers') + if classifiers is None: + return + new_classifiers = update_classifiers(classifiers, new_versions) + update_setup_py_keyword(setup_py, 'classifiers', new_classifiers) + + +def get_python_requires(setup_py='setup.py'): + python_requires = get_setup_py_keyword(setup_py, 'python_requires') + if python_requires is None: + return None + return parse_python_requires(python_requires) + + +def get_setup_py_keyword(setup_py, keyword): + with open(setup_py) as f: + try: + tree = ast.parse(f.read(), setup_py) + except SyntaxError as error: + warn(f'Could not parse {setup_py}: {error}') + return None + node = find_call_kwarg_in_ast(tree, 'setup', keyword) + return node and eval_ast_node(node, keyword) + + +def update_setup_py_keyword(setup_py, keyword, new_value): + with open(setup_py) as f: + lines = f.readlines() + new_lines = update_call_arg_in_source(lines, 'setup', keyword, new_value) + confirm_and_update_file(setup_py, lines, new_lines) + + +def confirm_and_update_file(filename, old_lines, new_lines): + print_diff(old_lines, new_lines, filename) + if new_lines != old_lines and confirm(f"Write changes to {filename}?"): + mode = stat.S_IMODE(os.stat(filename).st_mode) + tempfile = filename + '.tmp' + with open(tempfile, 'w') as f: + os.fchmod(f.fileno(), mode) + f.writelines(new_lines) + os.rename(tempfile, filename) + + +def print_diff(a, b, filename): + print(''.join(difflib.unified_diff( + a, b, + filename, filename, + "(original)", "(updated)", + ))) + + +def confirm(prompt): + while True: + try: + answer = input(f'{prompt} [y/N] ').strip().lower() + except EOFError: + answer = "" + if answer == 'y': + print() + return True + if answer == 'n' or not answer: + print() + return False + + +def to_literal(value, quote_style='"'): + # Because I don't want to deal with quoting, I'll require all values + # to contain only safe characters (i.e. no ' or " or \). Except some + # PyPI classifiers do include ' so I need to handle that at least. + safe_characters = string.ascii_letters + string.digits + " .:,-=><()/+'#" + assert all( + c in safe_characters for c in value + ), f'{value!r} has unexpected characters' + if quote_style == "'" and quote_style in value: + quote_style = '"' + assert quote_style not in value + return f'{quote_style}{value}{quote_style}' + + +def update_call_arg_in_source(source_lines, function, keyword, new_value): + lines = iter(enumerate(source_lines)) + for n, line in lines: + if line.startswith(f'{function}('): + break + else: + warn(f'Did not find {function}() call') + return source_lines + for n, line in lines: + stripped = line.lstrip() + if stripped.startswith(f'{keyword}='): + first_indent = len(line) - len(stripped) + must_fix_indents = not line.rstrip().endswith('=[') + break + else: + warn(f'Did not find {keyword}= argument in {function}() call') + return source_lines + + start = n + indent = first_indent + 4 + quote_style = '"' + for n, line in lines: + stripped = line.lstrip() + if stripped.startswith('],'): + end = n + 1 + break + elif stripped: + if not must_fix_indents: + indent = len(line) - len(stripped) + if stripped[0] in ('"', "'"): + quote_style = stripped[0] + if line.rstrip().endswith('],'): + end = n + 1 + break + else: + warn(f'Did not understand {keyword}= formatting in {function}() call') + return source_lines + + return source_lines[:start] + [ + f"{' ' * first_indent}{keyword}=[\n" + ] + [ + f"{' ' * indent}{to_literal(value, quote_style)},\n" + for value in new_value + ] + [ + f"{' ' * first_indent}],\n" + ] + source_lines[end:] + + +def find_call_kwarg_in_ast(tree, funcname, keyword, filename='setup.py'): + for node in ast.walk(tree): + if (isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == funcname): + for kwarg in node.keywords: + if kwarg.arg == keyword: + return kwarg.value + else: + return None + else: + warn(f'Could not find {funcname}() call in {filename}') + return None + + +def eval_ast_node(node, keyword): + if isinstance(node, ast.Str): + return node.s + if isinstance(node, (ast.List, ast.Tuple)): + try: + return ast.literal_eval(node) + except ValueError: + pass + if (isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute) + and isinstance(node.func.value, ast.Str) + and node.func.attr == 'join'): + try: + return node.func.value.s.join(ast.literal_eval(node.args[0])) + except ValueError: + pass + warn(f'Non-literal {keyword}= passed to setup()') + return None + + +def parse_python_requires(s): + # https://www.python.org/dev/peps/pep-0440/#version-specifiers + rx = re.compile(r'^(~=|==|!=|<=|>=|<|>|===)\s*(\d+(?:\.\d+)*(?:\.\*)?)$') + + class BadConstraint(Exception): + pass + + handlers = {} + handler = partial(partial, handlers.__setitem__) + + # + # We are not doing a strict PEP-440 implementation here because if + # python_reqiures allows, say, Python 2.7.16, then we want to report that + # as Python 2.7. In each handler ``canditate`` is a two-tuple (X, Y) + # that represents any Python version between X.Y.0 and X.Y.. + # + + @handler('~=') + def compatible_version(constraint): + if len(constraint) < 2: + raise BadConstraint('~= requires a version with at least one dot') + if constraint[-1] == '*': + raise BadConstraint('~= does not allow a .*') + return lambda candidate: candidate == constraint[:2] + + @handler('==') + def matching_version(constraint): + # we know len(candidate) == 2 + if len(constraint) == 2 and constraint[-1] == '*': + return lambda candidate: candidate[0] == constraint[0] + elif len(constraint) == 1: + # == X should imply Python X.0 + return lambda candidate: candidate == constraint + (0,) + else: + # == X.Y.* and == X.Y.Z both imply Python X.Y + return lambda candidate: candidate == constraint[:2] + + @handler('!=') + def excluded_version(constraint): + # we know len(candidate) == 2 + if constraint[-1] != '*': + # != X or != X.Y or != X.Y.Z all are meaningless for us, because + # there exists some W != Z where we allow X.Y.W and thus allow + # Python X.Y. + return lambda candidate: True + elif len(constraint) == 2: + # != X.* excludes the entirety of a major version + return lambda candidate: candidate[0] != constraint[0] + else: + # != X.Y.* excludes one particular minor version X.Y, + # != X.Y.Z.* does not exclude anything, but it's fine, + # len(candidate) != len(constraint[:-1] so it'll be equivalent to + # True anyway. + return lambda candidate: candidate != constraint[:-1] + + @handler('>=') + def greater_or_equal_version(constraint): + if constraint[-1] == '*': + raise BadConstraint('>= does not allow a .*') + # >= X, >= X.Y, >= X.Y.Z all work out nicely because in Python + # (3, 0) >= (3,) + return lambda candidate: candidate >= constraint[:2] + + @handler('<=') + def lesser_or_equal_version(constraint): + if constraint[-1] == '*': + raise BadConstraint('<= does not allow a .*') + if len(constraint) == 1: + # <= X allows up to X.0 + return lambda candidate: candidate <= constraint + (0,) + else: + # <= X.Y[.Z] allows up to X.Y + return lambda candidate: candidate <= constraint + + @handler('>') + def greater_version(constraint): + if constraint[-1] == '*': + raise BadConstraint('> does not allow a .*') + if len(constraint) == 1: + # > X allows X+1.0 etc + return lambda candidate: candidate[0] > constraint[0] + elif len(constraint) == 2: + # > X.Y allows X.Y+1 etc + return lambda candidate: candidate > constraint + else: + # > X.Y.Z allows X.Y + return lambda candidate: candidate >= constraint[:2] + + @handler('<') + def lesser_version(constraint): + if constraint[-1] == '*': + raise BadConstraint('< does not allow a .*') + # < X, < X.Y, < X.Y.Z all work out nicely because in Python + # (3, 0) > (3,), (3, 0) == (3, 0) and (3, 0) < (3, 0, 1) + return lambda candidate: candidate < constraint + + @handler('===') + def arbitrary_version(constraint): + if constraint[-1] == '*': + raise BadConstraint('=== does not allow a .*') + # === X does not allow anything + # === X.Y throws me into confusion; will pip compare Python's X.Y.Z === + # X.Y and reject all possible values of Z? + # === X.Y.Z allows X.Y + return lambda candidate: candidate == constraint[:2] + + constraints = [] + for specifier in map(str.strip, s.split(',')): + m = rx.match(specifier) + if not m: + warn(f'Bad python_requires specifier: {specifier}') + continue + op, ver = m.groups() + ver = tuple( + int(segment) if segment != '*' else segment + for segment in ver.split('.') + ) + try: + constraints.append(handlers[op](ver)) + except BadConstraint as error: + warn(f'Bad python_requires specifier: {specifier} ({error})') + + if not constraints: + return None + + versions = [] + for major, max_minor in [ + (1, MAX_PYTHON_1_VERSION), + (2, MAX_PYTHON_2_VERSION), + (3, CURRENT_PYTHON_3_VERSION)]: + for minor in range(0, max_minor + 1): + if all(constraint((major, minor)) for constraint in constraints): + versions.append(f'{major}.{minor}') + return versions + + +def get_tox_ini_python_versions(filename=TOX_INI): + conf = configparser.ConfigParser() + try: + conf.read(filename) + envlist = conf.get('tox', 'envlist') + except configparser.Error: + return [] + envlist = parse_envlist(envlist) + return sorted(set( + tox_env_to_py_version(e) for e in envlist if e.startswith('py'))) + + +def parse_envlist(envlist): + envs = [] + for part in re.split(r'((?:[{][^}]*[}]|[^,{\s])+)|,|\s+', envlist): + # NB: part can be None + part = (part or '').strip() + if not part: + continue + envs += brace_expand(part) + return envs + + +def brace_expand(s): + m = re.match('^([^{]*)[{]([^}]*)[}](.*)$', s) + if not m: + return [s] + left = m.group(1) + right = m.group(3) + res = [] + for alt in m.group(2).split(','): + res += brace_expand(left + alt + right) + return res + + +def tox_env_to_py_version(env): + if '-' in env: + # e.g. py34-coverage, pypy-subunit + env = env.partition('-')[0] + if env.startswith('pypy'): + return 'PyPy' + env[4:] + elif env.startswith('py') and len(env) >= 4: + return f'{env[2]}.{env[3:]}' + else: + return env + + +def get_travis_yml_python_versions(filename=TRAVIS_YML): + with open(filename) as fp: + conf = yaml.safe_load(fp) + versions = [] + if 'python' in conf: + versions += map(travis_normalize_py_version, conf['python']) + if 'matrix' in conf and 'include' in conf['matrix']: + for job in conf['matrix']['include']: + if 'python' in job: + versions.append(travis_normalize_py_version(job['python'])) + if 'jobs' in conf and 'include' in conf['jobs']: + for job in conf['jobs']['include']: + if 'python' in job: + versions.append(travis_normalize_py_version(job['python'])) + if 'env' in conf: + toxenvs = [] + for env in conf['env']: + if env.startswith('TOXENV='): + toxenvs.extend(parse_envlist(env.partition('=')[-1])) + versions.extend( + tox_env_to_py_version(e) for e in toxenvs if e.startswith('py')) + return sorted(set(versions)) + + +def travis_normalize_py_version(v): + v = str(v) + if v.startswith('pypy3'): + # could be pypy3, pypy3.5, pypy3.5-5.10.0 + return 'PyPy3' + elif v.startswith('pypy'): + # could be pypy, pypy2, pypy2.7, pypy2.7-5.10.0 + return 'PyPy' + else: + return v + + +def get_appveyor_yml_python_versions(filename=APPVEYOR_YML): + with open(filename) as fp: + conf = yaml.safe_load(fp) + # There's more than one way of doing this, I'm setting %PYTHON% to + # the directory that has a Python interpreter (C:\PythonXY) + versions = [] + for env in conf['environment']['matrix']: + for var, value in env.items(): + if var.lower() == 'python': + versions.append(appveyor_normalize_py_version(value)) + elif var == 'TOXENV': + toxenvs = parse_envlist(value) + versions.extend( + tox_env_to_py_version(e) + for e in toxenvs if e.startswith('py')) + return sorted(set(versions)) + + +def appveyor_normalize_py_version(ver): + ver = str(ver).lower() + if ver.startswith('c:\\python'): + ver = ver[len('c:\\python'):] + if ver.endswith('\\'): + ver = ver[:-1] + if ver.endswith('-x64'): + ver = ver[:-len('-x64')] + assert len(ver) >= 2 and ver[:2].isdigit() + return f'{ver[0]}.{ver[1:]}' + + +def get_manylinux_python_versions(filename=MANYLINUX_INSTALL_SH): + magic = re.compile(r'.*\[\[ "\$\{PYBIN\}" == \*"cp(\d)(\d)"\* \]\]') + versions = [] + with open(filename) as fp: + for line in fp: + m = magic.match(line) + if m: + versions.append('{}.{}'.format(*m.groups())) + return sorted(set(versions)) + + +def important(versions): + upcoming_release = f'3.{CURRENT_PYTHON_3_VERSION + 1}' + return { + v for v in versions + if not v.startswith(('PyPy', 'Jython')) and v != 'nightly' + and not v.endswith('-dev') and v != upcoming_release + } + + +def parse_version(v): + try: + major, minor = map(int, v.split('.', 1)) + except ValueError: + raise argparse.ArgumentTypeError(f'bad version: {v}') + return (major, minor) + + +def parse_version_list(v): + versions = set() + + for part in v.split(','): + if '-' in part: + lo, hi = part.split('-', 1) + else: + lo = hi = part + + if lo and hi: + lo_major, lo_minor = parse_version(lo) + hi_major, hi_minor = parse_version(hi) + elif hi and not lo: + hi_major, hi_minor = parse_version(hi) + lo_major, lo_minor = hi_major, 0 + elif lo and not hi: + lo_major, lo_minor = parse_version(lo) + try: + hi_major, hi_minor = lo_major, MAX_MINOR_FOR_MAJOR[lo_major] + except KeyError: + raise argparse.ArgumentTypeError( + f'bad range: {part}') + else: + raise argparse.ArgumentTypeError( + f'bad range: {part}') + + if lo_major != hi_major: + raise argparse.ArgumentTypeError( + f'bad range: {part} ({lo_major} != {hi_major})') + + for v in range(lo_minor, hi_minor + 1): + versions.add(f'{lo_major}.{v}') + + return sorted(versions) + + +def update_version_list(versions, add=None, drop=None, update=None): + if update: + return sorted(update) + else: + return sorted(set(versions).union(add or ()).difference(drop or ())) + + +def is_package(where='.'): + setup_py = os.path.join(where, 'setup.py') + return os.path.exists(setup_py) + + +def check_package(where='.', *, print=print): + + if not os.path.isdir(where): + print("not a directory") + return False + + setup_py = os.path.join(where, 'setup.py') + if not os.path.exists(setup_py): + print("no setup.py -- not a Python package?") + return False + + return True + + +def check_versions(where='.', *, print=print, expect=None): + + sources = [ + ('setup.py', get_supported_python_versions, None), + ('- python_requires', get_python_requires, 'setup.py'), + (TOX_INI, get_tox_ini_python_versions, TOX_INI), + (TRAVIS_YML, get_travis_yml_python_versions, TRAVIS_YML), + (APPVEYOR_YML, get_appveyor_yml_python_versions, APPVEYOR_YML), + (MANYLINUX_INSTALL_SH, get_manylinux_python_versions, + MANYLINUX_INSTALL_SH), + ] + + width = max(len(title) for title, *etc in sources) + len(" says:") + + version_sets = [] + + for (title, extractor, filename) in sources: + arg = os.path.join(where, filename) if filename else where + if not os.path.exists(arg): + continue + versions = extractor(arg) + if versions is None: + continue + print(f"{title} says:".ljust(width), ", ".join(versions) or "(empty)") + version_sets.append(important(versions)) + + if not expect: + expect = version_sets[0] + else: + print("expected:".ljust(width), ', '.join(expect)) + + expect = important(expect) + return all( + expect == v for v in version_sets + ) + + +def update_versions(where='.', *, add=None, drop=None, update=None): + + versions = get_supported_python_versions(where) + if versions is None: + return + + versions = sorted(important(versions)) + new_versions = update_version_list( + versions, add=add, drop=drop, update=update) + if versions != new_versions: + update_supported_python_versions(where, new_versions) + + +def _main(): + parser = argparse.ArgumentParser( + description="verify that supported Python versions are the same" + " in setup.py, tox.ini, .travis.yml and appveyor.yml") + parser.add_argument('--version', action='version', + version="%(prog)s version " + __version__) + parser.add_argument('--expect', metavar='VERSIONS', + type=parse_version_list, + help='expect these versions to be supported, e.g.' + ' --expect 2.7,3.5-3.7') + parser.add_argument('--skip-non-packages', action='store_true', + help='skip arguments that are not Python packages' + ' without warning about them') + parser.add_argument('where', nargs='*', + help='directory where a Python package with a setup.py' + ' and other files is located') + group = parser.add_argument_group( + "updating supported version lists (EXPERIMENTAL)") + group.add_argument('--add', metavar='VERSIONS', type=parse_version_list, + help='add these versions to supported ones, e.g' + ' --add 3.8') + group.add_argument('--drop', metavar='VERSIONS', type=parse_version_list, + help='drop these versions from supported ones, e.g' + ' --drop 2.6,3.4') + group.add_argument('--update', metavar='VERSIONS', type=parse_version_list, + help='update the set of supported versions, e.g.' + ' --update 2.7,3.5-3.7') + args = parser.parse_args() + + if args.update and args.add: + parser.error("argument --add: not allowed with argument --update") + if args.update and args.drop: + parser.error("argument --drop: not allowed with argument --update") + + where = args.where or ['.'] + if args.skip_non_packages: + where = [path for path in where if is_package(path)] + + multiple = len(where) > 1 + mismatches = [] + for n, path in enumerate(where): + if multiple: + if n: + print("\n") + print(f"{path}:\n") + if not check_package(path): + mismatches.append(path) + continue + if args.add or args.drop or args.update: + update_versions(path, add=args.add, drop=args.drop, + update=args.update) + if not check_versions(path, expect=args.expect): + mismatches.append(path) + continue + + if mismatches: + if multiple: + sys.exit(f"\n\nmismatch in {' '.join(mismatches)}!") + else: + sys.exit("\nmismatch!") + elif multiple: + print("\n\nall ok!") + + +def main(): + try: + _main() + except KeyboardInterrupt: + sys.exit(2) diff --git a/tests.py b/tests.py index 6ab68a0..b23276e 100644 --- a/tests.py +++ b/tests.py @@ -7,7 +7,7 @@ import pytest -import check_python_versions as cpv +import check_python_versions.cli as cpv needs_pyyaml = pytest.mark.skipIf(cpv.yaml is None, "PyYAML not installed") From a97a6ee4edd6ebbbaa58eb8cf6888aa3e0794a32 Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Sat, 13 Apr 2019 20:40:07 +0300 Subject: [PATCH 18/76] Allow python -m check_python_versions --- src/check_python_versions/__main__.py | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 src/check_python_versions/__main__.py diff --git a/src/check_python_versions/__main__.py b/src/check_python_versions/__main__.py new file mode 100644 index 0000000..71b440f --- /dev/null +++ b/src/check_python_versions/__main__.py @@ -0,0 +1,4 @@ +from .cli import main + +if __name__ == '__main__': + main() From 0150cd5af0befc726077c56454654ccf2c8af6bf Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Sat, 13 Apr 2019 21:16:30 +0300 Subject: [PATCH 19/76] Add 'make flake8' --- Makefile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Makefile b/Makefile index 9c670ed..06398c8 100644 --- a/Makefile +++ b/Makefile @@ -14,4 +14,8 @@ coverage: tox -e coverage +.PHONY: flake8 +flake8: + flake8 src setup.py + include release.mk From 1f9610abbfd3907da6660f8d1fb1a4edcae82189 Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Sat, 13 Apr 2019 21:42:38 +0300 Subject: [PATCH 20/76] Split cli.py and tests.py into several modules --- .gitignore | 1 + .travis.yml | 2 +- pytest.ini | 2 + src/check_python_versions/cli.py | 580 +---------- src/check_python_versions/parsers/__init__.py | 1 + src/check_python_versions/parsers/appveyor.py | 39 + .../parsers/manylinux.py | 14 + src/check_python_versions/parsers/python.py | 343 +++++++ src/check_python_versions/parsers/tox.py | 52 + src/check_python_versions/parsers/travis.py | 45 + src/check_python_versions/utils.py | 55 + src/check_python_versions/versions.py | 25 + tests.py | 969 ------------------ tests/conftest.py | 10 + tests/parsers/test_appveyor.py | 58 ++ tests/parsers/test_manylinux.py | 35 + tests/parsers/test_python.py | 493 +++++++++ tests/parsers/test_tox.py | 83 ++ tests/parsers/test_travis.py | 70 ++ tests/test___main__.py | 11 + tests/test_cli.py | 259 +++++ tests/test_utils.py | 5 + tests/test_versions.py | 30 + tox.ini | 7 +- 24 files changed, 1662 insertions(+), 1527 deletions(-) create mode 100644 pytest.ini mode change 100755 => 100644 src/check_python_versions/cli.py create mode 100644 src/check_python_versions/parsers/__init__.py create mode 100644 src/check_python_versions/parsers/appveyor.py create mode 100644 src/check_python_versions/parsers/manylinux.py create mode 100644 src/check_python_versions/parsers/python.py create mode 100644 src/check_python_versions/parsers/tox.py create mode 100644 src/check_python_versions/parsers/travis.py create mode 100644 src/check_python_versions/utils.py create mode 100644 src/check_python_versions/versions.py delete mode 100644 tests.py create mode 100644 tests/conftest.py create mode 100644 tests/parsers/test_appveyor.py create mode 100644 tests/parsers/test_manylinux.py create mode 100644 tests/parsers/test_python.py create mode 100644 tests/parsers/test_tox.py create mode 100644 tests/parsers/test_travis.py create mode 100644 tests/test___main__.py create mode 100644 tests/test_cli.py create mode 100644 tests/test_utils.py create mode 100644 tests/test_versions.py diff --git a/.gitignore b/.gitignore index 3993d43..90944dd 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ tmp/ .tox/ .coverage __pycache__/ +tags diff --git a/.travis.yml b/.travis.yml index df0597f..ebdb765 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,7 @@ install: - pip install pytest coverage coveralls flake8 - pip install -e . script: - - coverage run -m pytest tests.py + - coverage run -m pytest tests - coverage report -m --fail-under=100 - flake8 *.py after_script: diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..3591f3c --- /dev/null +++ b/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +addopts = -ra diff --git a/src/check_python_versions/cli.py b/src/check_python_versions/cli.py old mode 100755 new mode 100644 index 3ba4df6..4941406 --- a/src/check_python_versions/cli.py +++ b/src/check_python_versions/cli.py @@ -1,18 +1,34 @@ import argparse -import ast -import configparser -import difflib -import logging import os -import re -import stat -import string -import subprocess import sys -from functools import partial from . import __version__ - +from .versions import ( + MAX_MINOR_FOR_MAJOR, + important, + update_version_list, +) +from .parsers.python import ( + get_supported_python_versions, + get_python_requires, + update_supported_python_versions, +) +from .parsers.tox import ( + TOX_INI, + get_tox_ini_python_versions, +) +from .parsers.travis import ( + TRAVIS_YML, + get_travis_yml_python_versions, +) +from .parsers.appveyor import ( + APPVEYOR_YML, + get_appveyor_yml_python_versions, +) +from .parsers.manylinux import ( + MANYLINUX_INSTALL_SH, + get_manylinux_python_versions, +) try: import yaml @@ -24,543 +40,6 @@ " (apt install python3-yaml)") -log = logging.getLogger('check-python-versions') - - -TOX_INI = 'tox.ini' -TRAVIS_YML = '.travis.yml' -APPVEYOR_YML = 'appveyor.yml' -MANYLINUX_INSTALL_SH = '.manylinux-install.sh' - - -MAX_PYTHON_1_VERSION = 6 # i.e. 1.6 -MAX_PYTHON_2_VERSION = 7 # i.e. 2.7 -CURRENT_PYTHON_3_VERSION = 7 # i.e. 3.7 - -MAX_MINOR_FOR_MAJOR = { - 1: MAX_PYTHON_1_VERSION, - 2: MAX_PYTHON_2_VERSION, - 3: CURRENT_PYTHON_3_VERSION, -} - - -def warn(msg): - print(msg, file=sys.stderr) - - -def pipe(*cmd, **kwargs): - if 'cwd' in kwargs: - log.debug('EXEC cd %s && %s', kwargs['cwd'], ' '.join(cmd)) - else: - log.debug('EXEC %s', ' '.join(cmd)) - p = subprocess.Popen(cmd, stdout=subprocess.PIPE, **kwargs) - return p.communicate()[0].decode('UTF-8', 'replace') - - -def get_supported_python_versions(repo_path='.'): - setup_py = os.path.join(repo_path, 'setup.py') - classifiers = get_setup_py_keyword(setup_py, 'classifiers') - if classifiers is None: - # AST parsing is complicated - classifiers = pipe("python", "setup.py", "-q", "--classifiers", - cwd=repo_path).splitlines() - return get_versions_from_classifiers(classifiers) - - -def is_version_classifier(s): - prefix = 'Programming Language :: Python :: ' - return s.startswith(prefix) and s[len(prefix):len(prefix) + 1].isdigit() - - -def is_major_version_classifier(s): - prefix = 'Programming Language :: Python :: ' - return ( - s.startswith(prefix) - and s[len(prefix):].replace(' :: Only', '').isdigit() - ) - - -def get_versions_from_classifiers(classifiers): - # Based on - # https://github.com/mgedmin/project-summary/blob/master/summary.py#L221-L234 - prefix = 'Programming Language :: Python :: ' - impl_prefix = 'Programming Language :: Python :: Implementation :: ' - cpython = impl_prefix + 'CPython' - versions = { - s[len(prefix):].replace(' :: Only', '').rstrip() - for s in classifiers - if is_version_classifier(s) - } | { - s[len(impl_prefix):].rstrip() - for s in classifiers - if s.startswith(impl_prefix) and s != cpython - } - for major in '2', '3': - if major in versions and any( - v.startswith(f'{major}.') for v in versions): - versions.remove(major) - return sorted(versions) - - -def update_classifiers(classifiers, new_versions): - prefix = 'Programming Language :: Python :: ' - - for pos, s in enumerate(classifiers): - if is_version_classifier(s): - break - else: - pos = len(classifiers) - - if any(map(is_major_version_classifier, classifiers)): - new_versions = sorted( - set(new_versions).union( - v.partition('.')[0] for v in new_versions - ) - ) - - classifiers = [ - s for s in classifiers if not is_version_classifier(s) - ] - new_classifiers = [ - f'{prefix}{version}' - for version in new_versions - ] - classifiers[pos:pos] = new_classifiers - return classifiers - - -def update_supported_python_versions(repo_path, new_versions): - setup_py = os.path.join(repo_path, 'setup.py') - classifiers = get_setup_py_keyword(setup_py, 'classifiers') - if classifiers is None: - return - new_classifiers = update_classifiers(classifiers, new_versions) - update_setup_py_keyword(setup_py, 'classifiers', new_classifiers) - - -def get_python_requires(setup_py='setup.py'): - python_requires = get_setup_py_keyword(setup_py, 'python_requires') - if python_requires is None: - return None - return parse_python_requires(python_requires) - - -def get_setup_py_keyword(setup_py, keyword): - with open(setup_py) as f: - try: - tree = ast.parse(f.read(), setup_py) - except SyntaxError as error: - warn(f'Could not parse {setup_py}: {error}') - return None - node = find_call_kwarg_in_ast(tree, 'setup', keyword) - return node and eval_ast_node(node, keyword) - - -def update_setup_py_keyword(setup_py, keyword, new_value): - with open(setup_py) as f: - lines = f.readlines() - new_lines = update_call_arg_in_source(lines, 'setup', keyword, new_value) - confirm_and_update_file(setup_py, lines, new_lines) - - -def confirm_and_update_file(filename, old_lines, new_lines): - print_diff(old_lines, new_lines, filename) - if new_lines != old_lines and confirm(f"Write changes to {filename}?"): - mode = stat.S_IMODE(os.stat(filename).st_mode) - tempfile = filename + '.tmp' - with open(tempfile, 'w') as f: - os.fchmod(f.fileno(), mode) - f.writelines(new_lines) - os.rename(tempfile, filename) - - -def print_diff(a, b, filename): - print(''.join(difflib.unified_diff( - a, b, - filename, filename, - "(original)", "(updated)", - ))) - - -def confirm(prompt): - while True: - try: - answer = input(f'{prompt} [y/N] ').strip().lower() - except EOFError: - answer = "" - if answer == 'y': - print() - return True - if answer == 'n' or not answer: - print() - return False - - -def to_literal(value, quote_style='"'): - # Because I don't want to deal with quoting, I'll require all values - # to contain only safe characters (i.e. no ' or " or \). Except some - # PyPI classifiers do include ' so I need to handle that at least. - safe_characters = string.ascii_letters + string.digits + " .:,-=><()/+'#" - assert all( - c in safe_characters for c in value - ), f'{value!r} has unexpected characters' - if quote_style == "'" and quote_style in value: - quote_style = '"' - assert quote_style not in value - return f'{quote_style}{value}{quote_style}' - - -def update_call_arg_in_source(source_lines, function, keyword, new_value): - lines = iter(enumerate(source_lines)) - for n, line in lines: - if line.startswith(f'{function}('): - break - else: - warn(f'Did not find {function}() call') - return source_lines - for n, line in lines: - stripped = line.lstrip() - if stripped.startswith(f'{keyword}='): - first_indent = len(line) - len(stripped) - must_fix_indents = not line.rstrip().endswith('=[') - break - else: - warn(f'Did not find {keyword}= argument in {function}() call') - return source_lines - - start = n - indent = first_indent + 4 - quote_style = '"' - for n, line in lines: - stripped = line.lstrip() - if stripped.startswith('],'): - end = n + 1 - break - elif stripped: - if not must_fix_indents: - indent = len(line) - len(stripped) - if stripped[0] in ('"', "'"): - quote_style = stripped[0] - if line.rstrip().endswith('],'): - end = n + 1 - break - else: - warn(f'Did not understand {keyword}= formatting in {function}() call') - return source_lines - - return source_lines[:start] + [ - f"{' ' * first_indent}{keyword}=[\n" - ] + [ - f"{' ' * indent}{to_literal(value, quote_style)},\n" - for value in new_value - ] + [ - f"{' ' * first_indent}],\n" - ] + source_lines[end:] - - -def find_call_kwarg_in_ast(tree, funcname, keyword, filename='setup.py'): - for node in ast.walk(tree): - if (isinstance(node, ast.Call) - and isinstance(node.func, ast.Name) - and node.func.id == funcname): - for kwarg in node.keywords: - if kwarg.arg == keyword: - return kwarg.value - else: - return None - else: - warn(f'Could not find {funcname}() call in {filename}') - return None - - -def eval_ast_node(node, keyword): - if isinstance(node, ast.Str): - return node.s - if isinstance(node, (ast.List, ast.Tuple)): - try: - return ast.literal_eval(node) - except ValueError: - pass - if (isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute) - and isinstance(node.func.value, ast.Str) - and node.func.attr == 'join'): - try: - return node.func.value.s.join(ast.literal_eval(node.args[0])) - except ValueError: - pass - warn(f'Non-literal {keyword}= passed to setup()') - return None - - -def parse_python_requires(s): - # https://www.python.org/dev/peps/pep-0440/#version-specifiers - rx = re.compile(r'^(~=|==|!=|<=|>=|<|>|===)\s*(\d+(?:\.\d+)*(?:\.\*)?)$') - - class BadConstraint(Exception): - pass - - handlers = {} - handler = partial(partial, handlers.__setitem__) - - # - # We are not doing a strict PEP-440 implementation here because if - # python_reqiures allows, say, Python 2.7.16, then we want to report that - # as Python 2.7. In each handler ``canditate`` is a two-tuple (X, Y) - # that represents any Python version between X.Y.0 and X.Y.. - # - - @handler('~=') - def compatible_version(constraint): - if len(constraint) < 2: - raise BadConstraint('~= requires a version with at least one dot') - if constraint[-1] == '*': - raise BadConstraint('~= does not allow a .*') - return lambda candidate: candidate == constraint[:2] - - @handler('==') - def matching_version(constraint): - # we know len(candidate) == 2 - if len(constraint) == 2 and constraint[-1] == '*': - return lambda candidate: candidate[0] == constraint[0] - elif len(constraint) == 1: - # == X should imply Python X.0 - return lambda candidate: candidate == constraint + (0,) - else: - # == X.Y.* and == X.Y.Z both imply Python X.Y - return lambda candidate: candidate == constraint[:2] - - @handler('!=') - def excluded_version(constraint): - # we know len(candidate) == 2 - if constraint[-1] != '*': - # != X or != X.Y or != X.Y.Z all are meaningless for us, because - # there exists some W != Z where we allow X.Y.W and thus allow - # Python X.Y. - return lambda candidate: True - elif len(constraint) == 2: - # != X.* excludes the entirety of a major version - return lambda candidate: candidate[0] != constraint[0] - else: - # != X.Y.* excludes one particular minor version X.Y, - # != X.Y.Z.* does not exclude anything, but it's fine, - # len(candidate) != len(constraint[:-1] so it'll be equivalent to - # True anyway. - return lambda candidate: candidate != constraint[:-1] - - @handler('>=') - def greater_or_equal_version(constraint): - if constraint[-1] == '*': - raise BadConstraint('>= does not allow a .*') - # >= X, >= X.Y, >= X.Y.Z all work out nicely because in Python - # (3, 0) >= (3,) - return lambda candidate: candidate >= constraint[:2] - - @handler('<=') - def lesser_or_equal_version(constraint): - if constraint[-1] == '*': - raise BadConstraint('<= does not allow a .*') - if len(constraint) == 1: - # <= X allows up to X.0 - return lambda candidate: candidate <= constraint + (0,) - else: - # <= X.Y[.Z] allows up to X.Y - return lambda candidate: candidate <= constraint - - @handler('>') - def greater_version(constraint): - if constraint[-1] == '*': - raise BadConstraint('> does not allow a .*') - if len(constraint) == 1: - # > X allows X+1.0 etc - return lambda candidate: candidate[0] > constraint[0] - elif len(constraint) == 2: - # > X.Y allows X.Y+1 etc - return lambda candidate: candidate > constraint - else: - # > X.Y.Z allows X.Y - return lambda candidate: candidate >= constraint[:2] - - @handler('<') - def lesser_version(constraint): - if constraint[-1] == '*': - raise BadConstraint('< does not allow a .*') - # < X, < X.Y, < X.Y.Z all work out nicely because in Python - # (3, 0) > (3,), (3, 0) == (3, 0) and (3, 0) < (3, 0, 1) - return lambda candidate: candidate < constraint - - @handler('===') - def arbitrary_version(constraint): - if constraint[-1] == '*': - raise BadConstraint('=== does not allow a .*') - # === X does not allow anything - # === X.Y throws me into confusion; will pip compare Python's X.Y.Z === - # X.Y and reject all possible values of Z? - # === X.Y.Z allows X.Y - return lambda candidate: candidate == constraint[:2] - - constraints = [] - for specifier in map(str.strip, s.split(',')): - m = rx.match(specifier) - if not m: - warn(f'Bad python_requires specifier: {specifier}') - continue - op, ver = m.groups() - ver = tuple( - int(segment) if segment != '*' else segment - for segment in ver.split('.') - ) - try: - constraints.append(handlers[op](ver)) - except BadConstraint as error: - warn(f'Bad python_requires specifier: {specifier} ({error})') - - if not constraints: - return None - - versions = [] - for major, max_minor in [ - (1, MAX_PYTHON_1_VERSION), - (2, MAX_PYTHON_2_VERSION), - (3, CURRENT_PYTHON_3_VERSION)]: - for minor in range(0, max_minor + 1): - if all(constraint((major, minor)) for constraint in constraints): - versions.append(f'{major}.{minor}') - return versions - - -def get_tox_ini_python_versions(filename=TOX_INI): - conf = configparser.ConfigParser() - try: - conf.read(filename) - envlist = conf.get('tox', 'envlist') - except configparser.Error: - return [] - envlist = parse_envlist(envlist) - return sorted(set( - tox_env_to_py_version(e) for e in envlist if e.startswith('py'))) - - -def parse_envlist(envlist): - envs = [] - for part in re.split(r'((?:[{][^}]*[}]|[^,{\s])+)|,|\s+', envlist): - # NB: part can be None - part = (part or '').strip() - if not part: - continue - envs += brace_expand(part) - return envs - - -def brace_expand(s): - m = re.match('^([^{]*)[{]([^}]*)[}](.*)$', s) - if not m: - return [s] - left = m.group(1) - right = m.group(3) - res = [] - for alt in m.group(2).split(','): - res += brace_expand(left + alt + right) - return res - - -def tox_env_to_py_version(env): - if '-' in env: - # e.g. py34-coverage, pypy-subunit - env = env.partition('-')[0] - if env.startswith('pypy'): - return 'PyPy' + env[4:] - elif env.startswith('py') and len(env) >= 4: - return f'{env[2]}.{env[3:]}' - else: - return env - - -def get_travis_yml_python_versions(filename=TRAVIS_YML): - with open(filename) as fp: - conf = yaml.safe_load(fp) - versions = [] - if 'python' in conf: - versions += map(travis_normalize_py_version, conf['python']) - if 'matrix' in conf and 'include' in conf['matrix']: - for job in conf['matrix']['include']: - if 'python' in job: - versions.append(travis_normalize_py_version(job['python'])) - if 'jobs' in conf and 'include' in conf['jobs']: - for job in conf['jobs']['include']: - if 'python' in job: - versions.append(travis_normalize_py_version(job['python'])) - if 'env' in conf: - toxenvs = [] - for env in conf['env']: - if env.startswith('TOXENV='): - toxenvs.extend(parse_envlist(env.partition('=')[-1])) - versions.extend( - tox_env_to_py_version(e) for e in toxenvs if e.startswith('py')) - return sorted(set(versions)) - - -def travis_normalize_py_version(v): - v = str(v) - if v.startswith('pypy3'): - # could be pypy3, pypy3.5, pypy3.5-5.10.0 - return 'PyPy3' - elif v.startswith('pypy'): - # could be pypy, pypy2, pypy2.7, pypy2.7-5.10.0 - return 'PyPy' - else: - return v - - -def get_appveyor_yml_python_versions(filename=APPVEYOR_YML): - with open(filename) as fp: - conf = yaml.safe_load(fp) - # There's more than one way of doing this, I'm setting %PYTHON% to - # the directory that has a Python interpreter (C:\PythonXY) - versions = [] - for env in conf['environment']['matrix']: - for var, value in env.items(): - if var.lower() == 'python': - versions.append(appveyor_normalize_py_version(value)) - elif var == 'TOXENV': - toxenvs = parse_envlist(value) - versions.extend( - tox_env_to_py_version(e) - for e in toxenvs if e.startswith('py')) - return sorted(set(versions)) - - -def appveyor_normalize_py_version(ver): - ver = str(ver).lower() - if ver.startswith('c:\\python'): - ver = ver[len('c:\\python'):] - if ver.endswith('\\'): - ver = ver[:-1] - if ver.endswith('-x64'): - ver = ver[:-len('-x64')] - assert len(ver) >= 2 and ver[:2].isdigit() - return f'{ver[0]}.{ver[1:]}' - - -def get_manylinux_python_versions(filename=MANYLINUX_INSTALL_SH): - magic = re.compile(r'.*\[\[ "\$\{PYBIN\}" == \*"cp(\d)(\d)"\* \]\]') - versions = [] - with open(filename) as fp: - for line in fp: - m = magic.match(line) - if m: - versions.append('{}.{}'.format(*m.groups())) - return sorted(set(versions)) - - -def important(versions): - upcoming_release = f'3.{CURRENT_PYTHON_3_VERSION + 1}' - return { - v for v in versions - if not v.startswith(('PyPy', 'Jython')) and v != 'nightly' - and not v.endswith('-dev') and v != upcoming_release - } - - def parse_version(v): try: major, minor = map(int, v.split('.', 1)) @@ -605,13 +84,6 @@ def parse_version_list(v): return sorted(versions) -def update_version_list(versions, add=None, drop=None, update=None): - if update: - return sorted(update) - else: - return sorted(set(versions).union(add or ()).difference(drop or ())) - - def is_package(where='.'): setup_py = os.path.join(where, 'setup.py') return os.path.exists(setup_py) diff --git a/src/check_python_versions/parsers/__init__.py b/src/check_python_versions/parsers/__init__.py new file mode 100644 index 0000000..4014849 --- /dev/null +++ b/src/check_python_versions/parsers/__init__.py @@ -0,0 +1 @@ +# make a package diff --git a/src/check_python_versions/parsers/appveyor.py b/src/check_python_versions/parsers/appveyor.py new file mode 100644 index 0000000..7067428 --- /dev/null +++ b/src/check_python_versions/parsers/appveyor.py @@ -0,0 +1,39 @@ +try: + import yaml +except ImportError: # pragma: nocover + yaml = None + +from .tox import parse_envlist, tox_env_to_py_version + + +APPVEYOR_YML = 'appveyor.yml' + + +def get_appveyor_yml_python_versions(filename=APPVEYOR_YML): + with open(filename) as fp: + conf = yaml.safe_load(fp) + # There's more than one way of doing this, I'm setting %PYTHON% to + # the directory that has a Python interpreter (C:\PythonXY) + versions = [] + for env in conf['environment']['matrix']: + for var, value in env.items(): + if var.lower() == 'python': + versions.append(appveyor_normalize_py_version(value)) + elif var == 'TOXENV': + toxenvs = parse_envlist(value) + versions.extend( + tox_env_to_py_version(e) + for e in toxenvs if e.startswith('py')) + return sorted(set(versions)) + + +def appveyor_normalize_py_version(ver): + ver = str(ver).lower() + if ver.startswith('c:\\python'): + ver = ver[len('c:\\python'):] + if ver.endswith('\\'): + ver = ver[:-1] + if ver.endswith('-x64'): + ver = ver[:-len('-x64')] + assert len(ver) >= 2 and ver[:2].isdigit() + return f'{ver[0]}.{ver[1:]}' diff --git a/src/check_python_versions/parsers/manylinux.py b/src/check_python_versions/parsers/manylinux.py new file mode 100644 index 0000000..0d959de --- /dev/null +++ b/src/check_python_versions/parsers/manylinux.py @@ -0,0 +1,14 @@ +import re + +MANYLINUX_INSTALL_SH = '.manylinux-install.sh' + + +def get_manylinux_python_versions(filename=MANYLINUX_INSTALL_SH): + magic = re.compile(r'.*\[\[ "\$\{PYBIN\}" == \*"cp(\d)(\d)"\* \]\]') + versions = [] + with open(filename) as fp: + for line in fp: + m = magic.match(line) + if m: + versions.append('{}.{}'.format(*m.groups())) + return sorted(set(versions)) diff --git a/src/check_python_versions/parsers/python.py b/src/check_python_versions/parsers/python.py new file mode 100644 index 0000000..aa7d0d3 --- /dev/null +++ b/src/check_python_versions/parsers/python.py @@ -0,0 +1,343 @@ +import ast +import os +import re +import string +from functools import partial + +from ..utils import warn, pipe, confirm_and_update_file +from ..versions import MAX_MINOR_FOR_MAJOR + + +def get_supported_python_versions(repo_path='.'): + setup_py = os.path.join(repo_path, 'setup.py') + classifiers = get_setup_py_keyword(setup_py, 'classifiers') + if classifiers is None: + # AST parsing is complicated + classifiers = pipe("python", "setup.py", "-q", "--classifiers", + cwd=repo_path).splitlines() + return get_versions_from_classifiers(classifiers) + + +def get_python_requires(setup_py='setup.py'): + python_requires = get_setup_py_keyword(setup_py, 'python_requires') + if python_requires is None: + return None + return parse_python_requires(python_requires) + + +def is_version_classifier(s): + prefix = 'Programming Language :: Python :: ' + return s.startswith(prefix) and s[len(prefix):len(prefix) + 1].isdigit() + + +def is_major_version_classifier(s): + prefix = 'Programming Language :: Python :: ' + return ( + s.startswith(prefix) + and s[len(prefix):].replace(' :: Only', '').isdigit() + ) + + +def get_versions_from_classifiers(classifiers): + # Based on + # https://github.com/mgedmin/project-summary/blob/master/summary.py#L221-L234 + prefix = 'Programming Language :: Python :: ' + impl_prefix = 'Programming Language :: Python :: Implementation :: ' + cpython = impl_prefix + 'CPython' + versions = { + s[len(prefix):].replace(' :: Only', '').rstrip() + for s in classifiers + if is_version_classifier(s) + } | { + s[len(impl_prefix):].rstrip() + for s in classifiers + if s.startswith(impl_prefix) and s != cpython + } + for major in '2', '3': + if major in versions and any( + v.startswith(f'{major}.') for v in versions): + versions.remove(major) + return sorted(versions) + + +def update_classifiers(classifiers, new_versions): + prefix = 'Programming Language :: Python :: ' + + for pos, s in enumerate(classifiers): + if is_version_classifier(s): + break + else: + pos = len(classifiers) + + if any(map(is_major_version_classifier, classifiers)): + new_versions = sorted( + set(new_versions).union( + v.partition('.')[0] for v in new_versions + ) + ) + + classifiers = [ + s for s in classifiers if not is_version_classifier(s) + ] + new_classifiers = [ + f'{prefix}{version}' + for version in new_versions + ] + classifiers[pos:pos] = new_classifiers + return classifiers + + +def update_supported_python_versions(repo_path, new_versions): + setup_py = os.path.join(repo_path, 'setup.py') + classifiers = get_setup_py_keyword(setup_py, 'classifiers') + if classifiers is None: + return + new_classifiers = update_classifiers(classifiers, new_versions) + update_setup_py_keyword(setup_py, 'classifiers', new_classifiers) + + +def get_setup_py_keyword(setup_py, keyword): + with open(setup_py) as f: + try: + tree = ast.parse(f.read(), setup_py) + except SyntaxError as error: + warn(f'Could not parse {setup_py}: {error}') + return None + node = find_call_kwarg_in_ast(tree, 'setup', keyword) + return node and eval_ast_node(node, keyword) + + +def update_setup_py_keyword(setup_py, keyword, new_value): + with open(setup_py) as f: + lines = f.readlines() + new_lines = update_call_arg_in_source(lines, 'setup', keyword, new_value) + confirm_and_update_file(setup_py, lines, new_lines) + + +def to_literal(value, quote_style='"'): + # Because I don't want to deal with quoting, I'll require all values + # to contain only safe characters (i.e. no ' or " or \). Except some + # PyPI classifiers do include ' so I need to handle that at least. + safe_characters = string.ascii_letters + string.digits + " .:,-=><()/+'#" + assert all( + c in safe_characters for c in value + ), f'{value!r} has unexpected characters' + if quote_style == "'" and quote_style in value: + quote_style = '"' + assert quote_style not in value + return f'{quote_style}{value}{quote_style}' + + +def update_call_arg_in_source(source_lines, function, keyword, new_value): + lines = iter(enumerate(source_lines)) + for n, line in lines: + if line.startswith(f'{function}('): + break + else: + warn(f'Did not find {function}() call') + return source_lines + for n, line in lines: + stripped = line.lstrip() + if stripped.startswith(f'{keyword}='): + first_indent = len(line) - len(stripped) + must_fix_indents = not line.rstrip().endswith('=[') + break + else: + warn(f'Did not find {keyword}= argument in {function}() call') + return source_lines + + start = n + indent = first_indent + 4 + quote_style = '"' + for n, line in lines: + stripped = line.lstrip() + if stripped.startswith('],'): + end = n + 1 + break + elif stripped: + if not must_fix_indents: + indent = len(line) - len(stripped) + if stripped[0] in ('"', "'"): + quote_style = stripped[0] + if line.rstrip().endswith('],'): + end = n + 1 + break + else: + warn(f'Did not understand {keyword}= formatting in {function}() call') + return source_lines + + return source_lines[:start] + [ + f"{' ' * first_indent}{keyword}=[\n" + ] + [ + f"{' ' * indent}{to_literal(value, quote_style)},\n" + for value in new_value + ] + [ + f"{' ' * first_indent}],\n" + ] + source_lines[end:] + + +def find_call_kwarg_in_ast(tree, funcname, keyword, filename='setup.py'): + for node in ast.walk(tree): + if (isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == funcname): + for kwarg in node.keywords: + if kwarg.arg == keyword: + return kwarg.value + else: + return None + else: + warn(f'Could not find {funcname}() call in {filename}') + return None + + +def eval_ast_node(node, keyword): + if isinstance(node, ast.Str): + return node.s + if isinstance(node, (ast.List, ast.Tuple)): + try: + return ast.literal_eval(node) + except ValueError: + pass + if (isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute) + and isinstance(node.func.value, ast.Str) + and node.func.attr == 'join'): + try: + return node.func.value.s.join(ast.literal_eval(node.args[0])) + except ValueError: + pass + warn(f'Non-literal {keyword}= passed to setup()') + return None + + +def parse_python_requires(s): + # https://www.python.org/dev/peps/pep-0440/#version-specifiers + rx = re.compile(r'^(~=|==|!=|<=|>=|<|>|===)\s*(\d+(?:\.\d+)*(?:\.\*)?)$') + + class BadConstraint(Exception): + pass + + handlers = {} + handler = partial(partial, handlers.__setitem__) + + # + # We are not doing a strict PEP-440 implementation here because if + # python_reqiures allows, say, Python 2.7.16, then we want to report that + # as Python 2.7. In each handler ``canditate`` is a two-tuple (X, Y) + # that represents any Python version between X.Y.0 and X.Y.. + # + + @handler('~=') + def compatible_version(constraint): + if len(constraint) < 2: + raise BadConstraint('~= requires a version with at least one dot') + if constraint[-1] == '*': + raise BadConstraint('~= does not allow a .*') + return lambda candidate: candidate == constraint[:2] + + @handler('==') + def matching_version(constraint): + # we know len(candidate) == 2 + if len(constraint) == 2 and constraint[-1] == '*': + return lambda candidate: candidate[0] == constraint[0] + elif len(constraint) == 1: + # == X should imply Python X.0 + return lambda candidate: candidate == constraint + (0,) + else: + # == X.Y.* and == X.Y.Z both imply Python X.Y + return lambda candidate: candidate == constraint[:2] + + @handler('!=') + def excluded_version(constraint): + # we know len(candidate) == 2 + if constraint[-1] != '*': + # != X or != X.Y or != X.Y.Z all are meaningless for us, because + # there exists some W != Z where we allow X.Y.W and thus allow + # Python X.Y. + return lambda candidate: True + elif len(constraint) == 2: + # != X.* excludes the entirety of a major version + return lambda candidate: candidate[0] != constraint[0] + else: + # != X.Y.* excludes one particular minor version X.Y, + # != X.Y.Z.* does not exclude anything, but it's fine, + # len(candidate) != len(constraint[:-1] so it'll be equivalent to + # True anyway. + return lambda candidate: candidate != constraint[:-1] + + @handler('>=') + def greater_or_equal_version(constraint): + if constraint[-1] == '*': + raise BadConstraint('>= does not allow a .*') + # >= X, >= X.Y, >= X.Y.Z all work out nicely because in Python + # (3, 0) >= (3,) + return lambda candidate: candidate >= constraint[:2] + + @handler('<=') + def lesser_or_equal_version(constraint): + if constraint[-1] == '*': + raise BadConstraint('<= does not allow a .*') + if len(constraint) == 1: + # <= X allows up to X.0 + return lambda candidate: candidate <= constraint + (0,) + else: + # <= X.Y[.Z] allows up to X.Y + return lambda candidate: candidate <= constraint + + @handler('>') + def greater_version(constraint): + if constraint[-1] == '*': + raise BadConstraint('> does not allow a .*') + if len(constraint) == 1: + # > X allows X+1.0 etc + return lambda candidate: candidate[0] > constraint[0] + elif len(constraint) == 2: + # > X.Y allows X.Y+1 etc + return lambda candidate: candidate > constraint + else: + # > X.Y.Z allows X.Y + return lambda candidate: candidate >= constraint[:2] + + @handler('<') + def lesser_version(constraint): + if constraint[-1] == '*': + raise BadConstraint('< does not allow a .*') + # < X, < X.Y, < X.Y.Z all work out nicely because in Python + # (3, 0) > (3,), (3, 0) == (3, 0) and (3, 0) < (3, 0, 1) + return lambda candidate: candidate < constraint + + @handler('===') + def arbitrary_version(constraint): + if constraint[-1] == '*': + raise BadConstraint('=== does not allow a .*') + # === X does not allow anything + # === X.Y throws me into confusion; will pip compare Python's X.Y.Z === + # X.Y and reject all possible values of Z? + # === X.Y.Z allows X.Y + return lambda candidate: candidate == constraint[:2] + + constraints = [] + for specifier in map(str.strip, s.split(',')): + m = rx.match(specifier) + if not m: + warn(f'Bad python_requires specifier: {specifier}') + continue + op, ver = m.groups() + ver = tuple( + int(segment) if segment != '*' else segment + for segment in ver.split('.') + ) + try: + constraints.append(handlers[op](ver)) + except BadConstraint as error: + warn(f'Bad python_requires specifier: {specifier} ({error})') + + if not constraints: + return None + + versions = [] + for major in sorted(MAX_MINOR_FOR_MAJOR): + for minor in range(0, MAX_MINOR_FOR_MAJOR[major] + 1): + if all(constraint((major, minor)) for constraint in constraints): + versions.append(f'{major}.{minor}') + return versions diff --git a/src/check_python_versions/parsers/tox.py b/src/check_python_versions/parsers/tox.py new file mode 100644 index 0000000..ec664b4 --- /dev/null +++ b/src/check_python_versions/parsers/tox.py @@ -0,0 +1,52 @@ +import configparser +import re + + +TOX_INI = 'tox.ini' + + +def get_tox_ini_python_versions(filename=TOX_INI): + conf = configparser.ConfigParser() + try: + conf.read(filename) + envlist = conf.get('tox', 'envlist') + except configparser.Error: + return [] + envlist = parse_envlist(envlist) + return sorted(set( + tox_env_to_py_version(e) for e in envlist if e.startswith('py'))) + + +def parse_envlist(envlist): + envs = [] + for part in re.split(r'((?:[{][^}]*[}]|[^,{\s])+)|,|\s+', envlist): + # NB: part can be None + part = (part or '').strip() + if not part: + continue + envs += brace_expand(part) + return envs + + +def brace_expand(s): + m = re.match('^([^{]*)[{]([^}]*)[}](.*)$', s) + if not m: + return [s] + left = m.group(1) + right = m.group(3) + res = [] + for alt in m.group(2).split(','): + res += brace_expand(left + alt + right) + return res + + +def tox_env_to_py_version(env): + if '-' in env: + # e.g. py34-coverage, pypy-subunit + env = env.partition('-')[0] + if env.startswith('pypy'): + return 'PyPy' + env[4:] + elif env.startswith('py') and len(env) >= 4: + return f'{env[2]}.{env[3:]}' + else: + return env diff --git a/src/check_python_versions/parsers/travis.py b/src/check_python_versions/parsers/travis.py new file mode 100644 index 0000000..8fb0eab --- /dev/null +++ b/src/check_python_versions/parsers/travis.py @@ -0,0 +1,45 @@ +try: + import yaml +except ImportError: # pragma: nocover + yaml = None + +from .tox import parse_envlist, tox_env_to_py_version + + +TRAVIS_YML = '.travis.yml' + + +def get_travis_yml_python_versions(filename=TRAVIS_YML): + with open(filename) as fp: + conf = yaml.safe_load(fp) + versions = [] + if 'python' in conf: + versions += map(travis_normalize_py_version, conf['python']) + if 'matrix' in conf and 'include' in conf['matrix']: + for job in conf['matrix']['include']: + if 'python' in job: + versions.append(travis_normalize_py_version(job['python'])) + if 'jobs' in conf and 'include' in conf['jobs']: + for job in conf['jobs']['include']: + if 'python' in job: + versions.append(travis_normalize_py_version(job['python'])) + if 'env' in conf: + toxenvs = [] + for env in conf['env']: + if env.startswith('TOXENV='): + toxenvs.extend(parse_envlist(env.partition('=')[-1])) + versions.extend( + tox_env_to_py_version(e) for e in toxenvs if e.startswith('py')) + return sorted(set(versions)) + + +def travis_normalize_py_version(v): + v = str(v) + if v.startswith('pypy3'): + # could be pypy3, pypy3.5, pypy3.5-5.10.0 + return 'PyPy3' + elif v.startswith('pypy'): + # could be pypy, pypy2, pypy2.7, pypy2.7-5.10.0 + return 'PyPy' + else: + return v diff --git a/src/check_python_versions/utils.py b/src/check_python_versions/utils.py new file mode 100644 index 0000000..aa7e3ad --- /dev/null +++ b/src/check_python_versions/utils.py @@ -0,0 +1,55 @@ +import difflib +import logging +import os +import stat +import subprocess +import sys + + +log = logging.getLogger('check-python-versions') + + +def warn(msg): + print(msg, file=sys.stderr) + + +def pipe(*cmd, **kwargs): + if 'cwd' in kwargs: + log.debug('EXEC cd %s && %s', kwargs['cwd'], ' '.join(cmd)) + else: + log.debug('EXEC %s', ' '.join(cmd)) + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, **kwargs) + return p.communicate()[0].decode('UTF-8', 'replace') + + +def confirm_and_update_file(filename, old_lines, new_lines): + print_diff(old_lines, new_lines, filename) + if new_lines != old_lines and confirm(f"Write changes to {filename}?"): + mode = stat.S_IMODE(os.stat(filename).st_mode) + tempfile = filename + '.tmp' + with open(tempfile, 'w') as f: + os.fchmod(f.fileno(), mode) + f.writelines(new_lines) + os.rename(tempfile, filename) + + +def print_diff(a, b, filename): + print(''.join(difflib.unified_diff( + a, b, + filename, filename, + "(original)", "(updated)", + ))) + + +def confirm(prompt): + while True: + try: + answer = input(f'{prompt} [y/N] ').strip().lower() + except EOFError: + answer = "" + if answer == 'y': + print() + return True + if answer == 'n' or not answer: + print() + return False diff --git a/src/check_python_versions/versions.py b/src/check_python_versions/versions.py new file mode 100644 index 0000000..2e7743b --- /dev/null +++ b/src/check_python_versions/versions.py @@ -0,0 +1,25 @@ +MAX_PYTHON_1_VERSION = 6 # i.e. 1.6 +MAX_PYTHON_2_VERSION = 7 # i.e. 2.7 +CURRENT_PYTHON_3_VERSION = 7 # i.e. 3.7 + +MAX_MINOR_FOR_MAJOR = { + 1: MAX_PYTHON_1_VERSION, + 2: MAX_PYTHON_2_VERSION, + 3: CURRENT_PYTHON_3_VERSION, +} + + +def important(versions): + upcoming_release = f'3.{CURRENT_PYTHON_3_VERSION + 1}' + return { + v for v in versions + if not v.startswith(('PyPy', 'Jython')) and v != 'nightly' + and not v.endswith('-dev') and v != upcoming_release + } + + +def update_version_list(versions, add=None, drop=None, update=None): + if update: + return sorted(update) + else: + return sorted(set(versions).union(add or ()).difference(drop or ())) diff --git a/tests.py b/tests.py deleted file mode 100644 index b23276e..0000000 --- a/tests.py +++ /dev/null @@ -1,969 +0,0 @@ -import argparse -import ast -import os -import re -import sys -import textwrap - -import pytest - -import check_python_versions.cli as cpv - - -needs_pyyaml = pytest.mark.skipIf(cpv.yaml is None, "PyYAML not installed") - - -def test_pipe(): - assert cpv.pipe('echo', 'hi') == 'hi\n' - - -def test_get_supported_python_versions(tmp_path): - (tmp_path / "setup.py").write_text(textwrap.dedent("""\ - from setuptools import setup - setup( - name='foo', - classifiers=[ - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3.6', - ], - ) - """)) - assert cpv.get_supported_python_versions(tmp_path) == ['2.7', '3.6'] - - -def test_get_supported_python_versions_computed(tmp_path): - (tmp_path / "setup.py").write_text(textwrap.dedent("""\ - from setuptools import setup - setup( - name='foo', - classifiers=[ - 'Programming Language :: Python :: %s' % v - for v in ['2.7', '3.7'] - ], - ) - """)) - assert cpv.get_supported_python_versions(tmp_path) == ['2.7', '3.7'] - - -def test_get_versions_from_classifiers(): - assert cpv.get_versions_from_classifiers([ - 'Development Status :: 4 - Beta', - 'Environment :: Console', - 'Intended Audience :: Developers', - 'License :: OSI Approved :: GNU General Public License (GPL)', - 'Operating System :: OS Independent', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: Implementation :: CPython', - 'Programming Language :: Python :: Implementation :: PyPy', - ]) == ['2.7', '3.6', '3.7', 'PyPy'] - - -def test_get_versions_from_classifiers_major_only(): - assert cpv.get_versions_from_classifiers([ - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 3', - ]) == ['2', '3'] - - -def test_get_versions_from_classifiers_with_only_suffix(): - assert cpv.get_versions_from_classifiers([ - 'Programming Language :: Python :: 2 :: Only', - ]) == ['2'] - - -def test_get_versions_from_classifiers_with_trailing_whitespace(): - # I was surprised too that this is allowed! - assert cpv.get_versions_from_classifiers([ - 'Programming Language :: Python :: 3.6 ', - ]) == ['3.6'] - - -def test_update_classifiers(): - assert cpv.update_classifiers([ - 'Development Status :: 4 - Beta', - 'Environment :: Console', - 'Intended Audience :: Developers', - 'License :: OSI Approved :: GNU General Public License (GPL)', - 'Operating System :: OS Independent', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: Implementation :: CPython', - 'Programming Language :: Python :: Implementation :: PyPy', - 'Typing :: Typed', - ], ['2.7', '3.7']) == [ - 'Development Status :: 4 - Beta', - 'Environment :: Console', - 'Intended Audience :: Developers', - 'License :: OSI Approved :: GNU General Public License (GPL)', - 'Operating System :: OS Independent', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: Implementation :: CPython', - 'Programming Language :: Python :: Implementation :: PyPy', - 'Typing :: Typed', - ] - - -def test_update_classifiers_drop_major(): - assert cpv.update_classifiers([ - 'Development Status :: 4 - Beta', - 'Environment :: Console', - 'Intended Audience :: Developers', - 'License :: OSI Approved :: GNU General Public License (GPL)', - 'Operating System :: OS Independent', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: Implementation :: CPython', - 'Programming Language :: Python :: Implementation :: PyPy', - 'Typing :: Typed', - ], ['3.6', '3.7']) == [ - 'Development Status :: 4 - Beta', - 'Environment :: Console', - 'Intended Audience :: Developers', - 'License :: OSI Approved :: GNU General Public License (GPL)', - 'Operating System :: OS Independent', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: Implementation :: CPython', - 'Programming Language :: Python :: Implementation :: PyPy', - 'Typing :: Typed', - ] - - -def test_update_classifiers_no_major(): - assert cpv.update_classifiers([ - 'Development Status :: 4 - Beta', - 'Environment :: Console', - 'Intended Audience :: Developers', - 'License :: OSI Approved :: GNU General Public License (GPL)', - 'Operating System :: OS Independent', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: Implementation :: CPython', - 'Programming Language :: Python :: Implementation :: PyPy', - 'Typing :: Typed', - ], ['2.7', '3.7']) == [ - 'Development Status :: 4 - Beta', - 'Environment :: Console', - 'Intended Audience :: Developers', - 'License :: OSI Approved :: GNU General Public License (GPL)', - 'Operating System :: OS Independent', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: Implementation :: CPython', - 'Programming Language :: Python :: Implementation :: PyPy', - 'Typing :: Typed', - ] - - -def test_update_classifiers_none_were_present(): - assert cpv.update_classifiers([ - 'Development Status :: 4 - Beta', - 'Environment :: Console', - 'Intended Audience :: Developers', - 'License :: OSI Approved :: GNU General Public License (GPL)', - 'Operating System :: OS Independent', - ], ['2.7', '3.7']) == [ - 'Development Status :: 4 - Beta', - 'Environment :: Console', - 'Intended Audience :: Developers', - 'License :: OSI Approved :: GNU General Public License (GPL)', - 'Operating System :: OS Independent', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3.7', - ] - - -def test_get_python_requires(tmp_path, monkeypatch): - setup_py = tmp_path / "setup.py" - setup_py.write_text(textwrap.dedent("""\ - from setuptools import setup - setup( - name='foo', - python_requires='>= 3.6', - ) - """)) - monkeypatch.setattr(cpv, 'CURRENT_PYTHON_3_VERSION', 7) - assert cpv.get_python_requires(setup_py) == ['3.6', '3.7'] - - -def test_get_python_requires_not_specified(tmp_path, capsys): - setup_py = tmp_path / "setup.py" - setup_py.write_text(textwrap.dedent("""\ - from setuptools import setup - setup( - name='foo', - ) - """)) - assert cpv.get_python_requires(setup_py) is None - assert capsys.readouterr().err == '' - - -def test_get_setup_py_keyword_syntax_error(tmp_path, capsys): - setup_py = tmp_path / "setup.py" - setup_py.write_text(textwrap.dedent("""\ - from setuptools import setup - setup( - name='foo', - # uh do I need to close parens? what if I forget? ;) - """)) - assert cpv.get_setup_py_keyword(setup_py, 'name') is None - assert 'Could not parse' in capsys.readouterr().err - - -def test_find_call_kwarg_in_ast(): - tree = ast.parse('foo(bar="foo")') - ast.dump(tree) - node = cpv.find_call_kwarg_in_ast(tree, 'foo', 'bar') - assert isinstance(node, ast.Str) - assert node.s == "foo" - - -def test_find_call_kwarg_in_ast_no_arg(capsys): - tree = ast.parse('foo(baz="foo")') - ast.dump(tree) - node = cpv.find_call_kwarg_in_ast(tree, 'foo', 'bar') - assert node is None - assert capsys.readouterr().err == '' - - -def test_find_call_kwarg_in_ast_no_call(capsys): - tree = ast.parse('fooo(bar="foo")') - ast.dump(tree) - node = cpv.find_call_kwarg_in_ast(tree, 'foo', 'bar') - assert node is None - assert 'Could not find foo() call in setup.py' in capsys.readouterr().err - - -@pytest.mark.parametrize('code, expected', [ - ('"hi"', "hi"), - ('"hi\\n"', "hi\n"), - ('["a", "b"]', ["a", "b"]), - ('("a", "b")', ("a", "b")), - ('"-".join(["a", "b"])', "a-b"), -]) -def test_eval_ast_node(code, expected): - tree = ast.parse(f'foo(bar={code})') - node = cpv.find_call_kwarg_in_ast(tree, 'foo', 'bar') - assert node is not None - assert cpv.eval_ast_node(node, 'bar') == expected - - -def test_to_literal(): - assert cpv.to_literal("blah") == '"blah"' - assert cpv.to_literal("blah", "'") == "'blah'" - - -def test_to_literal_embedded_quote(): - assert cpv.to_literal( - "Environment :: Handhelds/PDA's" - ) == '"Environment :: Handhelds/PDA\'s"' - assert cpv.to_literal( - "Environment :: Handhelds/PDA's", "'" - ) == '"Environment :: Handhelds/PDA\'s"' - - -def test_to_literal_all_the_classifiers(): - with open('CLASSIFIERS') as f: - for line in f: - classifier = line.strip() - literal = cpv.to_literal(classifier) - assert ast.literal_eval(literal) == classifier - - -def test_update_call_arg_in_source(): - source_lines = textwrap.dedent("""\ - setup( - foo=1, - bar=[ - "a", - "b", - "c", - ], - baz=2, - ) - """).splitlines(True) - result = cpv.update_call_arg_in_source(source_lines, "setup", "bar", - ["x", "y"]) - assert "".join(result) == textwrap.dedent("""\ - setup( - foo=1, - bar=[ - "x", - "y", - ], - baz=2, - ) - """) - - -def test_update_call_arg_in_source_preserves_indent_and_quote_style(): - source_lines = textwrap.dedent("""\ - setup(foo=1, - bar=[ - 'a', - 'b', - 'c', - ], - ) - """).splitlines(True) - result = cpv.update_call_arg_in_source(source_lines, "setup", "bar", - ["x", "y"]) - assert "".join(result) == textwrap.dedent("""\ - setup(foo=1, - bar=[ - 'x', - 'y', - ], - ) - """) - - -def test_update_call_arg_in_source_fixes_closing_bracket(): - source_lines = textwrap.dedent("""\ - setup(foo=1, - bar=[ - 'a', - 'b', - 'c'], - baz=2, - ) - """).splitlines(True) - result = cpv.update_call_arg_in_source(source_lines, "setup", "bar", - ["x", "y"]) - assert "".join(result) == textwrap.dedent("""\ - setup(foo=1, - bar=[ - 'x', - 'y', - ], - baz=2, - ) - """) - - -def test_update_call_arg_in_source_fixes_opening_bracket(): - source_lines = textwrap.dedent("""\ - setup(foo=1, - bar=['a', - 'b', - 'c'], - baz=2, - ) - """).splitlines(True) - result = cpv.update_call_arg_in_source(source_lines, "setup", "bar", - ["x", "y"]) - assert "".join(result) == textwrap.dedent("""\ - setup(foo=1, - bar=[ - 'x', - 'y', - ], - baz=2, - ) - """) - - -@pytest.mark.parametrize('code', [ - '[2 * 2]', - '"".join([2 * 2])', -]) -def test_eval_ast_node_failures(code, capsys): - tree = ast.parse(f'foo(bar={code})') - node = cpv.find_call_kwarg_in_ast(tree, 'foo', 'bar') - assert cpv.eval_ast_node(node, 'bar') is None - assert 'Non-literal bar= passed to setup()' in capsys.readouterr().err - - -@pytest.mark.parametrize('constraint, result', [ - ('~= 2.7', ['2.7']), - ('~= 2.7.12', ['2.7']), -]) -def test_parse_python_requires_approximately(constraint, result): - assert cpv.parse_python_requires(constraint) == result - - -def test_parse_python_requires_approximately_not_enough_dots(capsys): - assert cpv.parse_python_requires('~= 2') is None - assert ( - 'Bad python_requires specifier: ~= 2' - ' (~= requires a version with at least one dot)' - in capsys.readouterr().err - ) - - -@pytest.mark.parametrize('constraint, result', [ - ('== 2.7', ['2.7']), - ('== 2.7.*', ['2.7']), - ('== 2.7.12', ['2.7']), - ('== 2.*, >= 2.6', ['2.6', '2.7']), - ('== 3.0', ['3.0']), - ('== 3', ['3.0']), -]) -def test_parse_python_requires_matching_version(constraint, result): - assert cpv.parse_python_requires(constraint) == result - - -def test_parse_python_requires_greater_than(monkeypatch): - monkeypatch.setattr(cpv, 'CURRENT_PYTHON_3_VERSION', 8) - assert cpv.parse_python_requires('>= 3.6') == ['3.6', '3.7', '3.8'] - - -@pytest.mark.parametrize('constraint, result', [ - ('>= 2.7, != 3.*', ['2.7']), - ('>= 2.7.12, != 3.*', ['2.7']), - ('>= 2.7, != 3.0.*, != 3.1.*', ['2.7', '3.2', '3.3']), - # != 3.2 means we reject 3.2.0 but still accept any other 3.2.x - ('>= 2.7, != 3.2', ['2.7', '3.0', '3.1', '3.2', '3.3']), - ('>= 2.7, != 3.2.1', ['2.7', '3.0', '3.1', '3.2', '3.3']), - ('>= 2.7, <= 3', ['2.7', '3.0']), - ('>= 2.7, <= 3.2', ['2.7', '3.0', '3.1', '3.2']), - ('>= 2.7, <= 3.2.1', ['2.7', '3.0', '3.1', '3.2']), - ('>= 3', ['3.0', '3.1', '3.2', '3.3']), -]) -def test_parse_python_requires_greater_than_with_exceptions( - monkeypatch, constraint, result -): - monkeypatch.setattr(cpv, 'CURRENT_PYTHON_3_VERSION', 3) - assert cpv.parse_python_requires(constraint) == result - - -def test_parse_python_requires_multiple_greater_than(monkeypatch): - monkeypatch.setattr(cpv, 'CURRENT_PYTHON_3_VERSION', 7) - assert cpv.parse_python_requires('>= 2.7, >= 3.6') == ['3.6', '3.7'] - - -@pytest.mark.parametrize('constraint, result', [ - ('> 2, < 3.1', ['3.0']), - ('> 2.6, < 3', ['2.7']), - ('> 2.7.12, < 3', ['2.7']), - ('> 2.7.12, < 3.0', ['2.7']), - ('> 2.7.12, < 3.1', ['2.7', '3.0']), - ('> 2.7.12, < 3.0.1', ['2.7', '3.0']), -]) -def test_parse_python_exclusive_ordering(constraint, result): - assert cpv.parse_python_requires(constraint) == result - - -@pytest.mark.parametrize('constraint, result', [ - ('=== 2.7', ['2.7']), - ('=== 2.7.12', ['2.7']), - ('=== 3', []), -]) -def test_parse_python_requires_arbitrary_version(constraint, result): - assert cpv.parse_python_requires(constraint) == result - - -@pytest.mark.parametrize('op', ['~=', '>=', '<=', '>', '<', '===']) -def test_parse_python_requires_unexpected_dot_star(monkeypatch, capsys, op): - monkeypatch.setattr(cpv, 'CURRENT_PYTHON_3_VERSION', 7) - assert cpv.parse_python_requires(f'{op} 3.6.*') is None - assert ( - f'Bad python_requires specifier: {op} 3.6.* ({op} does not allow a .*)' - in capsys.readouterr().err - ) - - -@pytest.mark.parametrize('specifier', [ - '%= 42', - '== nobody.knows', - '!= *.*.*', - 'xyzzy', -]) -def test_parse_python_requires_syntax_errors(capsys, specifier): - assert cpv.parse_python_requires(specifier) is None - assert ( - f'Bad python_requires specifier: {specifier}' - in capsys.readouterr().err - ) - - -def test_get_tox_ini_python_versions(tmp_path): - tox_ini = tmp_path / "tox.ini" - tox_ini.write_text(textwrap.dedent("""\ - [tox] - envlist = py27,py36,py27-docs - """)) - assert cpv.get_tox_ini_python_versions(tox_ini) == ['2.7', '3.6'] - - -def test_get_tox_ini_python_versions_no_tox_ini(tmp_path): - tox_ini = tmp_path / "tox.ini" - assert cpv.get_tox_ini_python_versions(tox_ini) == [] - - -def test_get_tox_ini_python_versions_syntax_error(tmp_path): - tox_ini = tmp_path / "tox.ini" - tox_ini.write_text(textwrap.dedent("""\ - ... - """)) - assert cpv.get_tox_ini_python_versions(tox_ini) == [] - - -def test_get_tox_ini_python_versions_no_tox_section(tmp_path): - tox_ini = tmp_path / "tox.ini" - tox_ini.write_text(textwrap.dedent("""\ - [flake8] - source = foo - """)) - assert cpv.get_tox_ini_python_versions(tox_ini) == [] - - -def test_get_tox_ini_python_versions_no_tox_envlist(tmp_path): - tox_ini = tmp_path / "tox.ini" - tox_ini.write_text(textwrap.dedent("""\ - [tox] - minversion = 3.4.0 - """)) - assert cpv.get_tox_ini_python_versions(tox_ini) == [] - - -@pytest.mark.parametrize('s, expected', [ - ('', []), - ('py36,py37', ['py36', 'py37']), - ('py36, py37', ['py36', 'py37']), - ('\n py36,\n py37', ['py36', 'py37']), - ('py3{6,7},pypy', ['py36', 'py37', 'pypy']), -]) -def test_parse_envlist(s, expected): - assert cpv.parse_envlist(s) == expected - - -@pytest.mark.parametrize('s, expected', [ - ('', ['']), - ('py36', ['py36']), - ('py3{6,7}', ['py36', 'py37']), - ('py3{6,7}-lint', ['py36-lint', 'py37-lint']), - ('py3{6,7}{,-lint}', ['py36', 'py36-lint', 'py37', 'py37-lint']), -]) -def test_brace_expand(s, expected): - assert cpv.brace_expand(s) == expected - - -@pytest.mark.parametrize('s, expected', [ - ('py36', '3.6'), - ('py37-lint', '3.7'), - ('pypy', 'PyPy'), - ('pypy3', 'PyPy3'), - ('flake8', 'flake8'), -]) -def test_tox_env_to_py_version(s, expected): - assert cpv.tox_env_to_py_version(s) == expected - - -@needs_pyyaml -def test_get_travis_yml_python_versions(tmp_path): - travis_yml = tmp_path / ".travis.yml" - travis_yml.write_text(textwrap.dedent("""\ - python: - - 2.7 - - 3.6 - matrix: - include: - - python: 3.7 - - name: something unrelated - jobs: - include: - - python: 3.4 - - name: something unrelated - env: - - TOXENV=py35-docs - - UNRELATED=variable - """)) - assert cpv.get_travis_yml_python_versions(travis_yml) == [ - '2.7', '3.4', '3.5', '3.6', '3.7', - ] - - -@needs_pyyaml -def test_get_travis_yml_python_versions_no_python_only_matrix(tmp_path): - travis_yml = tmp_path / ".travis.yml" - travis_yml.write_text(textwrap.dedent("""\ - matrix: - include: - - python: 3.7 - """)) - assert cpv.get_travis_yml_python_versions(travis_yml) == [ - '3.7', - ] - - -@pytest.mark.parametrize('s, expected', [ - (3.6, '3.6'), - ('3.7', '3.7'), - ('pypy', 'PyPy'), - ('pypy2', 'PyPy'), - ('pypy2.7', 'PyPy'), - ('pypy2.7-5.10.0', 'PyPy'), - ('pypy3', 'PyPy3'), - ('pypy3.5', 'PyPy3'), - ('pypy3.5-5.10.1', 'PyPy3'), - ('3.7-dev', '3.7-dev'), - ('nightly', 'nightly'), -]) -def test_travis_normalize_py_version(s, expected): - assert cpv.travis_normalize_py_version(s) == expected - - -@needs_pyyaml -def test_get_appveyor_yml_python_versions(tmp_path): - appveyor_yml = tmp_path / "appveyor.yml" - appveyor_yml.write_text(textwrap.dedent("""\ - environment: - matrix: - - PYTHON: c:\\python27 - - PYTHON: c:\\python27-x64 - - PYTHON: c:\\python36 - - PYTHON: c:\\python36-x64 - UNRELATED: variable - """)) - assert cpv.get_appveyor_yml_python_versions(appveyor_yml) == [ - '2.7', '3.6', - ] - - -@needs_pyyaml -def test_get_appveyor_yml_python_versions_using_toxenv(tmp_path): - appveyor_yml = tmp_path / "appveyor.yml" - appveyor_yml.write_text(textwrap.dedent("""\ - environment: - matrix: - - TOXENV: py27 - - TOXENV: py37 - """)) - assert cpv.get_appveyor_yml_python_versions(appveyor_yml) == [ - '2.7', '3.7', - ] - - -@pytest.mark.parametrize('s, expected', [ - ('37', '3.7'), - ('c:\\python34', '3.4'), - ('C:\\Python27\\', '2.7'), - ('C:\\Python27-x64', '2.7'), - ('C:\\PYTHON34-X64', '3.4'), -]) -def test_appveyor_normalize_py_version(s, expected): - assert cpv.appveyor_normalize_py_version(s) == expected - - -def test_get_manylinux_python_versions(tmp_path): - manylinux_install_sh = tmp_path / ".manylinux-install.sh" - manylinux_install_sh.write_text(textwrap.dedent(r""" - #!/usr/bin/env bash - - set -e -x - - # Compile wheels - for PYBIN in /opt/python/*/bin; do - if [[ "${PYBIN}" == *"cp27"* ]] || \ - [[ "${PYBIN}" == *"cp34"* ]] || \ - [[ "${PYBIN}" == *"cp35"* ]] || \ - [[ "${PYBIN}" == *"cp36"* ]] || \ - [[ "${PYBIN}" == *"cp37"* ]]; then - "${PYBIN}/pip" install -e /io/ - "${PYBIN}/pip" wheel /io/ -w wheelhouse/ - rm -rf /io/build /io/*.egg-info - fi - done - - # Bundle external shared libraries into the wheels - for whl in wheelhouse/zope.interface*.whl; do - auditwheel repair "$whl" -w /io/wheelhouse/ - done - """.lstrip('\n'))) - assert cpv.get_manylinux_python_versions(manylinux_install_sh) == [ - '2.7', '3.4', '3.5', '3.6', '3.7', - ] - - -def test_important(monkeypatch): - monkeypatch.setattr(cpv, 'CURRENT_PYTHON_3_VERSION', 7) - assert cpv.important({ - '2.7', '3.4', '3.7-dev', '3.8', 'nightly', 'PyPy3', 'Jython' - }) == {'2.7', '3.4'} - - -def test_parse_version_list(): - assert cpv.parse_version_list( - '2.7,3.4-3.6' - ) == ['2.7', '3.4', '3.5', '3.6'] - - -def test_parse_version_list_magic_range(monkeypatch): - monkeypatch.setattr(cpv, 'CURRENT_PYTHON_3_VERSION', 7) - assert cpv.parse_version_list( - '2.7,3.4-' - ) == ['2.7', '3.4', '3.5', '3.6', '3.7'] - assert cpv.parse_version_list( - '2.6,-3.4' - ) == ['2.6', '3.0', '3.1', '3.2', '3.3', '3.4'] - - -@pytest.mark.parametrize('v', [ - '4.1-', # unknown major version - '-', # both endpoints missing - '2.7-3.4', # major versions differ -]) -def test_parse_version_list_bad_range(v): - with pytest.raises(argparse.ArgumentTypeError, - match=re.escape(f'bad range: {v}')): - cpv.parse_version_list(v) - - - -def test_parse_version_list_bad_number(): - with pytest.raises(argparse.ArgumentTypeError): - cpv.parse_version_list('2.x') - - -def test_parse_version_list_too_few(): - with pytest.raises(argparse.ArgumentTypeError): - cpv.parse_version_list('2') - - -def test_parse_version_list_too_many_dots(): - with pytest.raises(argparse.ArgumentTypeError): - cpv.parse_version_list('2.7.1') - - -def test_update_version_list(): - assert cpv.update_version_list(['2.7', '3.4']) == ['2.7', '3.4'] - assert cpv.update_version_list(['2.7', '3.4'], add=['3.4', '3.5']) == [ - '2.7', '3.4', '3.5', - ] - assert cpv.update_version_list(['2.7', '3.4'], drop=['3.4', '3.5']) == [ - '2.7', - ] - assert cpv.update_version_list(['2.7', '3.4'], add=['3.5'], - drop=['2.7']) == [ - '3.4', '3.5', - ] - assert cpv.update_version_list(['2.7', '3.4'], drop=['3.4', '3.5']) == [ - '2.7', - ] - assert cpv.update_version_list(['2.7', '3.4'], update=['3.4', '3.5']) == [ - '3.4', '3.5', - ] - - -def test_is_package(tmp_path): - (tmp_path / "setup.py").write_text("") - assert cpv.is_package(tmp_path) - - -def test_is_package_no_setup_py(tmp_path): - assert not cpv.is_package(tmp_path) - - -def test_check_not_a_directory(tmp_path, capsys): - assert not cpv.check_package(tmp_path / "xyzzy") - assert capsys.readouterr().out == 'not a directory\n' - - -def test_check_not_a_package(tmp_path, capsys): - assert not cpv.check_package(tmp_path) - assert capsys.readouterr().out == 'no setup.py -- not a Python package?\n' - - -def test_check_package(tmp_path): - setup_py = tmp_path / "setup.py" - setup_py.write_text(textwrap.dedent("""\ - from setuptools import setup - setup( - name='foo', - ) - """)) - assert cpv.check_package(tmp_path) is True - - -def test_check_unknown(tmp_path, capsys): - setup_py = tmp_path / "setup.py" - setup_py.write_text(textwrap.dedent("""\ - from setuptools import setup - setup( - name='foo', - ) - """)) - assert cpv.check_versions(tmp_path) is True - assert capsys.readouterr().out == textwrap.dedent("""\ - setup.py says: (empty) - """) - - -def test_check_minimal(tmp_path, capsys): - setup_py = tmp_path / "setup.py" - setup_py.write_text(textwrap.dedent("""\ - from setuptools import setup - setup( - name='foo', - classifiers=[ - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3.6', - ], - ) - """)) - assert cpv.check_versions(tmp_path) is True - assert capsys.readouterr().out == textwrap.dedent("""\ - setup.py says: 2.7, 3.6 - """) - - -def test_check_mismatch(tmp_path, capsys): - setup_py = tmp_path / "setup.py" - setup_py.write_text(textwrap.dedent("""\ - from setuptools import setup - setup( - name='foo', - classifiers=[ - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3.6', - ], - ) - """)) - tox_ini = tmp_path / "tox.ini" - tox_ini.write_text(textwrap.dedent("""\ - [tox] - envlist = py27 - """)) - assert cpv.check_versions(tmp_path) is False - assert capsys.readouterr().out == textwrap.dedent("""\ - setup.py says: 2.7, 3.6 - tox.ini says: 2.7 - """) - - -def test_check_expectation(tmp_path, capsys): - setup_py = tmp_path / "setup.py" - setup_py.write_text(textwrap.dedent("""\ - from setuptools import setup - setup( - name='foo', - classifiers=[ - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3.6', - ], - ) - """)) - assert cpv.check_versions(tmp_path, expect=['2.7', '3.6', '3.7']) is False - assert capsys.readouterr().out == textwrap.dedent("""\ - setup.py says: 2.7, 3.6 - expected: 2.7, 3.6, 3.7 - """) - - -def test_main_help(monkeypatch): - monkeypatch.setattr(sys, 'argv', ['check-python-versions', '--help']) - with pytest.raises(SystemExit): - cpv.main() - - -@pytest.mark.parametrize('arg', [ - 'xyzzy', - '1,2,3', - '2.x', - '1.2.3', - '2.7-3.6', -]) -def test_main_expect_error_handling(monkeypatch, arg, capsys): - monkeypatch.setattr(sys, 'argv', [ - 'check-python-versions', '--expect', arg, - ]) - with pytest.raises(SystemExit): - cpv.main() - # the error is either 'bad version: ...' or 'bad range: ...' - assert f'--expect: bad' in capsys.readouterr().err - - -def test_main_here(monkeypatch, capsys): - monkeypatch.setattr(sys, 'argv', [ - 'check-python-versions', - ]) - cpv.main() - assert 'mismatch' not in capsys.readouterr().out - - -def test_main_skip_non_packages(monkeypatch, capsys, tmp_path): - monkeypatch.setattr(sys, 'argv', [ - 'check-python-versions', '--skip-non-packages', str(tmp_path), - ]) - cpv.main() - assert capsys.readouterr().out == '' - - -def test_main_single(monkeypatch, capsys, tmp_path): - monkeypatch.setattr(sys, 'argv', [ - 'check-python-versions', - str(tmp_path / "a"), - ]) - with pytest.raises(SystemExit) as exc_info: - cpv.main() - assert ( - capsys.readouterr().out + str(exc_info.value) + '\n' - ).replace(str(tmp_path), 'tmp') == textwrap.dedent("""\ - not a directory - - mismatch! - """) - - -def test_main_multiple(monkeypatch, capsys, tmp_path): - monkeypatch.setattr(sys, 'argv', [ - 'check-python-versions', - str(tmp_path / "a"), - str(tmp_path / "b"), - '--expect', '3.6, 3.7' - ]) - (tmp_path / "a").mkdir() - (tmp_path / "a" / "setup.py").write_text(textwrap.dedent("""\ - from setuptools import setup - setup( - name='foo', - classifiers=[ - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3.6', - ], - ) - """)) - with pytest.raises(SystemExit) as exc_info: - cpv.main() - assert ( - capsys.readouterr().out + str(exc_info.value) + '\n' - ).replace(str(tmp_path) + os.path.sep, 'tmp/') == textwrap.dedent("""\ - tmp/a: - - setup.py says: 2.7, 3.6 - expected: 3.6, 3.7 - - - tmp/b: - - not a directory - - - mismatch in tmp/a tmp/b! - """) - - -def test_main_multiple_ok(monkeypatch, capsys): - monkeypatch.setattr(sys, 'argv', [ - 'check-python-versions', '.', '.', - ]) - cpv.main() - assert ( - capsys.readouterr().out.endswith('\n\nall ok!\n') - ) diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..8906175 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,10 @@ +import pytest + +from check_python_versions.versions import MAX_MINOR_FOR_MAJOR + + +@pytest.fixture +def fix_max_python_3_version(monkeypatch): + def helper(ver): + monkeypatch.setitem(MAX_MINOR_FOR_MAJOR, 3, ver) + return helper diff --git a/tests/parsers/test_appveyor.py b/tests/parsers/test_appveyor.py new file mode 100644 index 0000000..ad7290a --- /dev/null +++ b/tests/parsers/test_appveyor.py @@ -0,0 +1,58 @@ +import textwrap + +import pytest + +try: + import yaml +except ImportError: + yaml = None + +from check_python_versions.parsers.appveyor import ( + appveyor_normalize_py_version, + get_appveyor_yml_python_versions, +) + + +needs_pyyaml = pytest.mark.skipIf(yaml is None, "PyYAML not installed") + + +@needs_pyyaml +def test_get_appveyor_yml_python_versions(tmp_path): + appveyor_yml = tmp_path / "appveyor.yml" + appveyor_yml.write_text(textwrap.dedent("""\ + environment: + matrix: + - PYTHON: c:\\python27 + - PYTHON: c:\\python27-x64 + - PYTHON: c:\\python36 + - PYTHON: c:\\python36-x64 + UNRELATED: variable + """)) + assert get_appveyor_yml_python_versions(appveyor_yml) == [ + '2.7', '3.6', + ] + + +@needs_pyyaml +def test_get_appveyor_yml_python_versions_using_toxenv(tmp_path): + appveyor_yml = tmp_path / "appveyor.yml" + appveyor_yml.write_text(textwrap.dedent("""\ + environment: + matrix: + - TOXENV: py27 + - TOXENV: py37 + """)) + assert get_appveyor_yml_python_versions(appveyor_yml) == [ + '2.7', '3.7', + ] + + +@pytest.mark.parametrize('s, expected', [ + ('37', '3.7'), + ('c:\\python34', '3.4'), + ('C:\\Python27\\', '2.7'), + ('C:\\Python27-x64', '2.7'), + ('C:\\PYTHON34-X64', '3.4'), +]) +def test_appveyor_normalize_py_version(s, expected): + assert appveyor_normalize_py_version(s) == expected diff --git a/tests/parsers/test_manylinux.py b/tests/parsers/test_manylinux.py new file mode 100644 index 0000000..5832ed0 --- /dev/null +++ b/tests/parsers/test_manylinux.py @@ -0,0 +1,35 @@ +import textwrap + +from check_python_versions.parsers.manylinux import ( + get_manylinux_python_versions, +) + + +def test_get_manylinux_python_versions(tmp_path): + manylinux_install_sh = tmp_path / ".manylinux-install.sh" + manylinux_install_sh.write_text(textwrap.dedent(r""" + #!/usr/bin/env bash + + set -e -x + + # Compile wheels + for PYBIN in /opt/python/*/bin; do + if [[ "${PYBIN}" == *"cp27"* ]] || \ + [[ "${PYBIN}" == *"cp34"* ]] || \ + [[ "${PYBIN}" == *"cp35"* ]] || \ + [[ "${PYBIN}" == *"cp36"* ]] || \ + [[ "${PYBIN}" == *"cp37"* ]]; then + "${PYBIN}/pip" install -e /io/ + "${PYBIN}/pip" wheel /io/ -w wheelhouse/ + rm -rf /io/build /io/*.egg-info + fi + done + + # Bundle external shared libraries into the wheels + for whl in wheelhouse/zope.interface*.whl; do + auditwheel repair "$whl" -w /io/wheelhouse/ + done + """.lstrip('\n'))) + assert get_manylinux_python_versions(manylinux_install_sh) == [ + '2.7', '3.4', '3.5', '3.6', '3.7', + ] diff --git a/tests/parsers/test_python.py b/tests/parsers/test_python.py new file mode 100644 index 0000000..0421342 --- /dev/null +++ b/tests/parsers/test_python.py @@ -0,0 +1,493 @@ +import ast +import textwrap + +import pytest + +from check_python_versions.parsers.python import ( + eval_ast_node, + find_call_kwarg_in_ast, + get_python_requires, + get_setup_py_keyword, + get_supported_python_versions, + get_versions_from_classifiers, + parse_python_requires, + to_literal, + update_call_arg_in_source, + update_classifiers, +) + + +def test_get_supported_python_versions(tmp_path): + (tmp_path / "setup.py").write_text(textwrap.dedent("""\ + from setuptools import setup + setup( + name='foo', + classifiers=[ + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.6', + ], + ) + """)) + assert get_supported_python_versions(tmp_path) == ['2.7', '3.6'] + + +def test_get_supported_python_versions_computed(tmp_path): + (tmp_path / "setup.py").write_text(textwrap.dedent("""\ + from setuptools import setup + setup( + name='foo', + classifiers=[ + 'Programming Language :: Python :: %s' % v + for v in ['2.7', '3.7'] + ], + ) + """)) + assert get_supported_python_versions(tmp_path) == ['2.7', '3.7'] + + +def test_get_versions_from_classifiers(): + assert get_versions_from_classifiers([ + 'Development Status :: 4 - Beta', + 'Environment :: Console', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: GNU General Public License (GPL)', + 'Operating System :: OS Independent', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: Implementation :: CPython', + 'Programming Language :: Python :: Implementation :: PyPy', + ]) == ['2.7', '3.6', '3.7', 'PyPy'] + + +def test_get_versions_from_classifiers_major_only(): + assert get_versions_from_classifiers([ + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 3', + ]) == ['2', '3'] + + +def test_get_versions_from_classifiers_with_only_suffix(): + assert get_versions_from_classifiers([ + 'Programming Language :: Python :: 2 :: Only', + ]) == ['2'] + + +def test_get_versions_from_classifiers_with_trailing_whitespace(): + # I was surprised too that this is allowed! + assert get_versions_from_classifiers([ + 'Programming Language :: Python :: 3.6 ', + ]) == ['3.6'] + + +def test_update_classifiers(): + assert update_classifiers([ + 'Development Status :: 4 - Beta', + 'Environment :: Console', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: GNU General Public License (GPL)', + 'Operating System :: OS Independent', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: Implementation :: CPython', + 'Programming Language :: Python :: Implementation :: PyPy', + 'Typing :: Typed', + ], ['2.7', '3.7']) == [ + 'Development Status :: 4 - Beta', + 'Environment :: Console', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: GNU General Public License (GPL)', + 'Operating System :: OS Independent', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: Implementation :: CPython', + 'Programming Language :: Python :: Implementation :: PyPy', + 'Typing :: Typed', + ] + + +def test_update_classifiers_drop_major(): + assert update_classifiers([ + 'Development Status :: 4 - Beta', + 'Environment :: Console', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: GNU General Public License (GPL)', + 'Operating System :: OS Independent', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: Implementation :: CPython', + 'Programming Language :: Python :: Implementation :: PyPy', + 'Typing :: Typed', + ], ['3.6', '3.7']) == [ + 'Development Status :: 4 - Beta', + 'Environment :: Console', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: GNU General Public License (GPL)', + 'Operating System :: OS Independent', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: Implementation :: CPython', + 'Programming Language :: Python :: Implementation :: PyPy', + 'Typing :: Typed', + ] + + +def test_update_classifiers_no_major(): + assert update_classifiers([ + 'Development Status :: 4 - Beta', + 'Environment :: Console', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: GNU General Public License (GPL)', + 'Operating System :: OS Independent', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: Implementation :: CPython', + 'Programming Language :: Python :: Implementation :: PyPy', + 'Typing :: Typed', + ], ['2.7', '3.7']) == [ + 'Development Status :: 4 - Beta', + 'Environment :: Console', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: GNU General Public License (GPL)', + 'Operating System :: OS Independent', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: Implementation :: CPython', + 'Programming Language :: Python :: Implementation :: PyPy', + 'Typing :: Typed', + ] + + +def test_update_classifiers_none_were_present(): + assert update_classifiers([ + 'Development Status :: 4 - Beta', + 'Environment :: Console', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: GNU General Public License (GPL)', + 'Operating System :: OS Independent', + ], ['2.7', '3.7']) == [ + 'Development Status :: 4 - Beta', + 'Environment :: Console', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: GNU General Public License (GPL)', + 'Operating System :: OS Independent', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.7', + ] + + +def test_get_python_requires(tmp_path, fix_max_python_3_version): + setup_py = tmp_path / "setup.py" + setup_py.write_text(textwrap.dedent("""\ + from setuptools import setup + setup( + name='foo', + python_requires='>= 3.6', + ) + """)) + fix_max_python_3_version(7) + assert get_python_requires(setup_py) == ['3.6', '3.7'] + + +def test_get_python_requires_not_specified(tmp_path, capsys): + setup_py = tmp_path / "setup.py" + setup_py.write_text(textwrap.dedent("""\ + from setuptools import setup + setup( + name='foo', + ) + """)) + assert get_python_requires(setup_py) is None + assert capsys.readouterr().err == '' + + +def test_get_setup_py_keyword_syntax_error(tmp_path, capsys): + setup_py = tmp_path / "setup.py" + setup_py.write_text(textwrap.dedent("""\ + from setuptools import setup + setup( + name='foo', + # uh do I need to close parens? what if I forget? ;) + """)) + assert get_setup_py_keyword(setup_py, 'name') is None + assert 'Could not parse' in capsys.readouterr().err + + +def test_find_call_kwarg_in_ast(): + tree = ast.parse('foo(bar="foo")') + ast.dump(tree) + node = find_call_kwarg_in_ast(tree, 'foo', 'bar') + assert isinstance(node, ast.Str) + assert node.s == "foo" + + +def test_find_call_kwarg_in_ast_no_arg(capsys): + tree = ast.parse('foo(baz="foo")') + ast.dump(tree) + node = find_call_kwarg_in_ast(tree, 'foo', 'bar') + assert node is None + assert capsys.readouterr().err == '' + + +def test_find_call_kwarg_in_ast_no_call(capsys): + tree = ast.parse('fooo(bar="foo")') + ast.dump(tree) + node = find_call_kwarg_in_ast(tree, 'foo', 'bar') + assert node is None + assert 'Could not find foo() call in setup.py' in capsys.readouterr().err + + +@pytest.mark.parametrize('code, expected', [ + ('"hi"', "hi"), + ('"hi\\n"', "hi\n"), + ('["a", "b"]', ["a", "b"]), + ('("a", "b")', ("a", "b")), + ('"-".join(["a", "b"])', "a-b"), +]) +def test_eval_ast_node(code, expected): + tree = ast.parse(f'foo(bar={code})') + node = find_call_kwarg_in_ast(tree, 'foo', 'bar') + assert node is not None + assert eval_ast_node(node, 'bar') == expected + + +def test_to_literal(): + assert to_literal("blah") == '"blah"' + assert to_literal("blah", "'") == "'blah'" + + +def test_to_literal_embedded_quote(): + assert to_literal( + "Environment :: Handhelds/PDA's" + ) == '"Environment :: Handhelds/PDA\'s"' + assert to_literal( + "Environment :: Handhelds/PDA's", "'" + ) == '"Environment :: Handhelds/PDA\'s"' + + +def test_to_literal_all_the_classifiers(): + with open('CLASSIFIERS') as f: + for line in f: + classifier = line.strip() + literal = to_literal(classifier) + assert ast.literal_eval(literal) == classifier + + +def test_update_call_arg_in_source(): + source_lines = textwrap.dedent("""\ + setup( + foo=1, + bar=[ + "a", + "b", + "c", + ], + baz=2, + ) + """).splitlines(True) + result = update_call_arg_in_source(source_lines, "setup", "bar", + ["x", "y"]) + assert "".join(result) == textwrap.dedent("""\ + setup( + foo=1, + bar=[ + "x", + "y", + ], + baz=2, + ) + """) + + +def test_update_call_arg_in_source_preserves_indent_and_quote_style(): + source_lines = textwrap.dedent("""\ + setup(foo=1, + bar=[ + 'a', + 'b', + 'c', + ], + ) + """).splitlines(True) + result = update_call_arg_in_source(source_lines, "setup", "bar", + ["x", "y"]) + assert "".join(result) == textwrap.dedent("""\ + setup(foo=1, + bar=[ + 'x', + 'y', + ], + ) + """) + + +def test_update_call_arg_in_source_fixes_closing_bracket(): + source_lines = textwrap.dedent("""\ + setup(foo=1, + bar=[ + 'a', + 'b', + 'c'], + baz=2, + ) + """).splitlines(True) + result = update_call_arg_in_source(source_lines, "setup", "bar", + ["x", "y"]) + assert "".join(result) == textwrap.dedent("""\ + setup(foo=1, + bar=[ + 'x', + 'y', + ], + baz=2, + ) + """) + + +def test_update_call_arg_in_source_fixes_opening_bracket(): + source_lines = textwrap.dedent("""\ + setup(foo=1, + bar=['a', + 'b', + 'c'], + baz=2, + ) + """).splitlines(True) + result = update_call_arg_in_source(source_lines, "setup", "bar", + ["x", "y"]) + assert "".join(result) == textwrap.dedent("""\ + setup(foo=1, + bar=[ + 'x', + 'y', + ], + baz=2, + ) + """) + + +@pytest.mark.parametrize('code', [ + '[2 * 2]', + '"".join([2 * 2])', +]) +def test_eval_ast_node_failures(code, capsys): + tree = ast.parse(f'foo(bar={code})') + node = find_call_kwarg_in_ast(tree, 'foo', 'bar') + assert eval_ast_node(node, 'bar') is None + assert 'Non-literal bar= passed to setup()' in capsys.readouterr().err + + +@pytest.mark.parametrize('constraint, result', [ + ('~= 2.7', ['2.7']), + ('~= 2.7.12', ['2.7']), +]) +def test_parse_python_requires_approximately(constraint, result): + assert parse_python_requires(constraint) == result + + +def test_parse_python_requires_approximately_not_enough_dots(capsys): + assert parse_python_requires('~= 2') is None + assert ( + 'Bad python_requires specifier: ~= 2' + ' (~= requires a version with at least one dot)' + in capsys.readouterr().err + ) + + +@pytest.mark.parametrize('constraint, result', [ + ('== 2.7', ['2.7']), + ('== 2.7.*', ['2.7']), + ('== 2.7.12', ['2.7']), + ('== 2.*, >= 2.6', ['2.6', '2.7']), + ('== 3.0', ['3.0']), + ('== 3', ['3.0']), +]) +def test_parse_python_requires_matching_version(constraint, result): + assert parse_python_requires(constraint) == result + + +def test_parse_python_requires_greater_than(fix_max_python_3_version): + fix_max_python_3_version(8) + assert parse_python_requires('>= 3.6') == ['3.6', '3.7', '3.8'] + + +@pytest.mark.parametrize('constraint, result', [ + ('>= 2.7, != 3.*', ['2.7']), + ('>= 2.7.12, != 3.*', ['2.7']), + ('>= 2.7, != 3.0.*, != 3.1.*', ['2.7', '3.2', '3.3']), + # != 3.2 means we reject 3.2.0 but still accept any other 3.2.x + ('>= 2.7, != 3.2', ['2.7', '3.0', '3.1', '3.2', '3.3']), + ('>= 2.7, != 3.2.1', ['2.7', '3.0', '3.1', '3.2', '3.3']), + ('>= 2.7, <= 3', ['2.7', '3.0']), + ('>= 2.7, <= 3.2', ['2.7', '3.0', '3.1', '3.2']), + ('>= 2.7, <= 3.2.1', ['2.7', '3.0', '3.1', '3.2']), + ('>= 3', ['3.0', '3.1', '3.2', '3.3']), +]) +def test_parse_python_requires_greater_than_with_exceptions( + fix_max_python_3_version, constraint, result +): + fix_max_python_3_version(3) + assert parse_python_requires(constraint) == result + + +def test_parse_python_requires_multiple_greater_than(fix_max_python_3_version): + fix_max_python_3_version(7) + assert parse_python_requires('>= 2.7, >= 3.6') == ['3.6', '3.7'] + + +@pytest.mark.parametrize('constraint, result', [ + ('> 2, < 3.1', ['3.0']), + ('> 2.6, < 3', ['2.7']), + ('> 2.7.12, < 3', ['2.7']), + ('> 2.7.12, < 3.0', ['2.7']), + ('> 2.7.12, < 3.1', ['2.7', '3.0']), + ('> 2.7.12, < 3.0.1', ['2.7', '3.0']), +]) +def test_parse_python_requires_exclusive_ordering(constraint, result): + assert parse_python_requires(constraint) == result + + +@pytest.mark.parametrize('constraint, result', [ + ('=== 2.7', ['2.7']), + ('=== 2.7.12', ['2.7']), + ('=== 3', []), +]) +def test_parse_python_requires_arbitrary_version(constraint, result): + assert parse_python_requires(constraint) == result + + +@pytest.mark.parametrize('op', ['~=', '>=', '<=', '>', '<', '===']) +def test_parse_python_requires_unexpected_dot_star(fix_max_python_3_version, + capsys, op): + fix_max_python_3_version(7) + assert parse_python_requires(f'{op} 3.6.*') is None + assert ( + f'Bad python_requires specifier: {op} 3.6.* ({op} does not allow a .*)' + in capsys.readouterr().err + ) + + +@pytest.mark.parametrize('specifier', [ + '%= 42', + '== nobody.knows', + '!= *.*.*', + 'xyzzy', +]) +def test_parse_python_requires_syntax_errors(capsys, specifier): + assert parse_python_requires(specifier) is None + assert ( + f'Bad python_requires specifier: {specifier}' + in capsys.readouterr().err + ) diff --git a/tests/parsers/test_tox.py b/tests/parsers/test_tox.py new file mode 100644 index 0000000..238b7dc --- /dev/null +++ b/tests/parsers/test_tox.py @@ -0,0 +1,83 @@ +import textwrap + +import pytest + +from check_python_versions.parsers.tox import ( + brace_expand, + get_tox_ini_python_versions, + parse_envlist, + tox_env_to_py_version, +) + + +def test_get_tox_ini_python_versions(tmp_path): + tox_ini = tmp_path / "tox.ini" + tox_ini.write_text(textwrap.dedent("""\ + [tox] + envlist = py27,py36,py27-docs + """)) + assert get_tox_ini_python_versions(tox_ini) == ['2.7', '3.6'] + + +def test_get_tox_ini_python_versions_no_tox_ini(tmp_path): + tox_ini = tmp_path / "tox.ini" + assert get_tox_ini_python_versions(tox_ini) == [] + + +def test_get_tox_ini_python_versions_syntax_error(tmp_path): + tox_ini = tmp_path / "tox.ini" + tox_ini.write_text(textwrap.dedent("""\ + ... + """)) + assert get_tox_ini_python_versions(tox_ini) == [] + + +def test_get_tox_ini_python_versions_no_tox_section(tmp_path): + tox_ini = tmp_path / "tox.ini" + tox_ini.write_text(textwrap.dedent("""\ + [flake8] + source = foo + """)) + assert get_tox_ini_python_versions(tox_ini) == [] + + +def test_get_tox_ini_python_versions_no_tox_envlist(tmp_path): + tox_ini = tmp_path / "tox.ini" + tox_ini.write_text(textwrap.dedent("""\ + [tox] + minversion = 3.4.0 + """)) + assert get_tox_ini_python_versions(tox_ini) == [] + + +@pytest.mark.parametrize('s, expected', [ + ('', []), + ('py36,py37', ['py36', 'py37']), + ('py36, py37', ['py36', 'py37']), + ('\n py36,\n py37', ['py36', 'py37']), + ('py3{6,7},pypy', ['py36', 'py37', 'pypy']), +]) +def test_parse_envlist(s, expected): + assert parse_envlist(s) == expected + + +@pytest.mark.parametrize('s, expected', [ + ('', ['']), + ('py36', ['py36']), + ('py3{6,7}', ['py36', 'py37']), + ('py3{6,7}-lint', ['py36-lint', 'py37-lint']), + ('py3{6,7}{,-lint}', ['py36', 'py36-lint', 'py37', 'py37-lint']), +]) +def test_brace_expand(s, expected): + assert brace_expand(s) == expected + + +@pytest.mark.parametrize('s, expected', [ + ('py36', '3.6'), + ('py37-lint', '3.7'), + ('pypy', 'PyPy'), + ('pypy3', 'PyPy3'), + ('flake8', 'flake8'), +]) +def test_tox_env_to_py_version(s, expected): + assert tox_env_to_py_version(s) == expected diff --git a/tests/parsers/test_travis.py b/tests/parsers/test_travis.py new file mode 100644 index 0000000..a9311a1 --- /dev/null +++ b/tests/parsers/test_travis.py @@ -0,0 +1,70 @@ +import textwrap + +import pytest + +try: + import yaml +except ImportError: + yaml = None + +from check_python_versions.parsers.travis import ( + get_travis_yml_python_versions, + travis_normalize_py_version, +) + + +needs_pyyaml = pytest.mark.skipIf(yaml is None, "PyYAML not installed") + + +@needs_pyyaml +def test_get_travis_yml_python_versions(tmp_path): + travis_yml = tmp_path / ".travis.yml" + travis_yml.write_text(textwrap.dedent("""\ + python: + - 2.7 + - 3.6 + matrix: + include: + - python: 3.7 + - name: something unrelated + jobs: + include: + - python: 3.4 + - name: something unrelated + env: + - TOXENV=py35-docs + - UNRELATED=variable + """)) + assert get_travis_yml_python_versions(travis_yml) == [ + '2.7', '3.4', '3.5', '3.6', '3.7', + ] + + +@needs_pyyaml +def test_get_travis_yml_python_versions_no_python_only_matrix(tmp_path): + travis_yml = tmp_path / ".travis.yml" + travis_yml.write_text(textwrap.dedent("""\ + matrix: + include: + - python: 3.7 + """)) + assert get_travis_yml_python_versions(travis_yml) == [ + '3.7', + ] + + +@pytest.mark.parametrize('s, expected', [ + (3.6, '3.6'), + ('3.7', '3.7'), + ('pypy', 'PyPy'), + ('pypy2', 'PyPy'), + ('pypy2.7', 'PyPy'), + ('pypy2.7-5.10.0', 'PyPy'), + ('pypy3', 'PyPy3'), + ('pypy3.5', 'PyPy3'), + ('pypy3.5-5.10.1', 'PyPy3'), + ('3.7-dev', '3.7-dev'), + ('nightly', 'nightly'), +]) +def test_travis_normalize_py_version(s, expected): + assert travis_normalize_py_version(s) == expected diff --git a/tests/test___main__.py b/tests/test___main__.py new file mode 100644 index 0000000..b922bac --- /dev/null +++ b/tests/test___main__.py @@ -0,0 +1,11 @@ +import sys + +import pytest + +from check_python_versions import __main__ + + +def test_main(monkeypatch): + monkeypatch.setattr(sys, 'argv', ['check-python-versions', '--help']) + with pytest.raises(SystemExit): + __main__.main() diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..7a84c8e --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,259 @@ +import argparse +import os +import re +import sys +import textwrap + +import pytest + +import check_python_versions.cli as cpv + + +def test_parse_version_list(): + assert cpv.parse_version_list( + '2.7,3.4-3.6' + ) == ['2.7', '3.4', '3.5', '3.6'] + + +def test_parse_version_list_magic_range(fix_max_python_3_version): + fix_max_python_3_version(7) + assert cpv.parse_version_list( + '2.7,3.4-' + ) == ['2.7', '3.4', '3.5', '3.6', '3.7'] + assert cpv.parse_version_list( + '2.6,-3.4' + ) == ['2.6', '3.0', '3.1', '3.2', '3.3', '3.4'] + + +@pytest.mark.parametrize('v', [ + '4.1-', # unknown major version + '-', # both endpoints missing + '2.7-3.4', # major versions differ +]) +def test_parse_version_list_bad_range(v): + with pytest.raises(argparse.ArgumentTypeError, + match=re.escape(f'bad range: {v}')): + cpv.parse_version_list(v) + + +def test_parse_version_list_bad_number(): + with pytest.raises(argparse.ArgumentTypeError): + cpv.parse_version_list('2.x') + + +def test_parse_version_list_too_few(): + with pytest.raises(argparse.ArgumentTypeError): + cpv.parse_version_list('2') + + +def test_parse_version_list_too_many_dots(): + with pytest.raises(argparse.ArgumentTypeError): + cpv.parse_version_list('2.7.1') + + +def test_is_package(tmp_path): + (tmp_path / "setup.py").write_text("") + assert cpv.is_package(tmp_path) + + +def test_is_package_no_setup_py(tmp_path): + assert not cpv.is_package(tmp_path) + + +def test_check_not_a_directory(tmp_path, capsys): + assert not cpv.check_package(tmp_path / "xyzzy") + assert capsys.readouterr().out == 'not a directory\n' + + +def test_check_not_a_package(tmp_path, capsys): + assert not cpv.check_package(tmp_path) + assert capsys.readouterr().out == 'no setup.py -- not a Python package?\n' + + +def test_check_package(tmp_path): + setup_py = tmp_path / "setup.py" + setup_py.write_text(textwrap.dedent("""\ + from setuptools import setup + setup( + name='foo', + ) + """)) + assert cpv.check_package(tmp_path) is True + + +def test_check_unknown(tmp_path, capsys): + setup_py = tmp_path / "setup.py" + setup_py.write_text(textwrap.dedent("""\ + from setuptools import setup + setup( + name='foo', + ) + """)) + assert cpv.check_versions(tmp_path) is True + assert capsys.readouterr().out == textwrap.dedent("""\ + setup.py says: (empty) + """) + + +def test_check_minimal(tmp_path, capsys): + setup_py = tmp_path / "setup.py" + setup_py.write_text(textwrap.dedent("""\ + from setuptools import setup + setup( + name='foo', + classifiers=[ + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.6', + ], + ) + """)) + assert cpv.check_versions(tmp_path) is True + assert capsys.readouterr().out == textwrap.dedent("""\ + setup.py says: 2.7, 3.6 + """) + + +def test_check_mismatch(tmp_path, capsys): + setup_py = tmp_path / "setup.py" + setup_py.write_text(textwrap.dedent("""\ + from setuptools import setup + setup( + name='foo', + classifiers=[ + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.6', + ], + ) + """)) + tox_ini = tmp_path / "tox.ini" + tox_ini.write_text(textwrap.dedent("""\ + [tox] + envlist = py27 + """)) + assert cpv.check_versions(tmp_path) is False + assert capsys.readouterr().out == textwrap.dedent("""\ + setup.py says: 2.7, 3.6 + tox.ini says: 2.7 + """) + + +def test_check_expectation(tmp_path, capsys): + setup_py = tmp_path / "setup.py" + setup_py.write_text(textwrap.dedent("""\ + from setuptools import setup + setup( + name='foo', + classifiers=[ + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.6', + ], + ) + """)) + assert cpv.check_versions(tmp_path, expect=['2.7', '3.6', '3.7']) is False + assert capsys.readouterr().out == textwrap.dedent("""\ + setup.py says: 2.7, 3.6 + expected: 2.7, 3.6, 3.7 + """) + + +def test_main_help(monkeypatch): + monkeypatch.setattr(sys, 'argv', ['check-python-versions', '--help']) + with pytest.raises(SystemExit): + cpv.main() + + +@pytest.mark.parametrize('arg', [ + 'xyzzy', + '1,2,3', + '2.x', + '1.2.3', + '2.7-3.6', +]) +def test_main_expect_error_handling(monkeypatch, arg, capsys): + monkeypatch.setattr(sys, 'argv', [ + 'check-python-versions', '--expect', arg, + ]) + with pytest.raises(SystemExit): + cpv.main() + # the error is either 'bad version: ...' or 'bad range: ...' + assert f'--expect: bad' in capsys.readouterr().err + + +def test_main_here(monkeypatch, capsys): + monkeypatch.setattr(sys, 'argv', [ + 'check-python-versions', + ]) + cpv.main() + assert 'mismatch' not in capsys.readouterr().out + + +def test_main_skip_non_packages(monkeypatch, capsys, tmp_path): + monkeypatch.setattr(sys, 'argv', [ + 'check-python-versions', '--skip-non-packages', str(tmp_path), + ]) + cpv.main() + assert capsys.readouterr().out == '' + + +def test_main_single(monkeypatch, capsys, tmp_path): + monkeypatch.setattr(sys, 'argv', [ + 'check-python-versions', + str(tmp_path / "a"), + ]) + with pytest.raises(SystemExit) as exc_info: + cpv.main() + assert ( + capsys.readouterr().out + str(exc_info.value) + '\n' + ).replace(str(tmp_path), 'tmp') == textwrap.dedent("""\ + not a directory + + mismatch! + """) + + +def test_main_multiple(monkeypatch, capsys, tmp_path): + monkeypatch.setattr(sys, 'argv', [ + 'check-python-versions', + str(tmp_path / "a"), + str(tmp_path / "b"), + '--expect', '3.6, 3.7' + ]) + (tmp_path / "a").mkdir() + (tmp_path / "a" / "setup.py").write_text(textwrap.dedent("""\ + from setuptools import setup + setup( + name='foo', + classifiers=[ + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.6', + ], + ) + """)) + with pytest.raises(SystemExit) as exc_info: + cpv.main() + assert ( + capsys.readouterr().out + str(exc_info.value) + '\n' + ).replace(str(tmp_path) + os.path.sep, 'tmp/') == textwrap.dedent("""\ + tmp/a: + + setup.py says: 2.7, 3.6 + expected: 3.6, 3.7 + + + tmp/b: + + not a directory + + + mismatch in tmp/a tmp/b! + """) + + +def test_main_multiple_ok(monkeypatch, capsys): + monkeypatch.setattr(sys, 'argv', [ + 'check-python-versions', '.', '.', + ]) + cpv.main() + assert ( + capsys.readouterr().out.endswith('\n\nall ok!\n') + ) diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 0000000..779b40a --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,5 @@ +from check_python_versions.utils import pipe + + +def test_pipe(): + assert pipe('echo', 'hi') == 'hi\n' diff --git a/tests/test_versions.py b/tests/test_versions.py new file mode 100644 index 0000000..31eb514 --- /dev/null +++ b/tests/test_versions.py @@ -0,0 +1,30 @@ +from check_python_versions.versions import ( + important, + update_version_list, +) + + +def test_important(fix_max_python_3_version): + fix_max_python_3_version(7) + assert important({ + '2.7', '3.4', '3.7-dev', '3.8', 'nightly', 'PyPy3', 'Jython' + }) == {'2.7', '3.4'} + + +def test_update_version_list(): + assert update_version_list(['2.7', '3.4']) == ['2.7', '3.4'] + assert update_version_list(['2.7', '3.4'], add=['3.4', '3.5']) == [ + '2.7', '3.4', '3.5', + ] + assert update_version_list(['2.7', '3.4'], drop=['3.4', '3.5']) == [ + '2.7', + ] + assert update_version_list(['2.7', '3.4'], add=['3.5'], drop=['2.7']) == [ + '3.4', '3.5', + ] + assert update_version_list(['2.7', '3.4'], drop=['3.4', '3.5']) == [ + '2.7', + ] + assert update_version_list(['2.7', '3.4'], update=['3.4', '3.5']) == [ + '3.4', '3.5', + ] diff --git a/tox.ini b/tox.ini index d651019..a7b2b38 100644 --- a/tox.ini +++ b/tox.ini @@ -4,18 +4,19 @@ envlist = flake8,py36,py37 [testenv] deps = pytest commands = - pytest tests.py {posargs} + pytest {posargs:tests} [testenv:pypy36] basepython = pypy3.6 [testenv:coverage] +usedevelop = true basepython = python3.6 deps = {[testenv]deps} coverage commands = - coverage run -m pytest tests.py {posargs} + coverage run -m pytest tests {posargs} ## coverage report -m --fail-under=100 coverage report -m @@ -24,4 +25,4 @@ basepython = python3.6 deps = flake8 commands = - flake8 *.py + flake8 src *.py From 2912c945516da3c343ec3a0e39d655540e88cd54 Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Sat, 13 Apr 2019 23:38:10 +0300 Subject: [PATCH 21/76] Back to 100% test coverage (tox.ini still has the --fail-under=100 commented out because I intend to do more rapid prototyping on this branch.) --- src/check_python_versions/cli.py | 3 - tests/parsers/test_python.py | 58 ++++++++++++- tests/test_cli.py | 143 +++++++++++++++++++++++++++++++ tests/test_utils.py | 30 ++++++- 4 files changed, 229 insertions(+), 5 deletions(-) diff --git a/src/check_python_versions/cli.py b/src/check_python_versions/cli.py index 4941406..3295732 100644 --- a/src/check_python_versions/cli.py +++ b/src/check_python_versions/cli.py @@ -143,9 +143,6 @@ def check_versions(where='.', *, print=print, expect=None): def update_versions(where='.', *, add=None, drop=None, update=None): versions = get_supported_python_versions(where) - if versions is None: - return - versions = sorted(important(versions)) new_versions = update_version_list( versions, add=add, drop=drop, update=update) diff --git a/tests/parsers/test_python.py b/tests/parsers/test_python.py index 0421342..debddf1 100644 --- a/tests/parsers/test_python.py +++ b/tests/parsers/test_python.py @@ -14,6 +14,7 @@ to_literal, update_call_arg_in_source, update_classifiers, + update_supported_python_versions, ) @@ -188,6 +189,24 @@ def test_update_classifiers_none_were_present(): ] +def test_update_supported_python_versions(tmp_path, capsys): + (tmp_path / "setup.py").write_text(textwrap.dedent("""\ + from setuptools import setup + setup( + name='foo', + classifiers=[ + 'Programming Language :: Python :: %s' % v + for v in ['2.7', '3.7'] + ], + ) + """)) + update_supported_python_versions(tmp_path, ['3.7', '3.8']) + assert ( + 'Non-literal classifiers= passed to setup()' + in capsys.readouterr().err + ) + + def test_get_python_requires(tmp_path, fix_max_python_3_version): setup_py = tmp_path / "setup.py" setup_py.write_text(textwrap.dedent("""\ @@ -292,7 +311,8 @@ def test_update_call_arg_in_source(): bar=[ "a", "b", - "c", + + r"c", ], baz=2, ) @@ -378,6 +398,42 @@ def test_update_call_arg_in_source_fixes_opening_bracket(): """) +def test_update_call_arg_in_source_no_function_call(capsys): + source_lines = textwrap.dedent("""\ + """).splitlines(True) + result = update_call_arg_in_source(source_lines, "setup", "bar", + ["x", "y"]) + assert result == source_lines + assert "Did not find setup() call" in capsys.readouterr().err + + +def test_update_call_arg_in_source_no_keyword(capsys): + source_lines = textwrap.dedent("""\ + setup() + """).splitlines(True) + result = update_call_arg_in_source(source_lines, "setup", "bar", + ["x", "y"]) + assert result == source_lines + assert ( + "Did not find bar= argument in setup() call" + in capsys.readouterr().err + ) + + +def test_update_call_arg_in_source_too_complicated(capsys): + source_lines = textwrap.dedent("""\ + setup( + bar=bar) + """).splitlines(True) + result = update_call_arg_in_source(source_lines, "setup", "bar", + ["x", "y"]) + assert result == source_lines + assert ( + "Did not understand bar= formatting in setup() call" + in capsys.readouterr().err + ) + + @pytest.mark.parametrize('code', [ '[2 * 2]', '"".join([2 * 2])', diff --git a/tests/test_cli.py b/tests/test_cli.py index 7a84c8e..b8179ed 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -3,6 +3,7 @@ import re import sys import textwrap +from io import StringIO import pytest @@ -156,6 +157,20 @@ def test_check_expectation(tmp_path, capsys): """) +def test_update_versions_no_change(tmp_path): + (tmp_path / "setup.py").write_text(textwrap.dedent("""\ + from setuptools import setup + setup( + name='foo', + classifiers=[ + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.6', + ], + ) + """)) + cpv.update_versions(tmp_path, add=['3.6']) + + def test_main_help(monkeypatch): monkeypatch.setattr(sys, 'argv', ['check-python-versions', '--help']) with pytest.raises(SystemExit): @@ -179,6 +194,22 @@ def test_main_expect_error_handling(monkeypatch, arg, capsys): assert f'--expect: bad' in capsys.readouterr().err +@pytest.mark.parametrize('arg', ['--add', '--drop']) +def test_main_conflicting_args(monkeypatch, tmp_path, capsys, arg): + monkeypatch.setattr(sys, 'argv', [ + 'check-python-versions', + str(tmp_path), + arg, '3.8', + '--update', '3.6-3.7', + ]) + with pytest.raises(SystemExit): + cpv.main() + assert ( + f'argument {arg}: not allowed with argument --update' + in capsys.readouterr().err + ) + + def test_main_here(monkeypatch, capsys): monkeypatch.setattr(sys, 'argv', [ 'check-python-versions', @@ -257,3 +288,115 @@ def test_main_multiple_ok(monkeypatch, capsys): assert ( capsys.readouterr().out.endswith('\n\nall ok!\n') ) + + +def test_main_update(monkeypatch, capsys, tmp_path): + monkeypatch.setattr(sys, 'stdin', StringIO('y\n')) + monkeypatch.setattr(sys, 'argv', [ + 'check-python-versions', + str(tmp_path), + '--add', '3.7,3.8', + ]) + (tmp_path / "setup.py").write_text(textwrap.dedent("""\ + from setuptools import setup + setup( + name='foo', + classifiers=[ + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.6', + ], + ) + """)) + cpv.main() + assert ( + capsys.readouterr().out + .replace(str(tmp_path) + os.path.sep, 'tmp/') + .expandtabs() + .replace(' \n', '\n\n') + ) == textwrap.dedent("""\ + --- tmp/setup.py (original) + +++ tmp/setup.py (updated) + @@ -4,5 +4,7 @@ + classifiers=[ + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.6', + + 'Programming Language :: Python :: 3.7', + + 'Programming Language :: Python :: 3.8', + ], + ) + + Write changes to tmp/setup.py? [y/N] + + setup.py says: 2.7, 3.6, 3.7, 3.8 + """) + assert (tmp_path / "setup.py").read_text() == textwrap.dedent("""\ + from setuptools import setup + setup( + name='foo', + classifiers=[ + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + ], + ) + """) + + +def test_main_update_rejected(monkeypatch, capsys, tmp_path): + monkeypatch.setattr(sys, 'stdin', StringIO('n\n')) + monkeypatch.setattr(sys, 'argv', [ + 'check-python-versions', + str(tmp_path), + '--add', '3.7,3.8', + ]) + (tmp_path / "setup.py").write_text(textwrap.dedent("""\ + from setuptools import setup + setup( + name='foo', + classifiers=[ + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.6', + ], + ) + """)) + cpv.main() + assert ( + capsys.readouterr().out + .replace(str(tmp_path) + os.path.sep, 'tmp/') + .expandtabs() + .replace(' \n', '\n\n') + ) == textwrap.dedent("""\ + --- tmp/setup.py (original) + +++ tmp/setup.py (updated) + @@ -4,5 +4,7 @@ + classifiers=[ + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.6', + + 'Programming Language :: Python :: 3.7', + + 'Programming Language :: Python :: 3.8', + ], + ) + + Write changes to tmp/setup.py? [y/N] + + setup.py says: 2.7, 3.6 + """) + assert (tmp_path / "setup.py").read_text() == textwrap.dedent("""\ + from setuptools import setup + setup( + name='foo', + classifiers=[ + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.6', + ], + ) + """) + + +def test_main_handles_ctrl_c(monkeypatch): + def raise_keyboard_interrupt(): + raise KeyboardInterrupt() + monkeypatch.setattr(cpv, '_main', raise_keyboard_interrupt) + with pytest.raises(SystemExit): + cpv.main() diff --git a/tests/test_utils.py b/tests/test_utils.py index 779b40a..b7f73ab 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,5 +1,33 @@ -from check_python_versions.utils import pipe +import sys +from io import StringIO + +from check_python_versions.utils import pipe, confirm def test_pipe(): assert pipe('echo', 'hi') == 'hi\n' + + +def test_confirm_eof(monkeypatch): + monkeypatch.setattr(sys, 'stdin', StringIO()) + assert not confirm("Hello how are you?") + + +def test_confirm_default(monkeypatch): + monkeypatch.setattr(sys, 'stdin', StringIO("\n")) + assert not confirm("Hello how are you?") + + +def test_confirm_no(monkeypatch): + monkeypatch.setattr(sys, 'stdin', StringIO("n\n")) + assert not confirm("Hello how are you?") + + +def test_confirm_yes(monkeypatch): + monkeypatch.setattr(sys, 'stdin', StringIO("y\n")) + assert confirm("Hello how are you?") + + +def test_confirm_neither(monkeypatch): + monkeypatch.setattr(sys, 'stdin', StringIO("t\ny\n")) + assert confirm("Hello how are you?") From 8e775dadaa7bdd5ece048adc91913884aabfd53d Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Sat, 13 Apr 2019 23:46:48 +0300 Subject: [PATCH 22/76] Fix wrapper script --- check-python-versions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/check-python-versions b/check-python-versions index ad70c47..17c3b2f 100755 --- a/check-python-versions +++ b/check-python-versions @@ -10,5 +10,5 @@ import os here = os.path.dirname(__file__) sys.path.insert(0, os.path.join(here, 'src')) -from check_python_versions import main # noqa: E402 +from check_python_versions.cli import main # noqa: E402 main() From aa332b86e2b03c31f317f4abbbdb446f0f697ffa Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Sun, 14 Apr 2019 01:21:06 +0300 Subject: [PATCH 23/76] Add rudimentary support for updating .travis.yml --- src/check_python_versions/cli.py | 27 +++++++++++--- src/check_python_versions/parsers/travis.py | 41 +++++++++++++++++++++ src/check_python_versions/versions.py | 12 ++++-- 3 files changed, 71 insertions(+), 9 deletions(-) diff --git a/src/check_python_versions/cli.py b/src/check_python_versions/cli.py index 3295732..6e3a273 100644 --- a/src/check_python_versions/cli.py +++ b/src/check_python_versions/cli.py @@ -20,6 +20,7 @@ from .parsers.travis import ( TRAVIS_YML, get_travis_yml_python_versions, + update_travis_yml_python_versions, ) from .parsers.appveyor import ( APPVEYOR_YML, @@ -142,12 +143,26 @@ def check_versions(where='.', *, print=print, expect=None): def update_versions(where='.', *, add=None, drop=None, update=None): - versions = get_supported_python_versions(where) - versions = sorted(important(versions)) - new_versions = update_version_list( - versions, add=add, drop=drop, update=update) - if versions != new_versions: - update_supported_python_versions(where, new_versions) + sources = [ + (None, get_supported_python_versions, + update_supported_python_versions), + (TRAVIS_YML, get_travis_yml_python_versions, + update_travis_yml_python_versions), + ] + + for (filename, extractor, updater) in sources: + arg = os.path.join(where, filename) if filename else where + if not os.path.exists(arg): + continue + versions = extractor(arg) + if versions is None: + continue + + versions = sorted(important(versions)) + new_versions = update_version_list( + versions, add=add, drop=drop, update=update) + if versions != new_versions: + updater(arg, new_versions) def _main(): diff --git a/src/check_python_versions/parsers/travis.py b/src/check_python_versions/parsers/travis.py index 8fb0eab..4b49132 100644 --- a/src/check_python_versions/parsers/travis.py +++ b/src/check_python_versions/parsers/travis.py @@ -4,6 +4,8 @@ yaml = None from .tox import parse_envlist, tox_env_to_py_version +from ..utils import warn, confirm_and_update_file +from ..versions import is_important TRAVIS_YML = '.travis.yml' @@ -43,3 +45,42 @@ def travis_normalize_py_version(v): return 'PyPy' else: return v + + +def update_travis_yml_python_versions(filename, new_versions): + with open(filename) as fp: + orig_lines = fp.readlines() + + lines = iter(enumerate(orig_lines)) + for n, line in lines: + if line == 'python:\n': + break + else: + warn(f'Did not find python setting in {filename}') + return + + start = end = n + 1 + indent = 2 + keep = [] + for n, line in lines: + stripped = line.lstrip() + if stripped.startswith('- '): + indent = len(line) - len(stripped) + end = n + 1 + ver = stripped[2:].strip() + if not is_important(travis_normalize_py_version(ver)): + keep.append(line) + elif stripped.startswith('#'): + keep.append(line) + end = n + 1 + if line and line[0] != ' ': + break + + # XXX: if python 3.7 was enabled via matrix.include, we'll add a + # second 3.7 entry directly to top-level python, without even + # checking for dist: xenial. + new_lines = orig_lines[:start] + [ + f"{' ' * indent}- {ver}\n" + for ver in new_versions + ] + keep + orig_lines[end:] + confirm_and_update_file(filename, orig_lines, new_lines) diff --git a/src/check_python_versions/versions.py b/src/check_python_versions/versions.py index 2e7743b..9c100cb 100644 --- a/src/check_python_versions/versions.py +++ b/src/check_python_versions/versions.py @@ -9,12 +9,18 @@ } -def important(versions): +def is_important(v): upcoming_release = f'3.{CURRENT_PYTHON_3_VERSION + 1}' + return ( + not v.startswith(('PyPy', 'Jython')) and v != 'nightly' + and not v.endswith('-dev') and v != upcoming_release + ) + + +def important(versions): return { v for v in versions - if not v.startswith(('PyPy', 'Jython')) and v != 'nightly' - and not v.endswith('-dev') and v != upcoming_release + if is_important(v) } From 32fd42b152145bfcf715cfcb383034fc0efaaff3 Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Sun, 14 Apr 2019 15:55:44 +0300 Subject: [PATCH 24/76] distutils does NOT like trailing slashes --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 619aa3d..ebe2f95 100755 --- a/setup.py +++ b/setup.py @@ -48,7 +48,7 @@ license='GPL', python_requires=">=3.6", packages=find_packages('src'), - package_dir={'': 'src/'}, + package_dir={'': 'src'}, entry_points={ 'console_scripts': [ 'check-python-versions = check_python_versions.cli:main', From 12824605623042247bb8979363f56d1396a3279a Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Sun, 14 Apr 2019 15:55:58 +0300 Subject: [PATCH 25/76] Update MANIFEST.in --- MANIFEST.in | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/MANIFEST.in b/MANIFEST.in index 655dd9d..29d311f 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -14,3 +14,7 @@ include *.yml # added by check_manifest.py include CLASSIFIERS include check-python-versions + +# added by check_manifest.py +include pytest.ini +recursive-include tests *.py From 098f610887e8b01931d7eba359d364fc94838f4f Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Tue, 16 Apr 2019 18:06:19 +0300 Subject: [PATCH 26/76] Add some tests for the YAML update logic --- src/check_python_versions/parsers/travis.py | 40 +++++++++++------- tests/parsers/test_travis.py | 45 +++++++++++++++++++++ 2 files changed, 71 insertions(+), 14 deletions(-) diff --git a/src/check_python_versions/parsers/travis.py b/src/check_python_versions/parsers/travis.py index 4b49132..6593722 100644 --- a/src/check_python_versions/parsers/travis.py +++ b/src/check_python_versions/parsers/travis.py @@ -51,36 +51,48 @@ def update_travis_yml_python_versions(filename, new_versions): with open(filename) as fp: orig_lines = fp.readlines() + # XXX: if python 3.7 was enabled via matrix.include, we'll add a + # second 3.7 entry directly to top-level python, without even + # checking for dist: xenial. + + def keep_old(ver): + return not is_important(travis_normalize_py_version(ver)) + + new_lines = update_yaml_list( + orig_lines, "python", new_versions, filename=filename, keep=keep_old, + ) + confirm_and_update_file(filename, orig_lines, new_lines) + + +def update_yaml_list( + orig_lines, key, new_value, filename=TRAVIS_YML, keep=None, +): lines = iter(enumerate(orig_lines)) for n, line in lines: - if line == 'python:\n': + if line == f'{key}:\n': break else: - warn(f'Did not find python setting in {filename}') + warn(f'Did not find {key}: setting in {filename}') return start = end = n + 1 indent = 2 - keep = [] + lines_to_keep = [] for n, line in lines: stripped = line.lstrip() if stripped.startswith('- '): indent = len(line) - len(stripped) end = n + 1 - ver = stripped[2:].strip() - if not is_important(travis_normalize_py_version(ver)): - keep.append(line) + if keep and keep(stripped[2:].strip()): + lines_to_keep.append(line) elif stripped.startswith('#'): - keep.append(line) + lines_to_keep.append(line) end = n + 1 if line and line[0] != ' ': break - # XXX: if python 3.7 was enabled via matrix.include, we'll add a - # second 3.7 entry directly to top-level python, without even - # checking for dist: xenial. new_lines = orig_lines[:start] + [ - f"{' ' * indent}- {ver}\n" - for ver in new_versions - ] + keep + orig_lines[end:] - confirm_and_update_file(filename, orig_lines, new_lines) + f"{' ' * indent}- {value}\n" + for value in new_value + ] + lines_to_keep + orig_lines[end:] + return new_lines diff --git a/tests/parsers/test_travis.py b/tests/parsers/test_travis.py index a9311a1..f22d991 100644 --- a/tests/parsers/test_travis.py +++ b/tests/parsers/test_travis.py @@ -10,6 +10,7 @@ from check_python_versions.parsers.travis import ( get_travis_yml_python_versions, travis_normalize_py_version, + update_yaml_list, ) @@ -68,3 +69,47 @@ def test_get_travis_yml_python_versions_no_python_only_matrix(tmp_path): ]) def test_travis_normalize_py_version(s, expected): assert travis_normalize_py_version(s) == expected + + +def test_update_yaml_list(): + source_lines = textwrap.dedent("""\ + language: python + python: + - 2.6 + - 2.7 + install: pip install -e . + script: pytest tests + """).splitlines(True) + result = update_yaml_list(source_lines, "python", ["2.7", "3.7"]) + assert "".join(result) == textwrap.dedent("""\ + language: python + python: + - 2.7 + - 3.7 + install: pip install -e . + script: pytest tests + """) + + +def test_update_yaml_list_keep_indent_comments_and_pypy(): + source_lines = textwrap.dedent("""\ + language: python + python: + - 2.6 + # XXX: should probably remove 2.6 + - 2.7 + - pypy + - 3.3 + script: pytest tests + """).splitlines(True) + result = update_yaml_list(source_lines, "python", ["2.7", "3.7"], + keep=lambda line: line == 'pypy') + assert "".join(result) == textwrap.dedent("""\ + language: python + python: + - 2.7 + - 3.7 + # XXX: should probably remove 2.6 + - pypy + script: pytest tests + """) From 5998e5dde10d3e15977edfae5df8b68b949c0690 Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Tue, 16 Apr 2019 18:28:56 +0300 Subject: [PATCH 27/76] Deal with old matrix.include way of enabling 3.7 --- src/check_python_versions/parsers/travis.py | 70 ++++++++++++++-- tests/parsers/test_travis.py | 93 +++++++++++++++++++++ 2 files changed, 158 insertions(+), 5 deletions(-) diff --git a/src/check_python_versions/parsers/travis.py b/src/check_python_versions/parsers/travis.py index 6593722..41e2b77 100644 --- a/src/check_python_versions/parsers/travis.py +++ b/src/check_python_versions/parsers/travis.py @@ -47,13 +47,16 @@ def travis_normalize_py_version(v): return v +def needs_xenial(v): + major, minor = map(int, v.split('.')) + return major == 3 and minor >= 7 + + def update_travis_yml_python_versions(filename, new_versions): with open(filename) as fp: orig_lines = fp.readlines() - - # XXX: if python 3.7 was enabled via matrix.include, we'll add a - # second 3.7 entry directly to top-level python, without even - # checking for dist: xenial. + fp.seek(0) + conf = yaml.safe_load(fp) def keep_old(ver): return not is_important(travis_normalize_py_version(ver)) @@ -61,6 +64,27 @@ def keep_old(ver): new_lines = update_yaml_list( orig_lines, "python", new_versions, filename=filename, keep=keep_old, ) + + # If python 3.7 was enabled via matrix.include, we've just added a + # second 3.7 entry directly to top-level python by the above code. + # So let's drop the matrix. + + if ( + 'matrix' in conf + and 'include' in conf['matrix'] + and all( + job.get('dist') == 'xenial' + for job in conf['matrix']['include'] + ) + ): + # XXX: this may drop too much or too little! + new_lines = drop_yaml_node(new_lines, "matrix") + + # Make sure we're using dist: xenial if we want to use Python 3.7 or newer. + if any(map(needs_xenial, new_versions)) and conf.get('dist') != 'xenial': + new_lines = drop_yaml_node(new_lines, 'dist') + new_lines = add_yaml_node(new_lines, 'dist', 'xenial', before='python') + confirm_and_update_file(filename, orig_lines, new_lines) @@ -73,7 +97,7 @@ def update_yaml_list( break else: warn(f'Did not find {key}: setting in {filename}') - return + return orig_lines start = end = n + 1 indent = 2 @@ -90,9 +114,45 @@ def update_yaml_list( end = n + 1 if line and line[0] != ' ': break + # TODO: else? new_lines = orig_lines[:start] + [ f"{' ' * indent}- {value}\n" for value in new_value ] + lines_to_keep + orig_lines[end:] return new_lines + + +def drop_yaml_node(orig_lines, key): + lines = iter(enumerate(orig_lines)) + for n, line in lines: + if line == f'{key}:\n': + break + else: + return orig_lines + + start = n + end = n + 1 + for n, line in lines: + if line and line[0] != ' ': + break + else: + end = n + 1 + new_lines = orig_lines[:start] + orig_lines[end:] + return new_lines + + +def add_yaml_node(orig_lines, key, value, before=None): + lines = iter(enumerate(orig_lines)) + where = len(orig_lines) + if before: + lines = iter(enumerate(orig_lines)) + for n, line in lines: + if line == f'{before}:\n': + where = n + break + + new_lines = orig_lines[:where] + [ + f'{key}: {value}\n' + ] + orig_lines[where:] + return new_lines diff --git a/tests/parsers/test_travis.py b/tests/parsers/test_travis.py index f22d991..8e6e12e 100644 --- a/tests/parsers/test_travis.py +++ b/tests/parsers/test_travis.py @@ -8,6 +8,8 @@ yaml = None from check_python_versions.parsers.travis import ( + add_yaml_node, + drop_yaml_node, get_travis_yml_python_versions, travis_normalize_py_version, update_yaml_list, @@ -113,3 +115,94 @@ def test_update_yaml_list_keep_indent_comments_and_pypy(): - pypy script: pytest tests """) + + +def test_drop_yaml_node(): + source_lines = textwrap.dedent("""\ + language: python + python: + - 3.6 + matrix: + include: + - python: 3.7 + dist: xenial + sudo: required + script: pytest tests + """).splitlines(True) + result = drop_yaml_node(source_lines, 'matrix') + assert "".join(result) == textwrap.dedent("""\ + language: python + python: + - 3.6 + script: pytest tests + """) + + +def test_drop_yaml_node_when_empty(): + source_lines = textwrap.dedent("""\ + language: python + python: + - 3.6 + matrix: + script: pytest tests + """).splitlines(True) + result = drop_yaml_node(source_lines, 'matrix') + assert "".join(result) == textwrap.dedent("""\ + language: python + python: + - 3.6 + script: pytest tests + """) + + +def test_drop_yaml_node_when_last_in_file(): + source_lines = textwrap.dedent("""\ + language: python + python: + - 3.6 + matrix: + include: + - python: 3.7 + dist: xenial + sudo: required + """).splitlines(True) + result = drop_yaml_node(source_lines, 'matrix') + assert "".join(result) == textwrap.dedent("""\ + language: python + python: + - 3.6 + """) + + +def test_add_yaml_node(): + source_lines = textwrap.dedent("""\ + language: python + python: + - 3.6 + script: pytest tests + """).splitlines(True) + result = add_yaml_node(source_lines, 'dist', 'xenial', before='python') + assert "".join(result) == textwrap.dedent("""\ + language: python + dist: xenial + python: + - 3.6 + script: pytest tests + """) + + +def test_add_yaml_node_at_end(): + source_lines = textwrap.dedent("""\ + language: python + python: + - 3.6 + script: pytest tests + """).splitlines(True) + result = add_yaml_node(source_lines, 'dist', 'xenial', before='sudo') + assert "".join(result) == textwrap.dedent("""\ + language: python + python: + - 3.6 + script: pytest tests + dist: xenial + """) From 09f0a4b69c867509c861bde1124b1202a66c0bcb Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Tue, 16 Apr 2019 18:40:08 +0300 Subject: [PATCH 28/76] Refactoring: pass filename to all extractors & updaters --- src/check_python_versions/cli.py | 18 +++++++++--------- src/check_python_versions/parsers/python.py | 17 ++++++++--------- tests/parsers/test_python.py | 15 +++++++++------ 3 files changed, 26 insertions(+), 24 deletions(-) diff --git a/src/check_python_versions/cli.py b/src/check_python_versions/cli.py index 6e3a273..4a7f96a 100644 --- a/src/check_python_versions/cli.py +++ b/src/check_python_versions/cli.py @@ -107,7 +107,7 @@ def check_package(where='.', *, print=print): def check_versions(where='.', *, print=print, expect=None): sources = [ - ('setup.py', get_supported_python_versions, None), + ('setup.py', get_supported_python_versions, 'setup.py'), ('- python_requires', get_python_requires, 'setup.py'), (TOX_INI, get_tox_ini_python_versions, TOX_INI), (TRAVIS_YML, get_travis_yml_python_versions, TRAVIS_YML), @@ -121,10 +121,10 @@ def check_versions(where='.', *, print=print, expect=None): version_sets = [] for (title, extractor, filename) in sources: - arg = os.path.join(where, filename) if filename else where - if not os.path.exists(arg): + pathname = os.path.join(where, filename) + if not os.path.exists(pathname): continue - versions = extractor(arg) + versions = extractor(pathname) if versions is None: continue print(f"{title} says:".ljust(width), ", ".join(versions) or "(empty)") @@ -144,17 +144,17 @@ def check_versions(where='.', *, print=print, expect=None): def update_versions(where='.', *, add=None, drop=None, update=None): sources = [ - (None, get_supported_python_versions, + ('setup.py', get_supported_python_versions, update_supported_python_versions), (TRAVIS_YML, get_travis_yml_python_versions, update_travis_yml_python_versions), ] for (filename, extractor, updater) in sources: - arg = os.path.join(where, filename) if filename else where - if not os.path.exists(arg): + pathname = os.path.join(where, filename) + if not os.path.exists(pathname): continue - versions = extractor(arg) + versions = extractor(pathname) if versions is None: continue @@ -162,7 +162,7 @@ def update_versions(where='.', *, add=None, drop=None, update=None): new_versions = update_version_list( versions, add=add, drop=drop, update=update) if versions != new_versions: - updater(arg, new_versions) + updater(pathname, new_versions) def _main(): diff --git a/src/check_python_versions/parsers/python.py b/src/check_python_versions/parsers/python.py index aa7d0d3..83c1317 100644 --- a/src/check_python_versions/parsers/python.py +++ b/src/check_python_versions/parsers/python.py @@ -8,13 +8,13 @@ from ..versions import MAX_MINOR_FOR_MAJOR -def get_supported_python_versions(repo_path='.'): - setup_py = os.path.join(repo_path, 'setup.py') - classifiers = get_setup_py_keyword(setup_py, 'classifiers') +def get_supported_python_versions(filename='setup.py'): + classifiers = get_setup_py_keyword(filename, 'classifiers') if classifiers is None: # AST parsing is complicated - classifiers = pipe("python", "setup.py", "-q", "--classifiers", - cwd=repo_path).splitlines() + setup_py = os.path.basename(filename) + classifiers = pipe("python", setup_py, "-q", "--classifiers", + cwd=os.path.dirname(filename)).splitlines() return get_versions_from_classifiers(classifiers) @@ -87,13 +87,12 @@ def update_classifiers(classifiers, new_versions): return classifiers -def update_supported_python_versions(repo_path, new_versions): - setup_py = os.path.join(repo_path, 'setup.py') - classifiers = get_setup_py_keyword(setup_py, 'classifiers') +def update_supported_python_versions(filename, new_versions): + classifiers = get_setup_py_keyword(filename, 'classifiers') if classifiers is None: return new_classifiers = update_classifiers(classifiers, new_versions) - update_setup_py_keyword(setup_py, 'classifiers', new_classifiers) + update_setup_py_keyword(filename, 'classifiers', new_classifiers) def get_setup_py_keyword(setup_py, keyword): diff --git a/tests/parsers/test_python.py b/tests/parsers/test_python.py index debddf1..831bd83 100644 --- a/tests/parsers/test_python.py +++ b/tests/parsers/test_python.py @@ -19,7 +19,8 @@ def test_get_supported_python_versions(tmp_path): - (tmp_path / "setup.py").write_text(textwrap.dedent("""\ + filename = tmp_path / "setup.py" + filename.write_text(textwrap.dedent("""\ from setuptools import setup setup( name='foo', @@ -29,11 +30,12 @@ def test_get_supported_python_versions(tmp_path): ], ) """)) - assert get_supported_python_versions(tmp_path) == ['2.7', '3.6'] + assert get_supported_python_versions(filename) == ['2.7', '3.6'] def test_get_supported_python_versions_computed(tmp_path): - (tmp_path / "setup.py").write_text(textwrap.dedent("""\ + filename = tmp_path / "setup.py" + filename.write_text(textwrap.dedent("""\ from setuptools import setup setup( name='foo', @@ -43,7 +45,7 @@ def test_get_supported_python_versions_computed(tmp_path): ], ) """)) - assert get_supported_python_versions(tmp_path) == ['2.7', '3.7'] + assert get_supported_python_versions(filename) == ['2.7', '3.7'] def test_get_versions_from_classifiers(): @@ -190,7 +192,8 @@ def test_update_classifiers_none_were_present(): def test_update_supported_python_versions(tmp_path, capsys): - (tmp_path / "setup.py").write_text(textwrap.dedent("""\ + filename = tmp_path / "setup.py" + filename.write_text(textwrap.dedent("""\ from setuptools import setup setup( name='foo', @@ -200,7 +203,7 @@ def test_update_supported_python_versions(tmp_path, capsys): ], ) """)) - update_supported_python_versions(tmp_path, ['3.7', '3.8']) + update_supported_python_versions(filename, ['3.7', '3.8']) assert ( 'Non-literal classifiers= passed to setup()' in capsys.readouterr().err From f1cd2428d48a80158fe39cf89d2c7d010394e67f Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Tue, 16 Apr 2019 18:40:24 +0300 Subject: [PATCH 29/76] Refactoring: call confirm_and_update_file() externally Let the updaters return new file contents so we can do what we want with them. --- src/check_python_versions/cli.py | 5 ++++- src/check_python_versions/parsers/python.py | 8 ++++---- src/check_python_versions/parsers/travis.py | 4 ++-- src/check_python_versions/utils.py | 4 +++- 4 files changed, 13 insertions(+), 8 deletions(-) diff --git a/src/check_python_versions/cli.py b/src/check_python_versions/cli.py index 4a7f96a..9ed4f53 100644 --- a/src/check_python_versions/cli.py +++ b/src/check_python_versions/cli.py @@ -3,6 +3,7 @@ import sys from . import __version__ +from .utils import confirm_and_update_file from .versions import ( MAX_MINOR_FOR_MAJOR, important, @@ -162,7 +163,9 @@ def update_versions(where='.', *, add=None, drop=None, update=None): new_versions = update_version_list( versions, add=add, drop=drop, update=update) if versions != new_versions: - updater(pathname, new_versions) + new_lines = updater(pathname, new_versions) + if new_lines is not None: + confirm_and_update_file(pathname, new_lines) def _main(): diff --git a/src/check_python_versions/parsers/python.py b/src/check_python_versions/parsers/python.py index 83c1317..d7f3f5e 100644 --- a/src/check_python_versions/parsers/python.py +++ b/src/check_python_versions/parsers/python.py @@ -4,7 +4,7 @@ import string from functools import partial -from ..utils import warn, pipe, confirm_and_update_file +from ..utils import warn, pipe from ..versions import MAX_MINOR_FOR_MAJOR @@ -90,9 +90,9 @@ def update_classifiers(classifiers, new_versions): def update_supported_python_versions(filename, new_versions): classifiers = get_setup_py_keyword(filename, 'classifiers') if classifiers is None: - return + return None new_classifiers = update_classifiers(classifiers, new_versions) - update_setup_py_keyword(filename, 'classifiers', new_classifiers) + return update_setup_py_keyword(filename, 'classifiers', new_classifiers) def get_setup_py_keyword(setup_py, keyword): @@ -110,7 +110,7 @@ def update_setup_py_keyword(setup_py, keyword, new_value): with open(setup_py) as f: lines = f.readlines() new_lines = update_call_arg_in_source(lines, 'setup', keyword, new_value) - confirm_and_update_file(setup_py, lines, new_lines) + return new_lines def to_literal(value, quote_style='"'): diff --git a/src/check_python_versions/parsers/travis.py b/src/check_python_versions/parsers/travis.py index 41e2b77..87c9af4 100644 --- a/src/check_python_versions/parsers/travis.py +++ b/src/check_python_versions/parsers/travis.py @@ -4,7 +4,7 @@ yaml = None from .tox import parse_envlist, tox_env_to_py_version -from ..utils import warn, confirm_and_update_file +from ..utils import warn from ..versions import is_important @@ -85,7 +85,7 @@ def keep_old(ver): new_lines = drop_yaml_node(new_lines, 'dist') new_lines = add_yaml_node(new_lines, 'dist', 'xenial', before='python') - confirm_and_update_file(filename, orig_lines, new_lines) + return new_lines def update_yaml_list( diff --git a/src/check_python_versions/utils.py b/src/check_python_versions/utils.py index aa7e3ad..7d08876 100644 --- a/src/check_python_versions/utils.py +++ b/src/check_python_versions/utils.py @@ -22,7 +22,9 @@ def pipe(*cmd, **kwargs): return p.communicate()[0].decode('UTF-8', 'replace') -def confirm_and_update_file(filename, old_lines, new_lines): +def confirm_and_update_file(filename, new_lines): + with open(filename, 'r') as f: + old_lines = f.readlines() print_diff(old_lines, new_lines, filename) if new_lines != old_lines and confirm(f"Write changes to {filename}?"): mode = stat.S_IMODE(os.stat(filename).st_mode) From 9baab7cbb084671e2fa7568a40aad2718bbe3889 Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Tue, 16 Apr 2019 18:44:52 +0300 Subject: [PATCH 30/76] Implement --diff --- src/check_python_versions/cli.py | 42 ++++++++++++++++++------------ src/check_python_versions/utils.py | 13 ++++++--- 2 files changed, 34 insertions(+), 21 deletions(-) diff --git a/src/check_python_versions/cli.py b/src/check_python_versions/cli.py index 9ed4f53..d7ff00a 100644 --- a/src/check_python_versions/cli.py +++ b/src/check_python_versions/cli.py @@ -3,7 +3,7 @@ import sys from . import __version__ -from .utils import confirm_and_update_file +from .utils import confirm_and_update_file, show_diff from .versions import ( MAX_MINOR_FOR_MAJOR, important, @@ -142,7 +142,8 @@ def check_versions(where='.', *, print=print, expect=None): ) -def update_versions(where='.', *, add=None, drop=None, update=None): +def update_versions(where='.', *, add=None, drop=None, update=None, + diff=False): sources = [ ('setup.py', get_supported_python_versions, @@ -165,7 +166,10 @@ def update_versions(where='.', *, add=None, drop=None, update=None): if versions != new_versions: new_lines = updater(pathname, new_versions) if new_lines is not None: - confirm_and_update_file(pathname, new_lines) + if diff: + show_diff(pathname, new_lines) + else: + confirm_and_update_file(pathname, new_lines) def _main(): @@ -186,6 +190,8 @@ def _main(): ' and other files is located') group = parser.add_argument_group( "updating supported version lists (EXPERIMENTAL)") + group.add_argument('--diff', action='store_true', + help='show a diff of proposed changes') group.add_argument('--add', metavar='VERSIONS', type=parse_version_list, help='add these versions to supported ones, e.g' ' --add 3.8') @@ -209,27 +215,29 @@ def _main(): multiple = len(where) > 1 mismatches = [] for n, path in enumerate(where): - if multiple: + if multiple and not args.diff: if n: print("\n") print(f"{path}:\n") if not check_package(path): mismatches.append(path) continue - if args.add or args.drop or args.update: + if args.add or args.drop or args.update or args.diff: update_versions(path, add=args.add, drop=args.drop, - update=args.update) - if not check_versions(path, expect=args.expect): - mismatches.append(path) - continue - - if mismatches: - if multiple: - sys.exit(f"\n\nmismatch in {' '.join(mismatches)}!") - else: - sys.exit("\nmismatch!") - elif multiple: - print("\n\nall ok!") + update=args.update, diff=args.diff) + if not args.diff: + if not check_versions(path, expect=args.expect): + mismatches.append(path) + continue + + if not args.diff: + if mismatches: + if multiple: + sys.exit(f"\n\nmismatch in {' '.join(mismatches)}!") + else: + sys.exit("\nmismatch!") + elif multiple: + print("\n\nall ok!") def main(): diff --git a/src/check_python_versions/utils.py b/src/check_python_versions/utils.py index 7d08876..2770f16 100644 --- a/src/check_python_versions/utils.py +++ b/src/check_python_versions/utils.py @@ -23,10 +23,8 @@ def pipe(*cmd, **kwargs): def confirm_and_update_file(filename, new_lines): - with open(filename, 'r') as f: - old_lines = f.readlines() - print_diff(old_lines, new_lines, filename) - if new_lines != old_lines and confirm(f"Write changes to {filename}?"): + if (show_diff(filename, new_lines) + and confirm(f"Write changes to {filename}?")): mode = stat.S_IMODE(os.stat(filename).st_mode) tempfile = filename + '.tmp' with open(tempfile, 'w') as f: @@ -35,6 +33,13 @@ def confirm_and_update_file(filename, new_lines): os.rename(tempfile, filename) +def show_diff(filename, new_lines): + with open(filename, 'r') as f: + old_lines = f.readlines() + print_diff(old_lines, new_lines, filename) + return old_lines != new_lines + + def print_diff(a, b, filename): print(''.join(difflib.unified_diff( a, b, From 4ce7bfb8fed572284f4392c3d450286ddf25729d Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Tue, 16 Apr 2019 18:52:04 +0300 Subject: [PATCH 31/76] Experiment with dynamic classifiers I aim to handle the hack in https://github.com/mgedmin/findimports/blob/7ccfb55c91335de215588002fb8d120ed16ea136/setup.py#L46-L50 --- src/check_python_versions/parsers/python.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/check_python_versions/parsers/python.py b/src/check_python_versions/parsers/python.py index d7f3f5e..fdf9a11 100644 --- a/src/check_python_versions/parsers/python.py +++ b/src/check_python_versions/parsers/python.py @@ -197,7 +197,15 @@ def eval_ast_node(node, keyword): try: return ast.literal_eval(node) except ValueError: - pass + if any(isinstance(element, ast.Str) for element in node.elts): + # Let's try our best!!! + warn(f'Non-literal {keyword}= passed to setup(),' + ' skipping some values') + return [ + eval_ast_node(element, keyword) + for element in node.elts + if isinstance(element, ast.Str) + ] if (isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute) and isinstance(node.func.value, ast.Str) and node.func.attr == 'join'): From 5c0890650f4b02e307747a5bc8390a97e3402d25 Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Tue, 16 Apr 2019 18:52:48 +0300 Subject: [PATCH 32/76] Revert "Experiment with dynamic classifiers" If I do this, those dynamically-computed classifiers get dropped when I do an update. This reverts commit 4ce7bfb8fed572284f4392c3d450286ddf25729d. --- src/check_python_versions/parsers/python.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/src/check_python_versions/parsers/python.py b/src/check_python_versions/parsers/python.py index fdf9a11..d7f3f5e 100644 --- a/src/check_python_versions/parsers/python.py +++ b/src/check_python_versions/parsers/python.py @@ -197,15 +197,7 @@ def eval_ast_node(node, keyword): try: return ast.literal_eval(node) except ValueError: - if any(isinstance(element, ast.Str) for element in node.elts): - # Let's try our best!!! - warn(f'Non-literal {keyword}= passed to setup(),' - ' skipping some values') - return [ - eval_ast_node(element, keyword) - for element in node.elts - if isinstance(element, ast.Str) - ] + pass if (isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute) and isinstance(node.func.value, ast.Str) and node.func.attr == 'join'): From bf0c58cd7f001dc3fcbe3b30f0a786abd5eb8dcb Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Tue, 16 Apr 2019 18:59:29 +0300 Subject: [PATCH 33/76] Hack for fetchimports This allows us to parse and update partially-dynamic classifiers if specified like setup( ... classifiers=[ 'list', 'of', 'constant', 'literals', ] + [ some dynamic computation ], ... ) --- src/check_python_versions/parsers/python.py | 30 +++++++++++++++++---- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/src/check_python_versions/parsers/python.py b/src/check_python_versions/parsers/python.py index d7f3f5e..b9c991e 100644 --- a/src/check_python_versions/parsers/python.py +++ b/src/check_python_versions/parsers/python.py @@ -148,10 +148,11 @@ def update_call_arg_in_source(source_lines, function, keyword, new_value): start = n indent = first_indent + 4 quote_style = '"' + fix_closing_bracket = False for n, line in lines: stripped = line.lstrip() - if stripped.startswith('],'): - end = n + 1 + if stripped.startswith(']'): + end = n break elif stripped: if not must_fix_indents: @@ -160,6 +161,7 @@ def update_call_arg_in_source(source_lines, function, keyword, new_value): quote_style = stripped[0] if line.rstrip().endswith('],'): end = n + 1 + fix_closing_bracket = True break else: warn(f'Did not understand {keyword}= formatting in {function}() call') @@ -170,9 +172,10 @@ def update_call_arg_in_source(source_lines, function, keyword, new_value): ] + [ f"{' ' * indent}{to_literal(value, quote_style)},\n" for value in new_value - ] + [ + ] + ([ f"{' ' * first_indent}],\n" - ] + source_lines[end:] + ] if fix_closing_bracket else [ + ]) + source_lines[end:] def find_call_kwarg_in_ast(tree, funcname, keyword, filename='setup.py'): @@ -197,7 +200,15 @@ def eval_ast_node(node, keyword): try: return ast.literal_eval(node) except ValueError: - pass + if any(isinstance(element, ast.Str) for element in node.elts): + # Let's try our best!!! + warn(f'Non-literal {keyword}= passed to setup(),' + ' skipping some values') + return [ + eval_ast_node(element, keyword) + for element in node.elts + if isinstance(element, ast.Str) + ] if (isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute) and isinstance(node.func.value, ast.Str) and node.func.attr == 'join'): @@ -205,6 +216,15 @@ def eval_ast_node(node, keyword): return node.func.value.s.join(ast.literal_eval(node.args[0])) except ValueError: pass + if isinstance(node, ast.BinOp) and isinstance(node.op, ast.Add): + left = eval_ast_node(node.left, keyword) + right = eval_ast_node(node.right, keyword) + if left is not None and right is not None: + return left + right + if left is None and right is not None: + return right + if left is not None and right is None: + return left warn(f'Non-literal {keyword}= passed to setup()') return None From c4771acce883620fcc83e8cfa3f2e681c5e5125f Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Tue, 16 Apr 2019 19:18:50 +0300 Subject: [PATCH 34/76] Update Python versions in appveyor.yml --- src/check_python_versions/cli.py | 3 ++ src/check_python_versions/parsers/appveyor.py | 46 +++++++++++++++++++ src/check_python_versions/parsers/travis.py | 9 ++-- 3 files changed, 55 insertions(+), 3 deletions(-) diff --git a/src/check_python_versions/cli.py b/src/check_python_versions/cli.py index d7ff00a..948384b 100644 --- a/src/check_python_versions/cli.py +++ b/src/check_python_versions/cli.py @@ -26,6 +26,7 @@ from .parsers.appveyor import ( APPVEYOR_YML, get_appveyor_yml_python_versions, + update_appveyor_yml_python_versions, ) from .parsers.manylinux import ( MANYLINUX_INSTALL_SH, @@ -150,6 +151,8 @@ def update_versions(where='.', *, add=None, drop=None, update=None, update_supported_python_versions), (TRAVIS_YML, get_travis_yml_python_versions, update_travis_yml_python_versions), + (APPVEYOR_YML, get_appveyor_yml_python_versions, + update_appveyor_yml_python_versions), ] for (filename, extractor, updater) in sources: diff --git a/src/check_python_versions/parsers/appveyor.py b/src/check_python_versions/parsers/appveyor.py index 7067428..9842fb4 100644 --- a/src/check_python_versions/parsers/appveyor.py +++ b/src/check_python_versions/parsers/appveyor.py @@ -4,6 +4,7 @@ yaml = None from .tox import parse_envlist, tox_env_to_py_version +from .travis import update_yaml_list APPVEYOR_YML = 'appveyor.yml' @@ -37,3 +38,48 @@ def appveyor_normalize_py_version(ver): ver = ver[:-len('-x64')] assert len(ver) >= 2 and ver[:2].isdigit() return f'{ver[0]}.{ver[1:]}' + + +def appveyor_detect_py_version_pattern(ver): + ver = str(ver) + pattern = '{}' + if ver.lower().startswith('c:\\python'): + pos = len('c:\\python') + prefix, ver = ver[:pos], ver[pos:] + pattern = pattern.format(f'{prefix}{{}}') + if ver.endswith('\\'): + ver = ver[:-1] + pattern = pattern.format(f'{{}}\\') + if ver.lower().endswith('-x64'): + pos = -len('-x64') + ver, suffix = ver[:pos], ver[pos:] + pattern = pattern.format(f'{{}}{suffix}') + assert len(ver) >= 2 and ver[:2].isdigit() + return pattern.format('{}{}') + + +def escape(s): + return s.replace("\\", "\\\\").replace('"', '\\"') + + +def update_appveyor_yml_python_versions(filename, new_versions): + with open(filename) as fp: + orig_lines = fp.readlines() + fp.seek(0) + conf = yaml.safe_load(fp) + + varname = 'PYTHON' + pattern = '{}{}' + for env in conf['environment']['matrix']: + for var, value in env.items(): + if var.lower() == 'python': + varname = var + pattern = appveyor_detect_py_version_pattern(value) + break + + new_environments = [ + f'{varname}: "{escape(pattern.format(*ver.split(".", 1)))}"' + for ver in new_versions + ] + new_lines = update_yaml_list(orig_lines, ' matrix', new_environments) + return new_lines diff --git a/src/check_python_versions/parsers/travis.py b/src/check_python_versions/parsers/travis.py index 87c9af4..afe8ad6 100644 --- a/src/check_python_versions/parsers/travis.py +++ b/src/check_python_versions/parsers/travis.py @@ -101,10 +101,13 @@ def update_yaml_list( start = end = n + 1 indent = 2 - lines_to_keep = [] + keep_before = [] + keep_after = [] + lines_to_keep = keep_before for n, line in lines: stripped = line.lstrip() if stripped.startswith('- '): + lines_to_keep = keep_after indent = len(line) - len(stripped) end = n + 1 if keep and keep(stripped[2:].strip()): @@ -116,10 +119,10 @@ def update_yaml_list( break # TODO: else? - new_lines = orig_lines[:start] + [ + new_lines = orig_lines[:start] + keep_before + [ f"{' ' * indent}- {value}\n" for value in new_value - ] + lines_to_keep + orig_lines[end:] + ] + keep_after + orig_lines[end:] return new_lines From 83d94bd83c8c6ca010ec6dc17781cdfd3e5ed885 Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Tue, 16 Apr 2019 19:24:53 +0300 Subject: [PATCH 35/76] Drop sudo: false in .travis.yml files --- src/check_python_versions/parsers/travis.py | 15 +++++++++++---- tests/parsers/test_travis.py | 17 +++++++++++++++++ 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/src/check_python_versions/parsers/travis.py b/src/check_python_versions/parsers/travis.py index afe8ad6..c298b92 100644 --- a/src/check_python_versions/parsers/travis.py +++ b/src/check_python_versions/parsers/travis.py @@ -81,9 +81,16 @@ def keep_old(ver): new_lines = drop_yaml_node(new_lines, "matrix") # Make sure we're using dist: xenial if we want to use Python 3.7 or newer. - if any(map(needs_xenial, new_versions)) and conf.get('dist') != 'xenial': - new_lines = drop_yaml_node(new_lines, 'dist') - new_lines = add_yaml_node(new_lines, 'dist', 'xenial', before='python') + if any(map(needs_xenial, new_versions)): + if conf.get('dist') != 'xenial': + new_lines = drop_yaml_node(new_lines, 'dist') + new_lines = add_yaml_node(new_lines, 'dist', 'xenial', + before='python') + if conf.get('sudo') is False: + # sudo is ignored nowadays, but in earlier times + # you needed both dist: xenial and sudo: required + # to get Python 3.7 + new_lines = drop_yaml_node(new_lines, "sudo") return new_lines @@ -129,7 +136,7 @@ def update_yaml_list( def drop_yaml_node(orig_lines, key): lines = iter(enumerate(orig_lines)) for n, line in lines: - if line == f'{key}:\n': + if line.startswith(f'{key}:'): break else: return orig_lines diff --git a/tests/parsers/test_travis.py b/tests/parsers/test_travis.py index 8e6e12e..fcd796b 100644 --- a/tests/parsers/test_travis.py +++ b/tests/parsers/test_travis.py @@ -155,6 +155,23 @@ def test_drop_yaml_node_when_empty(): """) +def test_drop_yaml_node_when_text(): + source_lines = textwrap.dedent("""\ + language: python + sudo: false + python: + - 3.6 + script: pytest tests + """).splitlines(True) + result = drop_yaml_node(source_lines, 'sudo') + assert "".join(result) == textwrap.dedent("""\ + language: python + python: + - 3.6 + script: pytest tests + """) + + def test_drop_yaml_node_when_last_in_file(): source_lines = textwrap.dedent("""\ language: python From 3c6696660c48993bf242a3aeab303aa8f7c69876 Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Wed, 17 Apr 2019 14:11:40 +0300 Subject: [PATCH 36/76] Drop coverage failures on Travis Coveralls is now setting commit status properly, so I don't need this extra check. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index ebdb765..0e66ec0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,7 +8,7 @@ install: - pip install -e . script: - coverage run -m pytest tests - - coverage report -m --fail-under=100 + - coverage report -m - flake8 *.py after_script: - coveralls From fa395372694ebc75f4ca1f16445a75bfabe33359 Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Wed, 17 Apr 2019 14:13:21 +0300 Subject: [PATCH 37/76] Fix flake8 on Travis I forgot to check src/ when I moved all the files to a package. Also move flake8 into its own job. Also reindent using 2 spaces because. --- .travis.yml | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0e66ec0..760dd38 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,16 +1,21 @@ language: python dist: xenial python: - - 3.6 - - 3.7 + - 3.6 + - 3.7 +matrix: + include: + - name: flake8 + install: pip install flake8 + script: flake8 src *.py + after_script: install: - - pip install pytest coverage coveralls flake8 - - pip install -e . + - pip install pytest coverage coveralls + - pip install -e . script: - - coverage run -m pytest tests - - coverage report -m - - flake8 *.py + - coverage run -m pytest tests + - coverage report -m after_script: - - coveralls + - coveralls notifications: - email: false + email: false From 9763140ffde29b3e33919b09da96e2e850733081 Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Wed, 17 Apr 2019 14:19:25 +0300 Subject: [PATCH 38/76] Windows does not have fchmod??? --- src/check_python_versions/utils.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/check_python_versions/utils.py b/src/check_python_versions/utils.py index 2770f16..e1d73a0 100644 --- a/src/check_python_versions/utils.py +++ b/src/check_python_versions/utils.py @@ -28,7 +28,11 @@ def confirm_and_update_file(filename, new_lines): mode = stat.S_IMODE(os.stat(filename).st_mode) tempfile = filename + '.tmp' with open(tempfile, 'w') as f: - os.fchmod(f.fileno(), mode) + if hasattr(os, 'fchmod'): + os.fchmod(f.fileno(), mode) + else: + # Windows, what else? + os.chmod(tempfile, mode) f.writelines(new_lines) os.rename(tempfile, filename) From 4713b42911e690b9e4f0aeaf7a66851ad2cd8bcb Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Wed, 17 Apr 2019 14:28:59 +0300 Subject: [PATCH 39/76] More Windows pain --- src/check_python_versions/utils.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/check_python_versions/utils.py b/src/check_python_versions/utils.py index e1d73a0..a6b5c9c 100644 --- a/src/check_python_versions/utils.py +++ b/src/check_python_versions/utils.py @@ -34,7 +34,12 @@ def confirm_and_update_file(filename, new_lines): # Windows, what else? os.chmod(tempfile, mode) f.writelines(new_lines) - os.rename(tempfile, filename) + try: + os.rename(tempfile, filename) + except FileExistsError: + # No atomic replace on Windows + os.unlink(filename) + os.rename(tempfile, filename) def show_diff(filename, new_lines): From 75fd112ef9c78e845c9868230749d32f9f5fc849 Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Wed, 17 Apr 2019 14:54:41 +0300 Subject: [PATCH 40/76] Implement --dry-run --- src/check_python_versions/cli.py | 49 ++++++++++++++----- src/check_python_versions/parsers/appveyor.py | 5 +- src/check_python_versions/parsers/python.py | 12 ++--- src/check_python_versions/parsers/travis.py | 8 +-- src/check_python_versions/utils.py | 10 ++++ 5 files changed, 60 insertions(+), 24 deletions(-) diff --git a/src/check_python_versions/cli.py b/src/check_python_versions/cli.py index 948384b..c13ee44 100644 --- a/src/check_python_versions/cli.py +++ b/src/check_python_versions/cli.py @@ -1,6 +1,7 @@ import argparse import os import sys +from io import StringIO from . import __version__ from .utils import confirm_and_update_file, show_diff @@ -106,7 +107,7 @@ def check_package(where='.', *, print=print): return True -def check_versions(where='.', *, print=print, expect=None): +def check_versions(where='.', *, print=print, expect=None, replacements=None): sources = [ ('setup.py', get_supported_python_versions, 'setup.py'), @@ -126,7 +127,13 @@ def check_versions(where='.', *, print=print, expect=None): pathname = os.path.join(where, filename) if not os.path.exists(pathname): continue - versions = extractor(pathname) + if pathname in replacements: + new_lines = replacements[pathname] + buf = StringIO("\n".join(new_lines)) + buf.name = f'pathname (updated)' + versions = extractor(buf) + else: + versions = extractor(pathname) if versions is None: continue print(f"{title} says:".ljust(width), ", ".join(versions) or "(empty)") @@ -144,7 +151,7 @@ def check_versions(where='.', *, print=print, expect=None): def update_versions(where='.', *, add=None, drop=None, update=None, - diff=False): + diff=False, dry_run=False): sources = [ ('setup.py', get_supported_python_versions, @@ -154,6 +161,7 @@ def update_versions(where='.', *, add=None, drop=None, update=None, (APPVEYOR_YML, get_appveyor_yml_python_versions, update_appveyor_yml_python_versions), ] + replacements = {} for (filename, extractor, updater) in sources: pathname = os.path.join(where, filename) @@ -171,9 +179,13 @@ def update_versions(where='.', *, add=None, drop=None, update=None, if new_lines is not None: if diff: show_diff(pathname, new_lines) - else: + if dry_run: + replacements[pathname] = new_lines + if not diff and not dry_run: confirm_and_update_file(pathname, new_lines) + return replacements + def _main(): parser = argparse.ArgumentParser( @@ -193,8 +205,6 @@ def _main(): ' and other files is located') group = parser.add_argument_group( "updating supported version lists (EXPERIMENTAL)") - group.add_argument('--diff', action='store_true', - help='show a diff of proposed changes') group.add_argument('--add', metavar='VERSIONS', type=parse_version_list, help='add these versions to supported ones, e.g' ' --add 3.8') @@ -204,12 +214,23 @@ def _main(): group.add_argument('--update', metavar='VERSIONS', type=parse_version_list, help='update the set of supported versions, e.g.' ' --update 2.7,3.5-3.7') + group.add_argument('--diff', action='store_true', + help='show a diff of proposed changes') + group.add_argument('--dry-run', action='store_true', + help='verify proposed changes without' + ' writing them to disk') args = parser.parse_args() if args.update and args.add: parser.error("argument --add: not allowed with argument --update") if args.update and args.drop: parser.error("argument --drop: not allowed with argument --update") + if args.diff and not (args.update or args.add or args.drop): + parser.error( + "argument --diff: not allowed without --update/--add/--drop") + if args.dry_run and not (args.update or args.add or args.drop): + parser.error( + "argument --dry-run: not allowed without --update/--add/--drop") where = args.where or ['.'] if args.skip_non_packages: @@ -225,15 +246,19 @@ def _main(): if not check_package(path): mismatches.append(path) continue - if args.add or args.drop or args.update or args.diff: - update_versions(path, add=args.add, drop=args.drop, - update=args.update, diff=args.diff) - if not args.diff: - if not check_versions(path, expect=args.expect): + replacements = {} + if args.add or args.drop or args.update: + replacements = update_versions( + path, add=args.add, drop=args.drop, + update=args.update, diff=args.diff, + dry_run=args.dry_run) + if not args.diff or args.dry_run: + if not check_versions(path, expect=args.expect, + replacements=replacements): mismatches.append(path) continue - if not args.diff: + if not args.diff or args.dry_run: if mismatches: if multiple: sys.exit(f"\n\nmismatch in {' '.join(mismatches)}!") diff --git a/src/check_python_versions/parsers/appveyor.py b/src/check_python_versions/parsers/appveyor.py index 9842fb4..ecc3f52 100644 --- a/src/check_python_versions/parsers/appveyor.py +++ b/src/check_python_versions/parsers/appveyor.py @@ -5,13 +5,14 @@ from .tox import parse_envlist, tox_env_to_py_version from .travis import update_yaml_list +from ..utils import open_file APPVEYOR_YML = 'appveyor.yml' def get_appveyor_yml_python_versions(filename=APPVEYOR_YML): - with open(filename) as fp: + with open_file(filename) as fp: conf = yaml.safe_load(fp) # There's more than one way of doing this, I'm setting %PYTHON% to # the directory that has a Python interpreter (C:\PythonXY) @@ -63,7 +64,7 @@ def escape(s): def update_appveyor_yml_python_versions(filename, new_versions): - with open(filename) as fp: + with open_file(filename) as fp: orig_lines = fp.readlines() fp.seek(0) conf = yaml.safe_load(fp) diff --git a/src/check_python_versions/parsers/python.py b/src/check_python_versions/parsers/python.py index b9c991e..a5e8582 100644 --- a/src/check_python_versions/parsers/python.py +++ b/src/check_python_versions/parsers/python.py @@ -4,13 +4,13 @@ import string from functools import partial -from ..utils import warn, pipe +from ..utils import warn, pipe, open_file from ..versions import MAX_MINOR_FOR_MAJOR def get_supported_python_versions(filename='setup.py'): classifiers = get_setup_py_keyword(filename, 'classifiers') - if classifiers is None: + if classifiers is None and isinstance(filename, str): # AST parsing is complicated setup_py = os.path.basename(filename) classifiers = pipe("python", setup_py, "-q", "--classifiers", @@ -96,18 +96,18 @@ def update_supported_python_versions(filename, new_versions): def get_setup_py_keyword(setup_py, keyword): - with open(setup_py) as f: + with open_file(setup_py) as f: try: - tree = ast.parse(f.read(), setup_py) + tree = ast.parse(f.read(), f.name) except SyntaxError as error: - warn(f'Could not parse {setup_py}: {error}') + warn(f'Could not parse {f.name}: {error}') return None node = find_call_kwarg_in_ast(tree, 'setup', keyword) return node and eval_ast_node(node, keyword) def update_setup_py_keyword(setup_py, keyword, new_value): - with open(setup_py) as f: + with open_file(setup_py) as f: lines = f.readlines() new_lines = update_call_arg_in_source(lines, 'setup', keyword, new_value) return new_lines diff --git a/src/check_python_versions/parsers/travis.py b/src/check_python_versions/parsers/travis.py index c298b92..d7fa782 100644 --- a/src/check_python_versions/parsers/travis.py +++ b/src/check_python_versions/parsers/travis.py @@ -4,7 +4,7 @@ yaml = None from .tox import parse_envlist, tox_env_to_py_version -from ..utils import warn +from ..utils import warn, open_file from ..versions import is_important @@ -12,7 +12,7 @@ def get_travis_yml_python_versions(filename=TRAVIS_YML): - with open(filename) as fp: + with open_file(filename) as fp: conf = yaml.safe_load(fp) versions = [] if 'python' in conf: @@ -53,7 +53,7 @@ def needs_xenial(v): def update_travis_yml_python_versions(filename, new_versions): - with open(filename) as fp: + with open_file(filename) as fp: orig_lines = fp.readlines() fp.seek(0) conf = yaml.safe_load(fp) @@ -62,7 +62,7 @@ def keep_old(ver): return not is_important(travis_normalize_py_version(ver)) new_lines = update_yaml_list( - orig_lines, "python", new_versions, filename=filename, keep=keep_old, + orig_lines, "python", new_versions, filename=fp.name, keep=keep_old, ) # If python 3.7 was enabled via matrix.include, we've just added a diff --git a/src/check_python_versions/utils.py b/src/check_python_versions/utils.py index a6b5c9c..57885ad 100644 --- a/src/check_python_versions/utils.py +++ b/src/check_python_versions/utils.py @@ -4,6 +4,7 @@ import stat import subprocess import sys +from contextlib import contextmanager log = logging.getLogger('check-python-versions') @@ -13,6 +14,15 @@ def warn(msg): print(msg, file=sys.stderr) +@contextmanager +def open_file(filename_or_file_object): + if isinstance(filename_or_file_object, str): + with open(filename_or_file_object) as fp: + yield fp + else: + yield filename_or_file_object + + def pipe(*cmd, **kwargs): if 'cwd' in kwargs: log.debug('EXEC cd %s && %s', kwargs['cwd'], ' '.join(cmd)) From a10b2ddc51180441c3f465e7fe1170cd4b06b7b9 Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Wed, 17 Apr 2019 15:26:34 +0300 Subject: [PATCH 41/76] Fix tests (So it turns out pathlib.Path objects are not isinstance(..., str)!) (And there's no point checking what happens if tox.ini does not exist, as cli.py does an os.path.exists() before calling get_tox_ini_python_versions().) --- src/check_python_versions/cli.py | 2 +- src/check_python_versions/parsers/python.py | 4 ++-- src/check_python_versions/utils.py | 10 +++++++--- tests/parsers/test_tox.py | 5 ----- 4 files changed, 10 insertions(+), 11 deletions(-) diff --git a/src/check_python_versions/cli.py b/src/check_python_versions/cli.py index c13ee44..6066a44 100644 --- a/src/check_python_versions/cli.py +++ b/src/check_python_versions/cli.py @@ -127,7 +127,7 @@ def check_versions(where='.', *, print=print, expect=None, replacements=None): pathname = os.path.join(where, filename) if not os.path.exists(pathname): continue - if pathname in replacements: + if replacements and pathname in replacements: new_lines = replacements[pathname] buf = StringIO("\n".join(new_lines)) buf.name = f'pathname (updated)' diff --git a/src/check_python_versions/parsers/python.py b/src/check_python_versions/parsers/python.py index a5e8582..402f4b2 100644 --- a/src/check_python_versions/parsers/python.py +++ b/src/check_python_versions/parsers/python.py @@ -4,13 +4,13 @@ import string from functools import partial -from ..utils import warn, pipe, open_file +from ..utils import warn, pipe, open_file, is_file_object from ..versions import MAX_MINOR_FOR_MAJOR def get_supported_python_versions(filename='setup.py'): classifiers = get_setup_py_keyword(filename, 'classifiers') - if classifiers is None and isinstance(filename, str): + if classifiers is None and not is_file_object(filename): # AST parsing is complicated setup_py = os.path.basename(filename) classifiers = pipe("python", setup_py, "-q", "--classifiers", diff --git a/src/check_python_versions/utils.py b/src/check_python_versions/utils.py index 57885ad..b358e2d 100644 --- a/src/check_python_versions/utils.py +++ b/src/check_python_versions/utils.py @@ -14,13 +14,17 @@ def warn(msg): print(msg, file=sys.stderr) +def is_file_object(filename_or_file_object): + return hasattr(filename_or_file_object, 'read') + + @contextmanager def open_file(filename_or_file_object): - if isinstance(filename_or_file_object, str): + if is_file_object(filename_or_file_object): + yield filename_or_file_object + else: with open(filename_or_file_object) as fp: yield fp - else: - yield filename_or_file_object def pipe(*cmd, **kwargs): diff --git a/tests/parsers/test_tox.py b/tests/parsers/test_tox.py index 238b7dc..87cf78b 100644 --- a/tests/parsers/test_tox.py +++ b/tests/parsers/test_tox.py @@ -19,11 +19,6 @@ def test_get_tox_ini_python_versions(tmp_path): assert get_tox_ini_python_versions(tox_ini) == ['2.7', '3.6'] -def test_get_tox_ini_python_versions_no_tox_ini(tmp_path): - tox_ini = tmp_path / "tox.ini" - assert get_tox_ini_python_versions(tox_ini) == [] - - def test_get_tox_ini_python_versions_syntax_error(tmp_path): tox_ini = tmp_path / "tox.ini" tox_ini.write_text(textwrap.dedent("""\ From d327ffdbe343f487a26f7ce99f49aafb7c6a5f05 Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Wed, 17 Apr 2019 15:18:06 +0300 Subject: [PATCH 42/76] Rudimentary tox.ini updater --- src/check_python_versions/cli.py | 5 ++ src/check_python_versions/parsers/tox.py | 70 +++++++++++++++++++++++- src/check_python_versions/utils.py | 4 ++ tests/parsers/test_tox.py | 59 ++++++++++++++++++++ 4 files changed, 137 insertions(+), 1 deletion(-) diff --git a/src/check_python_versions/cli.py b/src/check_python_versions/cli.py index 6066a44..281761a 100644 --- a/src/check_python_versions/cli.py +++ b/src/check_python_versions/cli.py @@ -18,6 +18,7 @@ from .parsers.tox import ( TOX_INI, get_tox_ini_python_versions, + update_tox_ini_python_versions, ) from .parsers.travis import ( TRAVIS_YML, @@ -156,10 +157,14 @@ def update_versions(where='.', *, add=None, drop=None, update=None, sources = [ ('setup.py', get_supported_python_versions, update_supported_python_versions), + (TOX_INI, get_tox_ini_python_versions, + update_tox_ini_python_versions), (TRAVIS_YML, get_travis_yml_python_versions, update_travis_yml_python_versions), (APPVEYOR_YML, get_appveyor_yml_python_versions, update_appveyor_yml_python_versions), + # TODO: manylinux.sh + # TODO: CHANGES.rst ] replacements = {} diff --git a/src/check_python_versions/parsers/tox.py b/src/check_python_versions/parsers/tox.py index ec664b4..838d522 100644 --- a/src/check_python_versions/parsers/tox.py +++ b/src/check_python_versions/parsers/tox.py @@ -1,6 +1,8 @@ import configparser import re +from ..utils import warn, open_file, get_indent + TOX_INI = 'tox.ini' @@ -8,7 +10,8 @@ def get_tox_ini_python_versions(filename=TOX_INI): conf = configparser.ConfigParser() try: - conf.read(filename) + with open_file(filename) as fp: + conf.read_file(fp) envlist = conf.get('tox', 'envlist') except configparser.Error: return [] @@ -50,3 +53,68 @@ def tox_env_to_py_version(env): return f'{env[2]}.{env[3:]}' else: return env + + +def update_tox_ini_python_versions(filename, new_versions): + with open_file(filename) as fp: + orig_lines = fp.readlines() + fp.seek(0) + conf = configparser.ConfigParser() + try: + conf.read_file(fp) + envlist = conf.get('tox', 'envlist') + except configparser.Error: + return orig_lines + + sep = ',' + if ', ' in envlist: + sep = ', ' + + new_envlist = sep.join( + f"py{ver.replace('.', '')}" + for ver in new_versions + ) + + new_lines = update_ini_setting( + orig_lines, 'tox', 'envlist', new_envlist, + ) + return new_lines + + +def update_ini_setting(orig_lines, section, key, new_value, filename=TOX_INI): + lines = iter(enumerate(orig_lines)) + for n, line in lines: + if line.startswith(f'[{section}]'): + break + else: + warn(f'Did not find [{section}] in {filename}') + return orig_lines + + # TODO: use a regex to allow an arbitrary number of spaces around = + for n, line in lines: + if line.startswith(f'{key} ='): + start = n + break + else: + warn(f'Did not find {key}= in [{section}] in {filename}') + return orig_lines + + end = start + 1 + for n, line in lines: + if line.startswith(' '): + end = n + 1 + else: + break + + prefix = ' ' + firstline = orig_lines[start].strip().expandtabs().replace(' ', '') + if firstline == f'{key}=': + if end > start + 1: + indent = get_indent(orig_lines[start + 1]) + prefix = f'\n{indent}' + + new_lines = orig_lines[:start] + ( + f"{key} ={prefix}{new_value}\n" + ).splitlines(True) + orig_lines[end:] + + return new_lines diff --git a/src/check_python_versions/utils.py b/src/check_python_versions/utils.py index b358e2d..f54b5d6 100644 --- a/src/check_python_versions/utils.py +++ b/src/check_python_versions/utils.py @@ -10,6 +10,10 @@ log = logging.getLogger('check-python-versions') +def get_indent(line): + return line[:-len(line.lstrip())] + + def warn(msg): print(msg, file=sys.stderr) diff --git a/tests/parsers/test_tox.py b/tests/parsers/test_tox.py index 87cf78b..9ecedf0 100644 --- a/tests/parsers/test_tox.py +++ b/tests/parsers/test_tox.py @@ -1,4 +1,5 @@ import textwrap +from io import StringIO import pytest @@ -7,6 +8,8 @@ get_tox_ini_python_versions, parse_envlist, tox_env_to_py_version, + update_ini_setting, + update_tox_ini_python_versions, ) @@ -76,3 +79,59 @@ def test_brace_expand(s, expected): ]) def test_tox_env_to_py_version(s, expected): assert tox_env_to_py_version(s) == expected + + +def test_update_tox_ini_python_versions(): + fp = StringIO(textwrap.dedent("""\ + [tox] + envlist = py26, py27 + """)) + result = update_tox_ini_python_versions(fp, ['3.6', '3.7']) + assert "".join(result) == textwrap.dedent("""\ + [tox] + envlist = py36, py37 + """) + + +def test_update_ini_setting(): + source_lines = textwrap.dedent("""\ + [tox] + envlist = py26,py27 + usedevelop = true + """).splitlines(True) + result = update_ini_setting(source_lines, 'tox', 'envlist', 'py36,py37') + assert "".join(result) == textwrap.dedent("""\ + [tox] + envlist = py36,py37 + usedevelop = true + """) + + +def test_update_ini_setting_from_empty(): + source_lines = textwrap.dedent("""\ + [tox] + envlist = + usedevelop = true + """).splitlines(True) + result = update_ini_setting(source_lines, 'tox', 'envlist', 'py36,py37') + assert "".join(result) == textwrap.dedent("""\ + [tox] + envlist = py36,py37 + usedevelop = true + """) + + +def test_update_ini_setting_multiline(): + source_lines = textwrap.dedent("""\ + [tox] + envlist = + py26,py27 + usedevelop = true + """).splitlines(True) + result = update_ini_setting(source_lines, 'tox', 'envlist', 'py36,py37') + assert "".join(result) == textwrap.dedent("""\ + [tox] + envlist = + py36,py37 + usedevelop = true + """) From 7120ceb2d5ea4bc61fe06056610bf3709d8fee30 Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Wed, 17 Apr 2019 15:47:31 +0300 Subject: [PATCH 43/76] Preserve other tox environments This is still very hacky and e.g. expands brace expansion unnecessarily. --- src/check_python_versions/parsers/tox.py | 26 ++++++++++++++++++------ tests/parsers/test_tox.py | 6 ++++++ 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/src/check_python_versions/parsers/tox.py b/src/check_python_versions/parsers/tox.py index 838d522..93a7574 100644 --- a/src/check_python_versions/parsers/tox.py +++ b/src/check_python_versions/parsers/tox.py @@ -2,6 +2,7 @@ import re from ..utils import warn, open_file, get_indent +from ..versions import is_important TOX_INI = 'tox.ini' @@ -66,19 +67,32 @@ def update_tox_ini_python_versions(filename, new_versions): except configparser.Error: return orig_lines + new_envlist = update_tox_envlist(envlist, new_versions) + + new_lines = update_ini_setting( + orig_lines, 'tox', 'envlist', new_envlist, + ) + return new_lines + + +def update_tox_envlist(envlist, new_versions): sep = ',' if ', ' in envlist: sep = ', ' - new_envlist = sep.join( + envlist = parse_envlist(envlist) + keep = [] + for env in envlist: + if (not env.startswith('py') + or not is_important(tox_env_to_py_version(env))): + keep.append(env) + + new_envlist = sep.join([ f"py{ver.replace('.', '')}" for ver in new_versions - ) + ] + keep) - new_lines = update_ini_setting( - orig_lines, 'tox', 'envlist', new_envlist, - ) - return new_lines + return new_envlist def update_ini_setting(orig_lines, section, key, new_value, filename=TOX_INI): diff --git a/tests/parsers/test_tox.py b/tests/parsers/test_tox.py index 9ecedf0..c1694b8 100644 --- a/tests/parsers/test_tox.py +++ b/tests/parsers/test_tox.py @@ -9,6 +9,7 @@ parse_envlist, tox_env_to_py_version, update_ini_setting, + update_tox_envlist, update_tox_ini_python_versions, ) @@ -93,6 +94,11 @@ def test_update_tox_ini_python_versions(): """) +def test_update_tox_envlist(): + result = update_tox_envlist('py26,py27,pypy', ['3.6', '3.7']) + assert result == 'py36,py37,pypy' + + def test_update_ini_setting(): source_lines = textwrap.dedent("""\ [tox] From cf763d8707ca0ed069ef47ce373fea280a5de9b8 Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Wed, 17 Apr 2019 15:52:02 +0300 Subject: [PATCH 44/76] Preserve tox environments with dashes But only when the base environment is preserved. --- src/check_python_versions/parsers/tox.py | 11 +++++++++-- tests/parsers/test_tox.py | 7 +++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/src/check_python_versions/parsers/tox.py b/src/check_python_versions/parsers/tox.py index 93a7574..d8e4e30 100644 --- a/src/check_python_versions/parsers/tox.py +++ b/src/check_python_versions/parsers/tox.py @@ -83,9 +83,16 @@ def update_tox_envlist(envlist, new_versions): envlist = parse_envlist(envlist) keep = [] for env in envlist: - if (not env.startswith('py') - or not is_important(tox_env_to_py_version(env))): + if not env.startswith('py'): keep.append(env) + continue + if not is_important(tox_env_to_py_version(env)): + keep.append(env) + continue + if '-' in env: + baseversion = tox_env_to_py_version(env) + if baseversion in new_versions: + keep.append(env) new_envlist = sep.join([ f"py{ver.replace('.', '')}" diff --git a/tests/parsers/test_tox.py b/tests/parsers/test_tox.py index c1694b8..05236a7 100644 --- a/tests/parsers/test_tox.py +++ b/tests/parsers/test_tox.py @@ -99,6 +99,13 @@ def test_update_tox_envlist(): assert result == 'py36,py37,pypy' +def test_update_tox_envlist_with_suffixes(): + result = update_tox_envlist( + 'py27,py34,py35,py36,py37,py27-numpy,py37-numpy,pypy,pypy3', + ['3.6', '3.7']) + assert result == 'py36,py37,py37-numpy,pypy,pypy3' + + def test_update_ini_setting(): source_lines = textwrap.dedent("""\ [tox] From 08588bde74a193d1aca7ba21e61b39e3cc5c3f05 Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Wed, 17 Apr 2019 16:29:41 +0300 Subject: [PATCH 45/76] Code to update python_requires in setup.py --- src/check_python_versions/cli.py | 29 ++++-- src/check_python_versions/parsers/python.py | 103 ++++++++++++++------ src/check_python_versions/utils.py | 6 +- tests/parsers/test_python.py | 20 +++- 4 files changed, 114 insertions(+), 44 deletions(-) diff --git a/src/check_python_versions/cli.py b/src/check_python_versions/cli.py index 281761a..85e0530 100644 --- a/src/check_python_versions/cli.py +++ b/src/check_python_versions/cli.py @@ -13,6 +13,7 @@ from .parsers.python import ( get_supported_python_versions, get_python_requires, + update_python_requires, update_supported_python_versions, ) from .parsers.tox import ( @@ -108,6 +109,16 @@ def check_package(where='.', *, print=print): return True +def filename_or_replacement(pathname, replacements): + if replacements and pathname in replacements: + new_lines = replacements[pathname] + buf = StringIO("".join(new_lines)) + buf.name = pathname + return buf + else: + return pathname + + def check_versions(where='.', *, print=print, expect=None, replacements=None): sources = [ @@ -128,13 +139,7 @@ def check_versions(where='.', *, print=print, expect=None, replacements=None): pathname = os.path.join(where, filename) if not os.path.exists(pathname): continue - if replacements and pathname in replacements: - new_lines = replacements[pathname] - buf = StringIO("\n".join(new_lines)) - buf.name = f'pathname (updated)' - versions = extractor(buf) - else: - versions = extractor(pathname) + versions = extractor(filename_or_replacement(pathname, replacements)) if versions is None: continue print(f"{title} says:".ljust(width), ", ".join(versions) or "(empty)") @@ -157,6 +162,8 @@ def update_versions(where='.', *, add=None, drop=None, update=None, sources = [ ('setup.py', get_supported_python_versions, update_supported_python_versions), + ('setup.py', get_python_requires, + update_python_requires), (TOX_INI, get_tox_ini_python_versions, update_tox_ini_python_versions), (TRAVIS_YML, get_travis_yml_python_versions, @@ -172,7 +179,7 @@ def update_versions(where='.', *, add=None, drop=None, update=None, pathname = os.path.join(where, filename) if not os.path.exists(pathname): continue - versions = extractor(pathname) + versions = extractor(filename_or_replacement(pathname, replacements)) if versions is None: continue @@ -180,10 +187,12 @@ def update_versions(where='.', *, add=None, drop=None, update=None, new_versions = update_version_list( versions, add=add, drop=drop, update=update) if versions != new_versions: - new_lines = updater(pathname, new_versions) + fp = filename_or_replacement(pathname, replacements) + new_lines = updater(fp, new_versions) if new_lines is not None: if diff: - show_diff(pathname, new_lines) + fp = filename_or_replacement(pathname, replacements) + show_diff(fp, new_lines) if dry_run: replacements[pathname] = new_lines if not diff and not dry_run: diff --git a/src/check_python_versions/parsers/python.py b/src/check_python_versions/parsers/python.py index 402f4b2..3abd3e0 100644 --- a/src/check_python_versions/parsers/python.py +++ b/src/check_python_versions/parsers/python.py @@ -15,6 +15,8 @@ def get_supported_python_versions(filename='setup.py'): setup_py = os.path.basename(filename) classifiers = pipe("python", setup_py, "-q", "--classifiers", cwd=os.path.dirname(filename)).splitlines() + if classifiers is None: + return [] return get_versions_from_classifiers(classifiers) @@ -95,6 +97,17 @@ def update_supported_python_versions(filename, new_versions): return update_setup_py_keyword(filename, 'classifiers', new_classifiers) +def update_python_requires(filename, new_versions): + python_requires = get_setup_py_keyword(filename, 'python_requires') + if python_requires is None: + return None + new_python_requires = compute_python_requires(new_versions) + if is_file_object(filename): + filename.seek(0) + return update_setup_py_keyword(filename, 'python_requires', + new_python_requires) + + def get_setup_py_keyword(setup_py, keyword): with open_file(setup_py) as f: try: @@ -117,9 +130,10 @@ def to_literal(value, quote_style='"'): # Because I don't want to deal with quoting, I'll require all values # to contain only safe characters (i.e. no ' or " or \). Except some # PyPI classifiers do include ' so I need to handle that at least. - safe_characters = string.ascii_letters + string.digits + " .:,-=><()/+'#" + # And python_requires uses all sorts of comparisons like ~= 3.7.* + safe_chars = string.ascii_letters + string.digits + " .:,-=>={min_version}'] + for major in sorted(MAX_MINOR_FOR_MAJOR): + for minor in range(0, MAX_MINOR_FOR_MAJOR[major] + 1): + ver = f'{major}.{minor}' + if ver >= min_version and ver not in new_versions: + specifiers.append(f'!={ver}.*') + return ', '.join(specifiers) diff --git a/src/check_python_versions/utils.py b/src/check_python_versions/utils.py index f54b5d6..a657f09 100644 --- a/src/check_python_versions/utils.py +++ b/src/check_python_versions/utils.py @@ -60,10 +60,10 @@ def confirm_and_update_file(filename, new_lines): os.rename(tempfile, filename) -def show_diff(filename, new_lines): - with open(filename, 'r') as f: +def show_diff(filename_or_file_object, new_lines): + with open_file(filename_or_file_object) as f: old_lines = f.readlines() - print_diff(old_lines, new_lines, filename) + print_diff(old_lines, new_lines, f.name) return old_lines != new_lines diff --git a/tests/parsers/test_python.py b/tests/parsers/test_python.py index 831bd83..7fa3643 100644 --- a/tests/parsers/test_python.py +++ b/tests/parsers/test_python.py @@ -307,7 +307,25 @@ def test_to_literal_all_the_classifiers(): assert ast.literal_eval(literal) == classifier -def test_update_call_arg_in_source(): +def test_update_call_arg_in_source_string(): + source_lines = textwrap.dedent("""\ + setup( + foo=1, + bar="x", + baz=2, + ) + """).splitlines(True) + result = update_call_arg_in_source(source_lines, "setup", "bar", "y") + assert "".join(result) == textwrap.dedent("""\ + setup( + foo=1, + bar="y", + baz=2, + ) + """) + + +def test_update_call_arg_in_source_list(): source_lines = textwrap.dedent("""\ setup( foo=1, From 32e1429e5b5c50161a0f7714651e5f1c2ca9535e Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Wed, 17 Apr 2019 16:32:57 +0300 Subject: [PATCH 46/76] Show directory names when using --dry-run --diff --- src/check_python_versions/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/check_python_versions/cli.py b/src/check_python_versions/cli.py index 85e0530..00bbb5e 100644 --- a/src/check_python_versions/cli.py +++ b/src/check_python_versions/cli.py @@ -253,7 +253,7 @@ def _main(): multiple = len(where) > 1 mismatches = [] for n, path in enumerate(where): - if multiple and not args.diff: + if multiple and (not args.diff or args.dry_run): if n: print("\n") print(f"{path}:\n") From cb3a7dbfd5fe3c913d57ea619cfaec667f8fd980 Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Wed, 17 Apr 2019 16:56:20 +0300 Subject: [PATCH 47/76] Fix crash in get_travis_yml_python_versions I think this happens when the updater deletes all 'python 'lines, leaving a python: in YAML, which reads as `null` rather than an empty list. --- src/check_python_versions/parsers/travis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/check_python_versions/parsers/travis.py b/src/check_python_versions/parsers/travis.py index d7fa782..fda0078 100644 --- a/src/check_python_versions/parsers/travis.py +++ b/src/check_python_versions/parsers/travis.py @@ -15,7 +15,7 @@ def get_travis_yml_python_versions(filename=TRAVIS_YML): with open_file(filename) as fp: conf = yaml.safe_load(fp) versions = [] - if 'python' in conf: + if conf.get('python'): versions += map(travis_normalize_py_version, conf['python']) if 'matrix' in conf and 'include' in conf['matrix']: for job in conf['matrix']['include']: From 74fafa7a4a5525e0d092c7f82fa90f142c172ae2 Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Wed, 17 Apr 2019 16:56:56 +0300 Subject: [PATCH 48/76] Implement --only --- src/check_python_versions/cli.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/src/check_python_versions/cli.py b/src/check_python_versions/cli.py index 00bbb5e..a22729c 100644 --- a/src/check_python_versions/cli.py +++ b/src/check_python_versions/cli.py @@ -119,7 +119,8 @@ def filename_or_replacement(pathname, replacements): return pathname -def check_versions(where='.', *, print=print, expect=None, replacements=None): +def check_versions(where='.', *, print=print, expect=None, replacements=None, + only=None): sources = [ ('setup.py', get_supported_python_versions, 'setup.py'), @@ -136,6 +137,8 @@ def check_versions(where='.', *, print=print, expect=None, replacements=None): version_sets = [] for (title, extractor, filename) in sources: + if only and filename not in only: + continue pathname = os.path.join(where, filename) if not os.path.exists(pathname): continue @@ -157,7 +160,7 @@ def check_versions(where='.', *, print=print, expect=None, replacements=None): def update_versions(where='.', *, add=None, drop=None, update=None, - diff=False, dry_run=False): + diff=False, dry_run=False, only=None): sources = [ ('setup.py', get_supported_python_versions, @@ -176,6 +179,8 @@ def update_versions(where='.', *, add=None, drop=None, update=None, replacements = {} for (filename, extractor, updater) in sources: + if only and filename not in only: + continue pathname = os.path.join(where, filename) if not os.path.exists(pathname): continue @@ -214,6 +219,9 @@ def _main(): parser.add_argument('--skip-non-packages', action='store_true', help='skip arguments that are not Python packages' ' without warning about them') + parser.add_argument('--only', + help='check only the specified files' + ' (comma-separated list)') parser.add_argument('where', nargs='*', help='directory where a Python package with a setup.py' ' and other files is located') @@ -250,6 +258,8 @@ def _main(): if args.skip_non_packages: where = [path for path in where if is_package(path)] + only = [a.strip() for a in args.only.split(',')] if args.only else None + multiple = len(where) > 1 mismatches = [] for n, path in enumerate(where): @@ -265,10 +275,11 @@ def _main(): replacements = update_versions( path, add=args.add, drop=args.drop, update=args.update, diff=args.diff, - dry_run=args.dry_run) + dry_run=args.dry_run, only=only) if not args.diff or args.dry_run: if not check_versions(path, expect=args.expect, - replacements=replacements): + replacements=replacements, + only=only): mismatches.append(path) continue From d54bf5c86eb11eedb7a585bad4b63039ccb41c45 Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Wed, 17 Apr 2019 17:20:39 +0300 Subject: [PATCH 49/76] Allow spaces around = in setup.py --- src/check_python_versions/parsers/python.py | 25 ++++++++++++--------- tests/parsers/test_python.py | 20 +++++++++++++++++ 2 files changed, 34 insertions(+), 11 deletions(-) diff --git a/src/check_python_versions/parsers/python.py b/src/check_python_versions/parsers/python.py index 3abd3e0..3da8554 100644 --- a/src/check_python_versions/parsers/python.py +++ b/src/check_python_versions/parsers/python.py @@ -4,7 +4,7 @@ import string from functools import partial -from ..utils import warn, pipe, open_file, is_file_object +from ..utils import warn, pipe, open_file, is_file_object, get_indent from ..versions import MAX_MINOR_FOR_MAJOR @@ -149,11 +149,14 @@ def update_call_arg_in_source(source_lines, function, keyword, new_value): else: warn(f'Did not find {function}() call') return source_lines + eq = '=' + rx = re.compile(f'^(?P\\s*){re.escape(keyword)}(?P\\s*=\\s*)') for n, line in lines: - stripped = line.lstrip() - if stripped.startswith(f'{keyword}='): - first_indent = len(line) - len(stripped) - must_fix_indents = not line.rstrip().endswith('=[') + m = rx.match(line) + if m: + eq = m.group('eq') + first_indent = m.group('indent') + must_fix_indents = not line.rstrip().endswith('[') break else: warn(f'Did not find {keyword}= argument in {function}() call') @@ -163,7 +166,7 @@ def update_call_arg_in_source(source_lines, function, keyword, new_value): if isinstance(new_value, list): start = n - indent = first_indent + 4 + indent = first_indent + ' ' * 4 fix_closing_bracket = False for n, line in lines: stripped = line.lstrip() @@ -172,7 +175,7 @@ def update_call_arg_in_source(source_lines, function, keyword, new_value): break elif stripped: if not must_fix_indents: - indent = len(line) - len(stripped) + indent = get_indent(line) if stripped[0] in ('"', "'"): quote_style = stripped[0] if line.rstrip().endswith('],'): @@ -191,12 +194,12 @@ def update_call_arg_in_source(source_lines, function, keyword, new_value): if isinstance(new_value, list): return source_lines[:start] + [ - f"{' ' * first_indent}{keyword}=[\n" + f"{first_indent}{keyword}{eq}[\n" ] + [ - f"{' ' * indent}{to_literal(value, quote_style)},\n" + f"{indent}{to_literal(value, quote_style)},\n" for value in new_value ] + ([ - f"{' ' * first_indent}],\n" + f"{first_indent}],\n" ] if fix_closing_bracket else [ ]) + source_lines[end:] else: @@ -204,7 +207,7 @@ def update_call_arg_in_source(source_lines, function, keyword, new_value): quote_style = "'" new_value_quoted = to_literal(new_value, quote_style) return source_lines[:start] + [ - f"{' ' * first_indent}{keyword}={new_value_quoted},\n" + f"{first_indent}{keyword}{eq}{new_value_quoted},\n" ] + source_lines[end:] diff --git a/tests/parsers/test_python.py b/tests/parsers/test_python.py index 7fa3643..c52b778 100644 --- a/tests/parsers/test_python.py +++ b/tests/parsers/test_python.py @@ -325,6 +325,26 @@ def test_update_call_arg_in_source_string(): """) +def test_update_call_arg_in_source_string_spaces(): + # This is against PEP-8 but there are setup.py files out there that do + # not follow PEP-8. + source_lines = textwrap.dedent("""\ + setup( + foo = 1, + bar = "x", + baz = 2, + ) + """).splitlines(True) + result = update_call_arg_in_source(source_lines, "setup", "bar", "y") + assert "".join(result) == textwrap.dedent("""\ + setup( + foo = 1, + bar = "y", + baz = 2, + ) + """) + + def test_update_call_arg_in_source_list(): source_lines = textwrap.dedent("""\ setup( From eb8049f73303d3b3852d481c252921c1ab8a2373 Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Wed, 17 Apr 2019 17:27:10 +0300 Subject: [PATCH 50/76] Clean up Python source update logic a bit --- src/check_python_versions/parsers/python.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/check_python_versions/parsers/python.py b/src/check_python_versions/parsers/python.py index 3da8554..8b6930d 100644 --- a/src/check_python_versions/parsers/python.py +++ b/src/check_python_versions/parsers/python.py @@ -150,13 +150,15 @@ def update_call_arg_in_source(source_lines, function, keyword, new_value): warn(f'Did not find {function}() call') return source_lines eq = '=' - rx = re.compile(f'^(?P\\s*){re.escape(keyword)}(?P\\s*=\\s*)') + rx = re.compile( + f'^(?P\\s*){re.escape(keyword)}(?P\\s*=\\s*)(?P.*)' + ) for n, line in lines: m = rx.match(line) if m: + first_match = m eq = m.group('eq') first_indent = m.group('indent') - must_fix_indents = not line.rstrip().endswith('[') break else: warn(f'Did not find {keyword}= argument in {function}() call') @@ -167,6 +169,7 @@ def update_call_arg_in_source(source_lines, function, keyword, new_value): if isinstance(new_value, list): start = n indent = first_indent + ' ' * 4 + must_fix_indents = first_match.group('rest').rstrip() != '[' fix_closing_bracket = False for n, line in lines: stripped = line.lstrip() @@ -203,7 +206,7 @@ def update_call_arg_in_source(source_lines, function, keyword, new_value): ] if fix_closing_bracket else [ ]) + source_lines[end:] else: - if line.lstrip().startswith(f"{keyword}='"): + if first_match.group('rest').startswith("'"): quote_style = "'" new_value_quoted = to_literal(new_value, quote_style) return source_lines[:start] + [ From 3d54845d364a11b9354748bdd2ee55621f16b0de Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Wed, 17 Apr 2019 17:31:15 +0300 Subject: [PATCH 51/76] Handle classifiers=[] (an empty list on one line) --- src/check_python_versions/parsers/python.py | 46 +++++++++++---------- tests/parsers/test_python.py | 20 +++++++++ 2 files changed, 45 insertions(+), 21 deletions(-) diff --git a/src/check_python_versions/parsers/python.py b/src/check_python_versions/parsers/python.py index 8b6930d..2e27b1d 100644 --- a/src/check_python_versions/parsers/python.py +++ b/src/check_python_versions/parsers/python.py @@ -169,28 +169,32 @@ def update_call_arg_in_source(source_lines, function, keyword, new_value): if isinstance(new_value, list): start = n indent = first_indent + ' ' * 4 - must_fix_indents = first_match.group('rest').rstrip() != '[' - fix_closing_bracket = False - for n, line in lines: - stripped = line.lstrip() - if stripped.startswith(']'): - end = n - break - elif stripped: - if not must_fix_indents: - indent = get_indent(line) - if stripped[0] in ('"', "'"): - quote_style = stripped[0] - if line.rstrip().endswith('],'): - end = n + 1 - fix_closing_bracket = True - break + if first_match.group('rest').startswith('[]'): + fix_closing_bracket = True + end = n + 1 else: - warn( - f'Did not understand {keyword}= formatting' - f' in {function}() call' - ) - return source_lines + must_fix_indents = first_match.group('rest').rstrip() != '[' + fix_closing_bracket = False + for n, line in lines: + stripped = line.lstrip() + if stripped.startswith(']'): + end = n + break + elif stripped: + if not must_fix_indents: + indent = get_indent(line) + if stripped[0] in ('"', "'"): + quote_style = stripped[0] + if line.rstrip().endswith('],'): + end = n + 1 + fix_closing_bracket = True + break + else: + warn( + f'Did not understand {keyword}= formatting' + f' in {function}() call' + ) + return source_lines else: start = n end = n + 1 diff --git a/tests/parsers/test_python.py b/tests/parsers/test_python.py index c52b778..ea3ab8e 100644 --- a/tests/parsers/test_python.py +++ b/tests/parsers/test_python.py @@ -439,6 +439,26 @@ def test_update_call_arg_in_source_fixes_opening_bracket(): """) +def test_update_call_arg_in_source_handles_empty_list(): + source_lines = textwrap.dedent("""\ + setup(foo=1, + bar=[], + baz=2, + ) + """).splitlines(True) + result = update_call_arg_in_source(source_lines, "setup", "bar", + ["x", "y"]) + assert "".join(result) == textwrap.dedent("""\ + setup(foo=1, + bar=[ + "x", + "y", + ], + baz=2, + ) + """) + + def test_update_call_arg_in_source_no_function_call(capsys): source_lines = textwrap.dedent("""\ """).splitlines(True) From de190c2e990f68d10b7e0bbbe8292f19fc807dc9 Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Wed, 17 Apr 2019 17:39:26 +0300 Subject: [PATCH 52/76] Handle space before opening parenthesis --- src/check_python_versions/parsers/python.py | 3 ++- tests/parsers/test_python.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/check_python_versions/parsers/python.py b/src/check_python_versions/parsers/python.py index 2e27b1d..4c0e9a2 100644 --- a/src/check_python_versions/parsers/python.py +++ b/src/check_python_versions/parsers/python.py @@ -143,8 +143,9 @@ def to_literal(value, quote_style='"'): def update_call_arg_in_source(source_lines, function, keyword, new_value): lines = iter(enumerate(source_lines)) + rx = re.compile(f'^{re.escape(function)}\\s*\\(') for n, line in lines: - if line.startswith(f'{function}('): + if rx.match(line): break else: warn(f'Did not find {function}() call') diff --git a/tests/parsers/test_python.py b/tests/parsers/test_python.py index ea3ab8e..660e28c 100644 --- a/tests/parsers/test_python.py +++ b/tests/parsers/test_python.py @@ -329,7 +329,7 @@ def test_update_call_arg_in_source_string_spaces(): # This is against PEP-8 but there are setup.py files out there that do # not follow PEP-8. source_lines = textwrap.dedent("""\ - setup( + setup ( foo = 1, bar = "x", baz = 2, @@ -337,7 +337,7 @@ def test_update_call_arg_in_source_string_spaces(): """).splitlines(True) result = update_call_arg_in_source(source_lines, "setup", "bar", "y") assert "".join(result) == textwrap.dedent("""\ - setup( + setup ( foo = 1, bar = "y", baz = 2, From 8acef058aefe5a3ab0f686f7940dc0ea223ccebf Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Wed, 17 Apr 2019 17:53:31 +0300 Subject: [PATCH 53/76] Add a manylinux-install.sh updater --- src/check_python_versions/cli.py | 4 +- .../parsers/manylinux.py | 36 ++++++++++++- tests/parsers/test_manylinux.py | 51 +++++++++++++++++++ 3 files changed, 89 insertions(+), 2 deletions(-) diff --git a/src/check_python_versions/cli.py b/src/check_python_versions/cli.py index a22729c..08897bf 100644 --- a/src/check_python_versions/cli.py +++ b/src/check_python_versions/cli.py @@ -34,6 +34,7 @@ from .parsers.manylinux import ( MANYLINUX_INSTALL_SH, get_manylinux_python_versions, + update_manylinux_python_versions, ) try: @@ -173,7 +174,8 @@ def update_versions(where='.', *, add=None, drop=None, update=None, update_travis_yml_python_versions), (APPVEYOR_YML, get_appveyor_yml_python_versions, update_appveyor_yml_python_versions), - # TODO: manylinux.sh + (MANYLINUX_INSTALL_SH, get_manylinux_python_versions, + update_manylinux_python_versions), # TODO: CHANGES.rst ] replacements = {} diff --git a/src/check_python_versions/parsers/manylinux.py b/src/check_python_versions/parsers/manylinux.py index 0d959de..b275cee 100644 --- a/src/check_python_versions/parsers/manylinux.py +++ b/src/check_python_versions/parsers/manylinux.py @@ -1,14 +1,48 @@ import re +from ..utils import open_file, warn + MANYLINUX_INSTALL_SH = '.manylinux-install.sh' def get_manylinux_python_versions(filename=MANYLINUX_INSTALL_SH): magic = re.compile(r'.*\[\[ "\$\{PYBIN\}" == \*"cp(\d)(\d)"\* \]\]') versions = [] - with open(filename) as fp: + with open_file(filename) as fp: for line in fp: m = magic.match(line) if m: versions.append('{}.{}'.format(*m.groups())) return sorted(set(versions)) + + +def update_manylinux_python_versions(filename, new_versions): + magic = re.compile(r'.*\[\[ "\$\{PYBIN\}" == \*"cp(\d)(\d)"\* \]\]') + with open_file(filename) as f: + orig_lines = f.readlines() + lines = iter(enumerate(orig_lines)) + for n, line in lines: + m = magic.match(line) + if m: + start = n + break + else: + warn(f'Failed to understand {filename}') + for n, line in lines: + m = magic.match(line) + if not m: + end = n + break + else: + warn(f'Failed to understand {filename}') + + indent = ' ' * 4 + conditions = f' || \\\n{indent} '.join( + f'[[ "${{PYBIN}}" == *"cp{ver.replace(".", "")}"* ]]' + for ver in new_versions + ) + new_lines = orig_lines[:start] + ( + f'{indent}if {conditions}; then\n' + ).splitlines(True) + orig_lines[end:] + + return new_lines diff --git a/tests/parsers/test_manylinux.py b/tests/parsers/test_manylinux.py index 5832ed0..598949e 100644 --- a/tests/parsers/test_manylinux.py +++ b/tests/parsers/test_manylinux.py @@ -1,7 +1,9 @@ import textwrap +from io import StringIO from check_python_versions.parsers.manylinux import ( get_manylinux_python_versions, + update_manylinux_python_versions, ) @@ -33,3 +35,52 @@ def test_get_manylinux_python_versions(tmp_path): assert get_manylinux_python_versions(manylinux_install_sh) == [ '2.7', '3.4', '3.5', '3.6', '3.7', ] + + +def test_update_manylinux_python_versions(): + manylinux_install_sh = StringIO(textwrap.dedent(r""" + #!/usr/bin/env bash + + set -e -x + + # Compile wheels + for PYBIN in /opt/python/*/bin; do + if [[ "${PYBIN}" == *"cp27"* ]] || \ + [[ "${PYBIN}" == *"cp34"* ]] || \ + [[ "${PYBIN}" == *"cp35"* ]] || \ + [[ "${PYBIN}" == *"cp36"* ]] || \ + [[ "${PYBIN}" == *"cp37"* ]]; then + "${PYBIN}/pip" install -e /io/ + "${PYBIN}/pip" wheel /io/ -w wheelhouse/ + rm -rf /io/build /io/*.egg-info + fi + done + + # Bundle external shared libraries into the wheels + for whl in wheelhouse/zope.interface*.whl; do + auditwheel repair "$whl" -w /io/wheelhouse/ + done + """).lstrip('\n')) + result = update_manylinux_python_versions( + manylinux_install_sh, ['3.6', '3.7', '3.8']) + assert "".join(result) == textwrap.dedent(r""" + #!/usr/bin/env bash + + set -e -x + + # Compile wheels + for PYBIN in /opt/python/*/bin; do + if [[ "${PYBIN}" == *"cp36"* ]] || \ + [[ "${PYBIN}" == *"cp37"* ]] || \ + [[ "${PYBIN}" == *"cp38"* ]]; then + "${PYBIN}/pip" install -e /io/ + "${PYBIN}/pip" wheel /io/ -w wheelhouse/ + rm -rf /io/build /io/*.egg-info + fi + done + + # Bundle external shared libraries into the wheels + for whl in wheelhouse/zope.interface*.whl; do + auditwheel repair "$whl" -w /io/wheelhouse/ + done + """).lstrip('\n') From b3b7834a4712767192a087f97fe5a964dbe70e57 Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Wed, 17 Apr 2019 18:07:55 +0300 Subject: [PATCH 54/76] Handle comments in tox.ini --- src/check_python_versions/parsers/tox.py | 8 ++++++-- tests/parsers/test_tox.py | 20 ++++++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/src/check_python_versions/parsers/tox.py b/src/check_python_versions/parsers/tox.py index d8e4e30..a3a7032 100644 --- a/src/check_python_versions/parsers/tox.py +++ b/src/check_python_versions/parsers/tox.py @@ -121,9 +121,14 @@ def update_ini_setting(orig_lines, section, key, new_value, filename=TOX_INI): return orig_lines end = start + 1 + comments = [] + indent = ' ' for n, line in lines: if line.startswith(' '): + indent = get_indent(line) end = n + 1 + elif line.lstrip().startswith('#'): + comments.append(line) else: break @@ -131,8 +136,7 @@ def update_ini_setting(orig_lines, section, key, new_value, filename=TOX_INI): firstline = orig_lines[start].strip().expandtabs().replace(' ', '') if firstline == f'{key}=': if end > start + 1: - indent = get_indent(orig_lines[start + 1]) - prefix = f'\n{indent}' + prefix = f'\n{"".join(comments)}{indent}' new_lines = orig_lines[:start] + ( f"{key} ={prefix}{new_value}\n" diff --git a/tests/parsers/test_tox.py b/tests/parsers/test_tox.py index 05236a7..55d7ee2 100644 --- a/tests/parsers/test_tox.py +++ b/tests/parsers/test_tox.py @@ -148,3 +148,23 @@ def test_update_ini_setting_multiline(): py36,py37 usedevelop = true """) + + +def test_update_ini_setting_multiline_with_comments(): + source_lines = textwrap.dedent("""\ + [tox] + envlist = + # blah blah + # py26,py27,pypy + py26,py27 + usedevelop = true + """).splitlines(True) + result = update_ini_setting(source_lines, 'tox', 'envlist', 'py36,py37') + assert "".join(result) == textwrap.dedent("""\ + [tox] + envlist = + # blah blah + # py26,py27,pypy + py36,py37 + usedevelop = true + """) From 2b799aecf32935c0c25f0b44d9aed21ec7be1996 Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Wed, 17 Apr 2019 18:23:29 +0300 Subject: [PATCH 55/76] More robust appveyor.yml updates Support testing both 32-bit and 64-bit Pythons. --- src/check_python_versions/parsers/appveyor.py | 26 +++++++++++-- tests/parsers/test_appveyor.py | 38 +++++++++++++++++++ 2 files changed, 60 insertions(+), 4 deletions(-) diff --git a/src/check_python_versions/parsers/appveyor.py b/src/check_python_versions/parsers/appveyor.py index ecc3f52..9723645 100644 --- a/src/check_python_versions/parsers/appveyor.py +++ b/src/check_python_versions/parsers/appveyor.py @@ -70,17 +70,35 @@ def update_appveyor_yml_python_versions(filename, new_versions): conf = yaml.safe_load(fp) varname = 'PYTHON' - pattern = '{}{}' + patterns = set() for env in conf['environment']['matrix']: for var, value in env.items(): if var.lower() == 'python': varname = var - pattern = appveyor_detect_py_version_pattern(value) + patterns.add(appveyor_detect_py_version_pattern(value)) break + if not patterns: + patterns.add('{}{}') - new_environments = [ - f'{varname}: "{escape(pattern.format(*ver.split(".", 1)))}"' + quote = any(f'{varname}: "' in line for line in orig_lines) + + patterns = sorted(patterns) + + new_pythons = [ + pattern.format(*ver.split(".", 1)) for ver in new_versions + for pattern in patterns ] + + if quote: + new_environments = [ + f'{varname}: "{escape(python)}"' + for python in new_pythons + ] + else: + new_environments = [ + f'{varname}: {python}' + for python in new_pythons + ] new_lines = update_yaml_list(orig_lines, ' matrix', new_environments) return new_lines diff --git a/tests/parsers/test_appveyor.py b/tests/parsers/test_appveyor.py index ad7290a..254b024 100644 --- a/tests/parsers/test_appveyor.py +++ b/tests/parsers/test_appveyor.py @@ -1,4 +1,5 @@ import textwrap +from io import StringIO import pytest @@ -10,6 +11,7 @@ from check_python_versions.parsers.appveyor import ( appveyor_normalize_py_version, get_appveyor_yml_python_versions, + update_appveyor_yml_python_versions, ) @@ -56,3 +58,39 @@ def test_get_appveyor_yml_python_versions_using_toxenv(tmp_path): ]) def test_appveyor_normalize_py_version(s, expected): assert appveyor_normalize_py_version(s) == expected + + +def test_update_appveyor_yml_python_versions(): + appveyor_yml = StringIO(textwrap.dedent(r""" + environment: + matrix: + - PYTHON: "c:\\python27" + - PYTHON: "c:\\python36" + """).lstrip('\n')) + result = update_appveyor_yml_python_versions(appveyor_yml, ['2.7', '3.7']) + assert ''.join(result) == textwrap.dedent(r""" + environment: + matrix: + - PYTHON: "c:\\python27" + - PYTHON: "c:\\python37" + """.lstrip('\n')) + + +def test_update_appveyor_yml_python_versions_multiple_of_each(): + appveyor_yml = StringIO(textwrap.dedent("""\ + environment: + matrix: + - PYTHON: c:\\python27 + - PYTHON: c:\\python27-x64 + - PYTHON: c:\\python36 + - PYTHON: c:\\python36-x64 + """)) + result = update_appveyor_yml_python_versions(appveyor_yml, ['2.7', '3.7']) + assert ''.join(result) == textwrap.dedent("""\ + environment: + matrix: + - PYTHON: c:\\python27 + - PYTHON: c:\\python27-x64 + - PYTHON: c:\\python37 + - PYTHON: c:\\python37-x64 + """) From 9dabc6067c922ec541084500300cf393b05209df Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Wed, 17 Apr 2019 18:33:22 +0300 Subject: [PATCH 56/76] Upgrade pypy versions in .travis.yml --- src/check_python_versions/parsers/travis.py | 46 ++++++++++++++------- tests/parsers/test_travis.py | 25 +++++++++++ 2 files changed, 56 insertions(+), 15 deletions(-) diff --git a/src/check_python_versions/parsers/travis.py b/src/check_python_versions/parsers/travis.py index fda0078..67d4eb9 100644 --- a/src/check_python_versions/parsers/travis.py +++ b/src/check_python_versions/parsers/travis.py @@ -10,6 +10,11 @@ TRAVIS_YML = '.travis.yml' +XENIAL_SUPPORTED_PYPY_VERSIONS = { + 'pypy': 'pypy2.7-6.0.0', + 'pypy3': 'pypy3.5-6.0.0', +} + def get_travis_yml_python_versions(filename=TRAVIS_YML): with open_file(filename) as fp: @@ -57,12 +62,28 @@ def update_travis_yml_python_versions(filename, new_versions): orig_lines = fp.readlines() fp.seek(0) conf = yaml.safe_load(fp) + new_lines = orig_lines + + # Make sure we're using dist: xenial if we want to use Python 3.7 or newer. + replacements = {} + if any(map(needs_xenial, new_versions)): + replacements.update(XENIAL_SUPPORTED_PYPY_VERSIONS) + if conf.get('dist') != 'xenial': + new_lines = drop_yaml_node(new_lines, 'dist') + new_lines = add_yaml_node(new_lines, 'dist', 'xenial', + before='python') + if conf.get('sudo') is False: + # sudo is ignored nowadays, but in earlier times + # you needed both dist: xenial and sudo: required + # to get Python 3.7 + new_lines = drop_yaml_node(new_lines, "sudo") def keep_old(ver): return not is_important(travis_normalize_py_version(ver)) new_lines = update_yaml_list( - orig_lines, "python", new_versions, filename=fp.name, keep=keep_old, + new_lines, "python", new_versions, filename=fp.name, keep=keep_old, + replacements=replacements, ) # If python 3.7 was enabled via matrix.include, we've just added a @@ -80,23 +101,12 @@ def keep_old(ver): # XXX: this may drop too much or too little! new_lines = drop_yaml_node(new_lines, "matrix") - # Make sure we're using dist: xenial if we want to use Python 3.7 or newer. - if any(map(needs_xenial, new_versions)): - if conf.get('dist') != 'xenial': - new_lines = drop_yaml_node(new_lines, 'dist') - new_lines = add_yaml_node(new_lines, 'dist', 'xenial', - before='python') - if conf.get('sudo') is False: - # sudo is ignored nowadays, but in earlier times - # you needed both dist: xenial and sudo: required - # to get Python 3.7 - new_lines = drop_yaml_node(new_lines, "sudo") - return new_lines def update_yaml_list( orig_lines, key, new_value, filename=TRAVIS_YML, keep=None, + replacements=None, ): lines = iter(enumerate(orig_lines)) for n, line in lines: @@ -117,8 +127,14 @@ def update_yaml_list( lines_to_keep = keep_after indent = len(line) - len(stripped) end = n + 1 - if keep and keep(stripped[2:].strip()): - lines_to_keep.append(line) + value = stripped[2:].strip() + if keep and keep(value): + if replacements and value in replacements: + lines_to_keep.append( + f"{' '* indent}- {replacements[value]}\n" + ) + else: + lines_to_keep.append(line) elif stripped.startswith('#'): lines_to_keep.append(line) end = n + 1 diff --git a/tests/parsers/test_travis.py b/tests/parsers/test_travis.py index fcd796b..bf9d4ec 100644 --- a/tests/parsers/test_travis.py +++ b/tests/parsers/test_travis.py @@ -1,4 +1,5 @@ import textwrap +from io import StringIO import pytest @@ -12,6 +13,7 @@ drop_yaml_node, get_travis_yml_python_versions, travis_normalize_py_version, + update_travis_yml_python_versions, update_yaml_list, ) @@ -73,6 +75,29 @@ def test_travis_normalize_py_version(s, expected): assert travis_normalize_py_version(s) == expected +def test_update_travis_yml_python_versions(): + travis_yml = StringIO(textwrap.dedent("""\ + language: python + python: + - 2.7 + - pypy + install: pip install -e . + script: pytest tests + """)) + travis_yml.name = '.travis.yml' + result = update_travis_yml_python_versions(travis_yml, ["2.7", "3.7"]) + assert "".join(result) == textwrap.dedent("""\ + language: python + dist: xenial + python: + - 2.7 + - 3.7 + - pypy2.7-6.0.0 + install: pip install -e . + script: pytest tests + """) + + def test_update_yaml_list(): source_lines = textwrap.dedent("""\ language: python From 304e52711718894b5b4d19defe30e65dda9eac1a Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Wed, 17 Apr 2019 18:51:25 +0300 Subject: [PATCH 57/76] Handle duplicate keys in YAML files --- src/check_python_versions/parsers/travis.py | 26 +++-- tests/parsers/test_travis.py | 109 ++++++++++++++++++++ 2 files changed, 127 insertions(+), 8 deletions(-) diff --git a/src/check_python_versions/parsers/travis.py b/src/check_python_versions/parsers/travis.py index 67d4eb9..74ebdaa 100644 --- a/src/check_python_versions/parsers/travis.py +++ b/src/check_python_versions/parsers/travis.py @@ -69,14 +69,14 @@ def update_travis_yml_python_versions(filename, new_versions): if any(map(needs_xenial, new_versions)): replacements.update(XENIAL_SUPPORTED_PYPY_VERSIONS) if conf.get('dist') != 'xenial': - new_lines = drop_yaml_node(new_lines, 'dist') + new_lines = drop_yaml_node(new_lines, 'dist', filename=fp.name) new_lines = add_yaml_node(new_lines, 'dist', 'xenial', before='python') if conf.get('sudo') is False: # sudo is ignored nowadays, but in earlier times # you needed both dist: xenial and sudo: required # to get Python 3.7 - new_lines = drop_yaml_node(new_lines, "sudo") + new_lines = drop_yaml_node(new_lines, "sudo", filename=fp.name) def keep_old(ver): return not is_important(travis_normalize_py_version(ver)) @@ -95,11 +95,12 @@ def keep_old(ver): and 'include' in conf['matrix'] and all( job.get('dist') == 'xenial' + and set(job) <= {'python', 'dist', 'sudo'} for job in conf['matrix']['include'] ) ): # XXX: this may drop too much or too little! - new_lines = drop_yaml_node(new_lines, "matrix") + new_lines = drop_yaml_node(new_lines, "matrix", filename=fp.name) return new_lines @@ -149,22 +150,31 @@ def update_yaml_list( return new_lines -def drop_yaml_node(orig_lines, key): +def drop_yaml_node(orig_lines, key, filename=TRAVIS_YML): lines = iter(enumerate(orig_lines)) + where = None for n, line in lines: if line.startswith(f'{key}:'): - break - else: + if where is not None: + warn( + f"Duplicate {key}: setting in {filename}" + f" (lines {where + 1} and {n + 1})" + ) + where = n + if where is None: return orig_lines - start = n - end = n + 1 + lines = iter(enumerate(orig_lines[where + 1:], where + 1)) + + start = where + end = start + 1 for n, line in lines: if line and line[0] != ' ': break else: end = n + 1 new_lines = orig_lines[:start] + orig_lines[end:] + return new_lines diff --git a/tests/parsers/test_travis.py b/tests/parsers/test_travis.py index bf9d4ec..8a55d20 100644 --- a/tests/parsers/test_travis.py +++ b/tests/parsers/test_travis.py @@ -98,6 +98,84 @@ def test_update_travis_yml_python_versions(): """) +def test_update_travis_yml_python_versions_drops_sudo(): + travis_yml = StringIO(textwrap.dedent("""\ + language: python + sudo: false + python: + - 2.7 + install: pip install -e . + script: pytest tests + """)) + travis_yml.name = '.travis.yml' + result = update_travis_yml_python_versions(travis_yml, ["2.7", "3.7"]) + assert "".join(result) == textwrap.dedent("""\ + language: python + dist: xenial + python: + - 2.7 + - 3.7 + install: pip install -e . + script: pytest tests + """) + + +def test_update_travis_yml_python_versions_drops_matrix(): + travis_yml = StringIO(textwrap.dedent("""\ + language: python + python: + - 2.6 + - 2.7 + matrix: + include: + - python: 3.7 + sudo: required + dist: xenial + install: pip install -e . + script: pytest tests + """)) + travis_yml.name = '.travis.yml' + result = update_travis_yml_python_versions(travis_yml, ["2.7", "3.7"]) + assert "".join(result) == textwrap.dedent("""\ + language: python + dist: xenial + python: + - 2.7 + - 3.7 + install: pip install -e . + script: pytest tests + """) + + +def test_update_travis_yml_python_versions_keeps_matrix(): + travis_yml = StringIO(textwrap.dedent("""\ + language: python + python: + - 2.7 + matrix: + include: + - python: 2.7 + env: MINIMAL=1 + install: pip install -e . + script: pytest tests + """)) + travis_yml.name = '.travis.yml' + result = update_travis_yml_python_versions(travis_yml, ["2.7", "3.7"]) + assert "".join(result) == textwrap.dedent("""\ + language: python + dist: xenial + python: + - 2.7 + - 3.7 + matrix: + include: + - python: 2.7 + env: MINIMAL=1 + install: pip install -e . + script: pytest tests + """) + + def test_update_yaml_list(): source_lines = textwrap.dedent("""\ language: python @@ -216,6 +294,37 @@ def test_drop_yaml_node_when_last_in_file(): """) +def test_drop_yaml_node_when_duplicate(capsys): + source_lines = textwrap.dedent("""\ + language: python + sudo: false + matrix: + include: + - python: 2.7 + python: + - 3.6 + matrix: + include: + - python: 3.7 + script: pytest tests + """).splitlines(True) + result = drop_yaml_node(source_lines, 'matrix') + assert "".join(result) == textwrap.dedent("""\ + language: python + sudo: false + matrix: + include: + - python: 2.7 + python: + - 3.6 + script: pytest tests + """) + assert ( + "Duplicate matrix: setting in .travis.yml (lines 3 and 8)" + in capsys.readouterr().err + ) + + def test_add_yaml_node(): source_lines = textwrap.dedent("""\ language: python From 18e78e00c0d4e4cbce98a303ed8e2eb9a1f9c38a Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Wed, 17 Apr 2019 19:01:49 +0300 Subject: [PATCH 58/76] Handle more complicated appveyor environments as long as they fit on one line. --- src/check_python_versions/parsers/appveyor.py | 16 +++++++++++++++- tests/parsers/test_appveyor.py | 18 ++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/src/check_python_versions/parsers/appveyor.py b/src/check_python_versions/parsers/appveyor.py index 9723645..80d8cf0 100644 --- a/src/check_python_versions/parsers/appveyor.py +++ b/src/check_python_versions/parsers/appveyor.py @@ -1,3 +1,5 @@ +from io import StringIO + try: import yaml except ImportError: # pragma: nocover @@ -100,5 +102,17 @@ def update_appveyor_yml_python_versions(filename, new_versions): f'{varname}: {python}' for python in new_pythons ] - new_lines = update_yaml_list(orig_lines, ' matrix', new_environments) + + def keep_complicated(value): + if value.startswith('{') and value.endswith('}'): + env = yaml.safe_load(StringIO(value)) + for var, value in env.items(): + if var.lower() == 'python': + ver = appveyor_normalize_py_version(value) + if ver in new_versions: + return True + return False + + new_lines = update_yaml_list(orig_lines, ' matrix', new_environments, + keep=keep_complicated) return new_lines diff --git a/tests/parsers/test_appveyor.py b/tests/parsers/test_appveyor.py index 254b024..6baea85 100644 --- a/tests/parsers/test_appveyor.py +++ b/tests/parsers/test_appveyor.py @@ -94,3 +94,21 @@ def test_update_appveyor_yml_python_versions_multiple_of_each(): - PYTHON: c:\\python37 - PYTHON: c:\\python37-x64 """) + + +def test_update_appveyor_yml_python_complicated_but_oneline(): + appveyor_yml = StringIO(textwrap.dedent("""\ + environment: + matrix: + - PYTHON: c:\\python27 + - PYTHON: c:\\python36 + - { PYTHON: c:\\python27, EXTRA_FEATURE: 1 } + - { PYTHON: c:\\python36, EXTRA_FEATURE: 1 } + """)) + result = update_appveyor_yml_python_versions(appveyor_yml, ['3.6']) + assert ''.join(result) == textwrap.dedent("""\ + environment: + matrix: + - PYTHON: c:\\python36 + - { PYTHON: c:\\python36, EXTRA_FEATURE: 1 } + """) From e17525f4652aa0a1e06776477d46403d9c4c4eef Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Wed, 17 Apr 2019 19:07:42 +0300 Subject: [PATCH 59/76] Test coverage and a bugfix for the Travis parser --- src/check_python_versions/parsers/travis.py | 5 +- tests/parsers/test_travis.py | 69 +++++++++++++++++++++ 2 files changed, 73 insertions(+), 1 deletion(-) diff --git a/src/check_python_versions/parsers/travis.py b/src/check_python_versions/parsers/travis.py index 74ebdaa..69d15c3 100644 --- a/src/check_python_versions/parsers/travis.py +++ b/src/check_python_versions/parsers/travis.py @@ -21,7 +21,10 @@ def get_travis_yml_python_versions(filename=TRAVIS_YML): conf = yaml.safe_load(fp) versions = [] if conf.get('python'): - versions += map(travis_normalize_py_version, conf['python']) + if isinstance(conf['python'], list): + versions += map(travis_normalize_py_version, conf['python']) + else: + versions.append(travis_normalize_py_version(conf['python'])) if 'matrix' in conf and 'include' in conf['matrix']: for job in conf['matrix']['include']: if 'python' in job: diff --git a/tests/parsers/test_travis.py b/tests/parsers/test_travis.py index 8a55d20..8dc2693 100644 --- a/tests/parsers/test_travis.py +++ b/tests/parsers/test_travis.py @@ -45,6 +45,17 @@ def test_get_travis_yml_python_versions(tmp_path): ] +@needs_pyyaml +def test_get_travis_yml_python_versions_no_list(tmp_path): + travis_yml = StringIO(textwrap.dedent("""\ + python: 3.7 + """)) + travis_yml.name = '.travis.yml' + assert get_travis_yml_python_versions(travis_yml) == [ + '3.7', + ] + + @needs_pyyaml def test_get_travis_yml_python_versions_no_python_only_matrix(tmp_path): travis_yml = tmp_path / ".travis.yml" @@ -76,6 +87,28 @@ def test_travis_normalize_py_version(s, expected): def test_update_travis_yml_python_versions(): + travis_yml = StringIO(textwrap.dedent("""\ + language: python + python: + - 2.7 + - pypy + install: pip install -e . + script: pytest tests + """)) + travis_yml.name = '.travis.yml' + result = update_travis_yml_python_versions(travis_yml, ["2.7", "3.4"]) + assert "".join(result) == textwrap.dedent("""\ + language: python + python: + - 2.7 + - 3.4 + - pypy + install: pip install -e . + script: pytest tests + """) + + +def test_update_travis_yml_python_versions_adds_dist_xenial(): travis_yml = StringIO(textwrap.dedent("""\ language: python python: @@ -102,6 +135,7 @@ def test_update_travis_yml_python_versions_drops_sudo(): travis_yml = StringIO(textwrap.dedent("""\ language: python sudo: false + dist: xenial python: - 2.7 install: pip install -e . @@ -220,6 +254,24 @@ def test_update_yaml_list_keep_indent_comments_and_pypy(): """) +def test_update_yaml_not_found(capsys): + source_lines = textwrap.dedent("""\ + language: python + install: pip install -e . + script: pytest tests + """).splitlines(True) + result = update_yaml_list(source_lines, "python", ["2.7", "3.7"]) + assert "".join(result) == textwrap.dedent("""\ + language: python + install: pip install -e . + script: pytest tests + """) + assert ( + "Did not find python: setting in .travis.yml" + in capsys.readouterr().err + ) + + def test_drop_yaml_node(): source_lines = textwrap.dedent("""\ language: python @@ -325,6 +377,23 @@ def test_drop_yaml_node_when_duplicate(capsys): ) +def test_add_yaml_node_before(): + source_lines = textwrap.dedent("""\ + language: python + python: + - 3.6 + script: pytest tests + """).splitlines(True) + result = add_yaml_node(source_lines, 'dist', 'xenial') + assert "".join(result) == textwrap.dedent("""\ + language: python + python: + - 3.6 + script: pytest tests + dist: xenial + """) + + def test_add_yaml_node(): source_lines = textwrap.dedent("""\ language: python From 7d034a1e0d4250927bb2e17d3c13a5cbb8ec7218 Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Wed, 17 Apr 2019 19:11:58 +0300 Subject: [PATCH 60/76] Handle non-list python: when updating .travis.yml --- src/check_python_versions/parsers/travis.py | 10 ++++++---- tests/parsers/test_travis.py | 19 +++++++++++++++++++ 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/src/check_python_versions/parsers/travis.py b/src/check_python_versions/parsers/travis.py index 69d15c3..6bdb031 100644 --- a/src/check_python_versions/parsers/travis.py +++ b/src/check_python_versions/parsers/travis.py @@ -114,13 +114,14 @@ def update_yaml_list( ): lines = iter(enumerate(orig_lines)) for n, line in lines: - if line == f'{key}:\n': + if line.startswith(f'{key}:'): break else: warn(f'Did not find {key}: setting in {filename}') return orig_lines - start = end = n + 1 + start = n + end = n + 1 indent = 2 keep_before = [] keep_after = [] @@ -144,9 +145,10 @@ def update_yaml_list( end = n + 1 if line and line[0] != ' ': break - # TODO: else? - new_lines = orig_lines[:start] + keep_before + [ + new_lines = orig_lines[:start] + [ + f"{key}:\n" + ] + keep_before + [ f"{' ' * indent}- {value}\n" for value in new_value ] + keep_after + orig_lines[end:] diff --git a/tests/parsers/test_travis.py b/tests/parsers/test_travis.py index 8dc2693..8f615b8 100644 --- a/tests/parsers/test_travis.py +++ b/tests/parsers/test_travis.py @@ -210,6 +210,25 @@ def test_update_travis_yml_python_versions_keeps_matrix(): """) +def test_update_travis_yml_python_versions_one_to_many(): + travis_yml = StringIO(textwrap.dedent("""\ + language: python + python: 2.7 + install: pip install -e . + script: pytest tests + """)) + travis_yml.name = '.travis.yml' + result = update_travis_yml_python_versions(travis_yml, ["2.7", "3.4"]) + assert "".join(result) == textwrap.dedent("""\ + language: python + python: + - 2.7 + - 3.4 + install: pip install -e . + script: pytest tests + """) + + def test_update_yaml_list(): source_lines = textwrap.dedent("""\ language: python From 28d92b06d83ff7cd1572e937409851695e34ea2f Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Wed, 17 Apr 2019 19:15:54 +0300 Subject: [PATCH 61/76] Fix manylinux updater error handling --- .../parsers/manylinux.py | 6 ++- tests/parsers/test_manylinux.py | 52 +++++++++++++++++++ 2 files changed, 56 insertions(+), 2 deletions(-) diff --git a/src/check_python_versions/parsers/manylinux.py b/src/check_python_versions/parsers/manylinux.py index b275cee..1d737e8 100644 --- a/src/check_python_versions/parsers/manylinux.py +++ b/src/check_python_versions/parsers/manylinux.py @@ -27,14 +27,16 @@ def update_manylinux_python_versions(filename, new_versions): start = n break else: - warn(f'Failed to understand {filename}') + warn(f'Failed to understand {f.name}') + return orig_lines for n, line in lines: m = magic.match(line) if not m: end = n break else: - warn(f'Failed to understand {filename}') + warn(f'Failed to understand {f.name}') + return orig_lines indent = ' ' * 4 conditions = f' || \\\n{indent} '.join( diff --git a/tests/parsers/test_manylinux.py b/tests/parsers/test_manylinux.py index 598949e..64086f7 100644 --- a/tests/parsers/test_manylinux.py +++ b/tests/parsers/test_manylinux.py @@ -84,3 +84,55 @@ def test_update_manylinux_python_versions(): auditwheel repair "$whl" -w /io/wheelhouse/ done """).lstrip('\n') + + +def test_update_manylinux_python_versions_failure(capsys): + manylinux_install_sh = StringIO(textwrap.dedent(r""" + #!/usr/bin/env bash + + # TBD + """).lstrip('\n')) + manylinux_install_sh.name = '.manylinux-install.sh' + result = update_manylinux_python_versions( + manylinux_install_sh, ['3.6', '3.7', '3.8']) + assert "".join(result) == textwrap.dedent(r""" + #!/usr/bin/env bash + + # TBD + """).lstrip('\n') + assert ( + "Failed to understand .manylinux-install.sh" + in capsys.readouterr().err + ) + + +def test_update_manylinux_python_versions_truncated(capsys): + manylinux_install_sh = StringIO(textwrap.dedent(r""" + #!/usr/bin/env bash + + set -e -x + + # Compile wheels + for PYBIN in /opt/python/*/bin; do + if [[ "${PYBIN}" == *"cp27"* ]] || \ + [[ "${PYBIN}" == *"cp34"* ]] || \ + [[ "${PYBIN}" == *"cp35"* ]] || \ + """).lstrip('\n')) + manylinux_install_sh.name = '.manylinux-install.sh' + result = update_manylinux_python_versions( + manylinux_install_sh, ['3.6', '3.7', '3.8']) + assert "".join(result) == textwrap.dedent(r""" + #!/usr/bin/env bash + + set -e -x + + # Compile wheels + for PYBIN in /opt/python/*/bin; do + if [[ "${PYBIN}" == *"cp27"* ]] || \ + [[ "${PYBIN}" == *"cp34"* ]] || \ + [[ "${PYBIN}" == *"cp35"* ]] || \ + """).lstrip('\n') + assert ( + "Failed to understand .manylinux-install.sh" + in capsys.readouterr().err + ) From cd59ec77f12eadec19c4174bdc2b714fa8c09983 Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Wed, 17 Apr 2019 19:23:04 +0300 Subject: [PATCH 62/76] Better appveyor error handling --- src/check_python_versions/parsers/appveyor.py | 6 ++-- tests/parsers/test_appveyor.py | 33 +++++++++++++++++++ 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/src/check_python_versions/parsers/appveyor.py b/src/check_python_versions/parsers/appveyor.py index 80d8cf0..c6de41f 100644 --- a/src/check_python_versions/parsers/appveyor.py +++ b/src/check_python_versions/parsers/appveyor.py @@ -7,7 +7,7 @@ from .tox import parse_envlist, tox_env_to_py_version from .travis import update_yaml_list -from ..utils import open_file +from ..utils import open_file, warn APPVEYOR_YML = 'appveyor.yml' @@ -79,8 +79,10 @@ def update_appveyor_yml_python_versions(filename, new_versions): varname = var patterns.add(appveyor_detect_py_version_pattern(value)) break + if not patterns: - patterns.add('{}{}') + warn(f"Did not recognize any PYTHON environments in {fp.name}") + return orig_lines quote = any(f'{varname}: "' in line for line in orig_lines) diff --git a/tests/parsers/test_appveyor.py b/tests/parsers/test_appveyor.py index 6baea85..37ee076 100644 --- a/tests/parsers/test_appveyor.py +++ b/tests/parsers/test_appveyor.py @@ -9,6 +9,7 @@ yaml = None from check_python_versions.parsers.appveyor import ( + appveyor_detect_py_version_pattern, appveyor_normalize_py_version, get_appveyor_yml_python_versions, update_appveyor_yml_python_versions, @@ -60,6 +61,17 @@ def test_appveyor_normalize_py_version(s, expected): assert appveyor_normalize_py_version(s) == expected +@pytest.mark.parametrize('s, expected', [ + ('37', '{}{}'), + ('c:\\python34', 'c:\\python{}{}'), + ('C:\\Python27\\', 'C:\\Python{}{}\\'), + ('C:\\Python27-x64', 'C:\\Python{}{}-x64'), + ('C:\\PYTHON34-X64', 'C:\\PYTHON{}{}-X64'), +]) +def test_appveyor_detect_py_version_pattern(s, expected): + assert appveyor_detect_py_version_pattern(s) == expected + + def test_update_appveyor_yml_python_versions(): appveyor_yml = StringIO(textwrap.dedent(r""" environment: @@ -112,3 +124,24 @@ def test_update_appveyor_yml_python_complicated_but_oneline(): - PYTHON: c:\\python36 - { PYTHON: c:\\python36, EXTRA_FEATURE: 1 } """) + + +def test_update_appveyor_yml_python_no_understanding(capsys): + appveyor_yml = StringIO(textwrap.dedent("""\ + environment: + matrix: + - FOO: 1 + - BAR: 2 + """)) + appveyor_yml.name = 'appveyor.yml' + result = update_appveyor_yml_python_versions(appveyor_yml, ['3.6']) + assert ''.join(result) == textwrap.dedent("""\ + environment: + matrix: + - FOO: 1 + - BAR: 2 + """) + assert ( + "Did not recognize any PYTHON environments in appveyor.yml" + in capsys.readouterr().err + ) From 02f7c1348aa72e08139f939ece9b3625337feaac Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Wed, 17 Apr 2019 19:27:32 +0300 Subject: [PATCH 63/76] Better tox error handling and tests --- src/check_python_versions/parsers/tox.py | 5 ++- tests/parsers/test_tox.py | 51 +++++++++++++++++++++++- 2 files changed, 52 insertions(+), 4 deletions(-) diff --git a/src/check_python_versions/parsers/tox.py b/src/check_python_versions/parsers/tox.py index a3a7032..6cb959b 100644 --- a/src/check_python_versions/parsers/tox.py +++ b/src/check_python_versions/parsers/tox.py @@ -64,7 +64,8 @@ def update_tox_ini_python_versions(filename, new_versions): try: conf.read_file(fp) envlist = conf.get('tox', 'envlist') - except configparser.Error: + except configparser.Error as error: + warn(f"Could not parse {fp.name}: {error}") return orig_lines new_envlist = update_tox_envlist(envlist, new_versions) @@ -108,7 +109,7 @@ def update_ini_setting(orig_lines, section, key, new_value, filename=TOX_INI): if line.startswith(f'[{section}]'): break else: - warn(f'Did not find [{section}] in {filename}') + warn(f'Did not find [{section}] section in {filename}') return orig_lines # TODO: use a regex to allow an arbitrary number of spaces around = diff --git a/tests/parsers/test_tox.py b/tests/parsers/test_tox.py index 55d7ee2..c4c336f 100644 --- a/tests/parsers/test_tox.py +++ b/tests/parsers/test_tox.py @@ -94,9 +94,26 @@ def test_update_tox_ini_python_versions(): """) +def test_update_tox_ini_python_syntax_error(capsys): + fp = StringIO(textwrap.dedent("""\ + [tox + envlist = py26, py27 + """)) + fp.name = 'tox.ini' + result = update_tox_ini_python_versions(fp, ['3.6', '3.7']) + assert "".join(result) == textwrap.dedent("""\ + [tox + envlist = py26, py27 + """) + assert ( + "Could not parse tox.ini:" + in capsys.readouterr().err + ) + + def test_update_tox_envlist(): - result = update_tox_envlist('py26,py27,pypy', ['3.6', '3.7']) - assert result == 'py36,py37,pypy' + result = update_tox_envlist('py26,py27,pypy,flake8', ['3.6', '3.7']) + assert result == 'py36,py37,pypy,flake8' def test_update_tox_envlist_with_suffixes(): @@ -168,3 +185,33 @@ def test_update_ini_setting_multiline_with_comments(): py36,py37 usedevelop = true """) + + +def test_update_ini_setting_no_section(capsys): + source_lines = textwrap.dedent("""\ + [toxx] + """).splitlines(True) + result = update_ini_setting(source_lines, 'tox', 'envlist', 'py36,py37') + assert "".join(result) == textwrap.dedent("""\ + [toxx] + """) + assert ( + "Did not find [tox] section in tox.ini" + in capsys.readouterr().err + ) + + +def test_update_ini_setting_no_key(capsys): + source_lines = textwrap.dedent("""\ + [tox] + usedevelop = true + """).splitlines(True) + result = update_ini_setting(source_lines, 'tox', 'envlist', 'py36,py37') + assert "".join(result) == textwrap.dedent("""\ + [tox] + usedevelop = true + """) + assert ( + "Did not find envlist= in [tox] in tox.ini" + in capsys.readouterr().err + ) From c2b1f5b1ad5f3ff1015f98c69b8b6f2658002eea Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Wed, 17 Apr 2019 19:29:49 +0300 Subject: [PATCH 64/76] Exclude Windows-only code paths from coverage --- .coveragerc | 1 + src/check_python_versions/utils.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.coveragerc b/.coveragerc index b4028c6..9c55f15 100644 --- a/.coveragerc +++ b/.coveragerc @@ -6,4 +6,5 @@ branch = True [report] exclude_lines = pragma: nocover + pragma: windows if __name__ == .__main__.: diff --git a/src/check_python_versions/utils.py b/src/check_python_versions/utils.py index a657f09..6649d68 100644 --- a/src/check_python_versions/utils.py +++ b/src/check_python_versions/utils.py @@ -48,13 +48,13 @@ def confirm_and_update_file(filename, new_lines): with open(tempfile, 'w') as f: if hasattr(os, 'fchmod'): os.fchmod(f.fileno(), mode) - else: + else: # pragma: windows # Windows, what else? os.chmod(tempfile, mode) f.writelines(new_lines) try: os.rename(tempfile, filename) - except FileExistsError: + except FileExistsError: # pragma: windows # No atomic replace on Windows os.unlink(filename) os.rename(tempfile, filename) From ee3a334c3a80fda1d317fb7f9e5165e39dafbb48 Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Wed, 17 Apr 2019 19:49:39 +0300 Subject: [PATCH 65/76] Tests and smarter python_requires for only one version --- src/check_python_versions/parsers/python.py | 2 + tests/parsers/test_python.py | 61 ++++++++++++++++----- 2 files changed, 50 insertions(+), 13 deletions(-) diff --git a/src/check_python_versions/parsers/python.py b/src/check_python_versions/parsers/python.py index 4c0e9a2..503e1f6 100644 --- a/src/check_python_versions/parsers/python.py +++ b/src/check_python_versions/parsers/python.py @@ -405,6 +405,8 @@ def arbitrary_version(constraint): def compute_python_requires(new_versions): new_versions = set(new_versions) + if len(new_versions) == 1: + return f'=={new_versions.pop()}.*' # XXX assumes all versions are X.Y and 3.10 will never be released min_version = min(new_versions) specifiers = [f'>={min_version}'] diff --git a/tests/parsers/test_python.py b/tests/parsers/test_python.py index 660e28c..39bf530 100644 --- a/tests/parsers/test_python.py +++ b/tests/parsers/test_python.py @@ -1,9 +1,11 @@ import ast import textwrap +from io import StringIO import pytest from check_python_versions.parsers.python import ( + compute_python_requires, eval_ast_node, find_call_kwarg_in_ast, get_python_requires, @@ -48,6 +50,21 @@ def test_get_supported_python_versions_computed(tmp_path): assert get_supported_python_versions(filename) == ['2.7', '3.7'] +def test_get_supported_python_versions_from_file_object_cannot_run_setup_py(): + fp = StringIO(textwrap.dedent("""\ + from setuptools import setup + setup( + name='foo', + classifiers=[ + 'Programming Language :: Python :: %s' % v + for v in ['2.7', '3.7'] + ], + ) + """)) + fp.name = 'setup.py' + assert get_supported_python_versions(fp) == [] + + def test_get_versions_from_classifiers(): assert get_versions_from_classifiers([ 'Development Status :: 4 - Beta', @@ -277,6 +294,10 @@ def test_find_call_kwarg_in_ast_no_call(capsys): ('["a", "b"]', ["a", "b"]), ('("a", "b")', ("a", "b")), ('"-".join(["a", "b"])', "a-b"), + ('["a", "b"] + ["c"]', ["a", "b", "c"]), + ('["a", "b"] + extra', ["a", "b"]), + ('extra + ["a", "b"]', ["a", "b"]), + ('["a", "b", extra]', ["a", "b"]), ]) def test_eval_ast_node(code, expected): tree = ast.parse(f'foo(bar={code})') @@ -285,6 +306,18 @@ def test_eval_ast_node(code, expected): assert eval_ast_node(node, 'bar') == expected +@pytest.mark.parametrize('code', [ + '[2 * 2]', + '"".join([2 * 2])', + 'extra + more', +]) +def test_eval_ast_node_failures(code, capsys): + tree = ast.parse(f'foo(bar={code})') + node = find_call_kwarg_in_ast(tree, 'foo', 'bar') + assert eval_ast_node(node, 'bar') is None + assert 'Non-literal bar= passed to setup()' in capsys.readouterr().err + + def test_to_literal(): assert to_literal("blah") == '"blah"' assert to_literal("blah", "'") == "'blah'" @@ -331,7 +364,7 @@ def test_update_call_arg_in_source_string_spaces(): source_lines = textwrap.dedent("""\ setup ( foo = 1, - bar = "x", + bar = 'x', baz = 2, ) """).splitlines(True) @@ -339,7 +372,7 @@ def test_update_call_arg_in_source_string_spaces(): assert "".join(result) == textwrap.dedent("""\ setup ( foo = 1, - bar = "y", + bar = 'y', baz = 2, ) """) @@ -495,17 +528,6 @@ def test_update_call_arg_in_source_too_complicated(capsys): ) -@pytest.mark.parametrize('code', [ - '[2 * 2]', - '"".join([2 * 2])', -]) -def test_eval_ast_node_failures(code, capsys): - tree = ast.parse(f'foo(bar={code})') - node = find_call_kwarg_in_ast(tree, 'foo', 'bar') - assert eval_ast_node(node, 'bar') is None - assert 'Non-literal bar= passed to setup()' in capsys.readouterr().err - - @pytest.mark.parametrize('constraint, result', [ ('~= 2.7', ['2.7']), ('~= 2.7.12', ['2.7']), @@ -608,3 +630,16 @@ def test_parse_python_requires_syntax_errors(capsys, specifier): f'Bad python_requires specifier: {specifier}' in capsys.readouterr().err ) + + +@pytest.mark.parametrize('versions, expected', [ + (['2.7'], '==2.7.*'), + (['3.6', '3.7'], '>=3.6'), + (['2.7', '3.4', '3.5', '3.6', '3.7'], + '>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*'), +]) +def test_compute_python_requires(versions, expected, fix_max_python_3_version): + fix_max_python_3_version(7) + result = compute_python_requires(versions) + assert result == expected + assert parse_python_requires(result) == versions From 303da0b4d049ebe9f0bab91aaf243b34ca42fa41 Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Wed, 17 Apr 2019 19:53:49 +0300 Subject: [PATCH 66/76] Tests for the python_requires updater --- tests/parsers/test_python.py | 54 ++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/tests/parsers/test_python.py b/tests/parsers/test_python.py index 39bf530..07e81c0 100644 --- a/tests/parsers/test_python.py +++ b/tests/parsers/test_python.py @@ -16,6 +16,7 @@ to_literal, update_call_arg_in_source, update_classifiers, + update_python_requires, update_supported_python_versions, ) @@ -264,6 +265,59 @@ def test_get_setup_py_keyword_syntax_error(tmp_path, capsys): assert 'Could not parse' in capsys.readouterr().err +def test_update_python_requires(tmp_path, fix_max_python_3_version): + fix_max_python_3_version(7) + filename = tmp_path / "setup.py" + filename.write_text(textwrap.dedent("""\ + from setuptools import setup + setup( + name='foo', + python_requires='>= 3.4', + ) + """)) + result = update_python_requires(filename, ['3.5', '3.6', '3.7']) + assert "".join(result) == textwrap.dedent("""\ + from setuptools import setup + setup( + name='foo', + python_requires='>=3.5', + ) + """) + + +def test_update_python_requires_file_object(fix_max_python_3_version): + fix_max_python_3_version(7) + fp = StringIO(textwrap.dedent("""\ + from setuptools import setup + setup( + name='foo', + python_requires='>= 3.4', + ) + """)) + fp.name = "setup.py" + result = update_python_requires(fp, ['3.5', '3.6', '3.7']) + assert "".join(result) == textwrap.dedent("""\ + from setuptools import setup + setup( + name='foo', + python_requires='>=3.5', + ) + """) + + +def test_update_python_requires_when_missing(capsys): + fp = StringIO(textwrap.dedent("""\ + from setuptools import setup + setup( + name='foo', + ) + """)) + fp.name = "setup.py" + result = update_python_requires(fp, ['3.5', '3.6', '3.7']) + assert result is None + assert capsys.readouterr().err == "" + + def test_find_call_kwarg_in_ast(): tree = ast.parse('foo(bar="foo")') ast.dump(tree) From 8942ddeb64a5cf039635830deaa8a9ca7fc20339 Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Wed, 17 Apr 2019 20:17:19 +0300 Subject: [PATCH 67/76] Back to 100% test coverage --- tests/test_cli.py | 343 ++++++++++++++++++++++++++++++++++++++++++++++ tox.ini | 3 +- 2 files changed, 344 insertions(+), 2 deletions(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index b8179ed..6dcc14c 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -157,6 +157,172 @@ def test_check_expectation(tmp_path, capsys): """) +def test_check_only(tmp_path, capsys): + setup_py = tmp_path / "setup.py" + setup_py.write_text(textwrap.dedent("""\ + from setuptools import setup + setup( + name='foo', + classifiers=[ + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.6', + ], + ) + """)) + tox_ini = tmp_path / "tox.ini" + tox_ini.write_text(textwrap.dedent("""\ + [tox] + envlist = py27 + """)) + assert cpv.check_versions(tmp_path, only='tox.ini') + assert capsys.readouterr().out == textwrap.dedent("""\ + tox.ini says: 2.7 + """) + + +def test_update_versions(tmp_path, monkeypatch): + monkeypatch.setattr(sys, 'stdin', StringIO('y\n')) + (tmp_path / "setup.py").write_text(textwrap.dedent("""\ + from setuptools import setup + setup( + name='foo', + classifiers=[ + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.6', + ], + ) + """)) + cpv.update_versions(tmp_path, add=['3.7']) + assert (tmp_path / "setup.py").read_text() == textwrap.dedent("""\ + from setuptools import setup + setup( + name='foo', + classifiers=[ + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + ], + ) + """) + + +def test_update_versions_dry_run(tmp_path): + (tmp_path / "setup.py").write_text(textwrap.dedent("""\ + from setuptools import setup + setup( + name='foo', + classifiers=[ + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.6', + ], + ) + """)) + replacements = cpv.update_versions(tmp_path, add=['3.7'], dry_run=True) + assert (tmp_path / "setup.py").read_text() == textwrap.dedent("""\ + from setuptools import setup + setup( + name='foo', + classifiers=[ + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.6', + ], + ) + """) + filename = str(tmp_path / "setup.py") + assert "".join(replacements[filename]) == textwrap.dedent("""\ + from setuptools import setup + setup( + name='foo', + classifiers=[ + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + ], + ) + """) + + +def test_update_versions_dry_run_two_updaters_one_file(tmp_path): + (tmp_path / "setup.py").write_text(textwrap.dedent("""\ + from setuptools import setup + setup( + name='foo', + python_requires='>= 2.7', + classifiers=[ + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.6', + ], + ) + """)) + replacements = cpv.update_versions( + tmp_path, update=['2.7', '3.4', '3.5', '3.6', '3.7'], dry_run=True, + ) + assert (tmp_path / "setup.py").read_text() == textwrap.dedent("""\ + from setuptools import setup + setup( + name='foo', + python_requires='>= 2.7', + classifiers=[ + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.6', + ], + ) + """) + filename = str(tmp_path / "setup.py") + assert "".join(replacements[filename]) == textwrap.dedent("""\ + from setuptools import setup + setup( + name='foo', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', + classifiers=[ + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + ], + ) + """) + + +def test_update_versions_diff(tmp_path, capsys): + (tmp_path / "setup.py").write_text(textwrap.dedent("""\ + from setuptools import setup + setup( + name='foo', + classifiers=[ + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.6', + ], + ) + """)) + cpv.update_versions(tmp_path, add=['3.7'], diff=True) + assert (tmp_path / "setup.py").read_text() == textwrap.dedent("""\ + from setuptools import setup + setup( + name='foo', + classifiers=[ + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.6', + ], + ) + """) + assert ( + capsys.readouterr().out.replace(str(tmp_path), 'tmp').expandtabs() + ) == textwrap.dedent("""\ + --- tmp/setup.py (original) + +++ tmp/setup.py (updated) + @@ -4,5 +4,6 @@ + classifiers=[ + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.6', + + 'Programming Language :: Python :: 3.7', + ], + ) + + """) + + def test_update_versions_no_change(tmp_path): (tmp_path / "setup.py").write_text(textwrap.dedent("""\ from setuptools import setup @@ -171,6 +337,44 @@ def test_update_versions_no_change(tmp_path): cpv.update_versions(tmp_path, add=['3.6']) +def test_update_versions_only(tmp_path): + (tmp_path / "setup.py").write_text(textwrap.dedent("""\ + from setuptools import setup + setup( + name='foo', + classifiers=[ + 'Programming Language :: Python :: 2.7', + ], + ) + """)) + tox_ini = tmp_path / "tox.ini" + tox_ini.write_text(textwrap.dedent("""\ + [tox] + envlist = py27 + """)) + replacements = cpv.update_versions( + tmp_path, add=['3.6'], only='tox.ini', dry_run=True, + ) + assert set(replacements) == {str(tmp_path / 'tox.ini')} + + +def test_update_versions_computed(tmp_path): + (tmp_path / "setup.py").write_text(textwrap.dedent("""\ + from setuptools import setup + setup( + name='foo', + classifiers=[ + 'Programming Language :: Python :: %s' % v + for v in ['2.7'] + ], + ) + """)) + replacements = cpv.update_versions( + tmp_path, add=['3.6'], dry_run=True, + ) + assert set(replacements) == set() + + def test_main_help(monkeypatch): monkeypatch.setattr(sys, 'argv', ['check-python-versions', '--help']) with pytest.raises(SystemExit): @@ -210,6 +414,21 @@ def test_main_conflicting_args(monkeypatch, tmp_path, capsys, arg): ) +@pytest.mark.parametrize('arg', ['--diff', '--dry-run']) +def test_main_required_args(monkeypatch, tmp_path, capsys, arg): + monkeypatch.setattr(sys, 'argv', [ + 'check-python-versions', + str(tmp_path), + arg, + ]) + with pytest.raises(SystemExit): + cpv.main() + assert ( + f'argument {arg}: not allowed without --update/--add/--drop' + in capsys.readouterr().err + ) + + def test_main_here(monkeypatch, capsys): monkeypatch.setattr(sys, 'argv', [ 'check-python-versions', @@ -242,6 +461,44 @@ def test_main_single(monkeypatch, capsys, tmp_path): """) +def test_main_only(monkeypatch, capsys, tmp_path): + monkeypatch.setattr(sys, 'argv', [ + 'check-python-versions', + str(tmp_path), + '--only', 'tox.ini,setup.py', + ]) + setup_py = tmp_path / "setup.py" + setup_py.write_text(textwrap.dedent("""\ + from setuptools import setup + setup( + name='foo', + classifiers=[ + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.6', + ], + ) + """)) + tox_ini = tmp_path / "tox.ini" + tox_ini.write_text(textwrap.dedent("""\ + [tox] + envlist = py27,py36 + """)) + travis_yml = tmp_path / ".travis.yml" + travis_yml.write_text(textwrap.dedent("""\ + python: + - 2.7 + - 3.5 + """)) + cpv.main() + assert ( + capsys.readouterr().out + '\n' + ).replace(str(tmp_path) + os.path.sep, 'tmp/') == textwrap.dedent("""\ + setup.py says: 2.7, 3.6 + tox.ini says: 2.7, 3.6 + + """) + + def test_main_multiple(monkeypatch, capsys, tmp_path): monkeypatch.setattr(sys, 'argv', [ 'check-python-versions', @@ -394,6 +651,92 @@ def test_main_update_rejected(monkeypatch, capsys, tmp_path): """) +def test_main_update_diff(monkeypatch, capsys, tmp_path): + monkeypatch.setattr(sys, 'argv', [ + 'check-python-versions', + str(tmp_path), + '--add', '3.7,3.8', + '--diff', + ]) + (tmp_path / "setup.py").write_text(textwrap.dedent("""\ + from setuptools import setup + setup( + name='foo', + classifiers=[ + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.6', + ], + ) + """)) + cpv.main() + assert ( + capsys.readouterr().out + .replace(str(tmp_path) + os.path.sep, 'tmp/') + .expandtabs() + .replace(' \n', '\n\n') + ) == textwrap.dedent("""\ + --- tmp/setup.py (original) + +++ tmp/setup.py (updated) + @@ -4,5 +4,7 @@ + classifiers=[ + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.6', + + 'Programming Language :: Python :: 3.7', + + 'Programming Language :: Python :: 3.8', + ], + ) + + """) + assert (tmp_path / "setup.py").read_text() == textwrap.dedent("""\ + from setuptools import setup + setup( + name='foo', + classifiers=[ + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.6', + ], + ) + """) + + +def test_main_update_dry_run(monkeypatch, capsys, tmp_path): + monkeypatch.setattr(sys, 'argv', [ + 'check-python-versions', + str(tmp_path), + '--add', '3.7,3.8', + '--dry-run', + ]) + (tmp_path / "setup.py").write_text(textwrap.dedent("""\ + from setuptools import setup + setup( + name='foo', + classifiers=[ + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.6', + ], + ) + """)) + cpv.main() + assert ( + capsys.readouterr().out + .replace(str(tmp_path) + os.path.sep, 'tmp/') + .expandtabs() + .replace(' \n', '\n\n') + ) == textwrap.dedent("""\ + setup.py says: 2.7, 3.6, 3.7, 3.8 + """) + assert (tmp_path / "setup.py").read_text() == textwrap.dedent("""\ + from setuptools import setup + setup( + name='foo', + classifiers=[ + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.6', + ], + ) + """) + + def test_main_handles_ctrl_c(monkeypatch): def raise_keyboard_interrupt(): raise KeyboardInterrupt() diff --git a/tox.ini b/tox.ini index a7b2b38..eb45252 100644 --- a/tox.ini +++ b/tox.ini @@ -17,8 +17,7 @@ deps = coverage commands = coverage run -m pytest tests {posargs} -## coverage report -m --fail-under=100 - coverage report -m + coverage report -m --fail-under=100 [testenv:flake8] basepython = python3.6 From b2dc0f7b6646b8396472a4b0ec7629967df5ba9f Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Wed, 17 Apr 2019 20:46:31 +0300 Subject: [PATCH 68/76] Update documentation --- README.rst | 75 ++++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 70 insertions(+), 5 deletions(-) diff --git a/README.rst b/README.rst index ace959f..8407651 100644 --- a/README.rst +++ b/README.rst @@ -87,7 +87,9 @@ Usage $ check-python-versions --help usage: check-python-versions [-h] [--version] [--expect VERSIONS] - [--skip-non-packages] + [--skip-non-packages] [--only ONLY] + [--add VERSIONS] [--drop VERSIONS] + [--update VERSIONS] [--diff] [--dry-run] [where [where ...]] verify that supported Python versions are the same in setup.py, tox.ini, @@ -101,9 +103,19 @@ Usage -h, --help show this help message and exit --version show program's version number and exit --expect VERSIONS expect these versions to be supported, e.g. --expect - 2.7,3.4-3.7 + 2.7,3.5-3.7 --skip-non-packages skip arguments that are not Python packages without warning about them + --only ONLY check only the specified files (comma-separated list) + + updating supported version lists (EXPERIMENTAL): + --add VERSIONS add these versions to supported ones, e.g --add 3.8 + --drop VERSIONS drop these versions from supported ones, e.g --drop + 2.6,3.4 + --update VERSIONS update the set of supported versions, e.g. --update + 2.7,3.5-3.7 + --diff show a diff of proposed changes + --dry-run verify proposed changes without writing them to disk If run without any arguments, check-python-versions will look for a setup.py in the current working directory. @@ -121,6 +133,20 @@ helpful when, e.g. you want to run :: to check all 380+ packages, and then want re-run the checks only on the failed ones, for a faster turnabout. +There's also experimental support for updating supported Python versions +so you can do things like :: + + check-python-versions ~/projects/* --add 3.8 --dry-run + check-python-versions ~/projects/* --drop 3.4 --diff + check-python-versions ~/projects/* --update 2.7,3.4- --dry-run --diff + check-python-versions ~/projects/* --add 3.8 --drop=-2.6,-3.4 + +(the last one will show a diff for each file and ask for interactive +confirmation before making any changes.) + +Programmatically updating human-writable files is difficult, so expect +bugs (and please file issues). + Files ----- @@ -141,6 +167,16 @@ they'll be ignored (and this will not considered a failure). extract classifiers, but if that fails, it'll execute ``python setup.py --classifiers`` and parse the output. + There's rudimentary support for dynamically-computed classifiers if at + least one part is a list literal, e.g. this can work and can even be + updated :: + + classifiers=[ + ... + "Programming Language :: Python :: x.y", + ... + ] + ... expression that computes extra classifiers ..., + - **setup.py**: the ``python_requires`` argument passed to ``setup()``, if present:: @@ -150,8 +186,6 @@ they'll be ignored (and this will not considered a failure). extract the ``python_requires`` value. It expects to find a string literal or a simple expression of the form ``"literal".join(["...", "..."])``. - Only ``>=`` and ``!=`` constraints are currently supported. - - **tox.ini**: if present, it's expected to have :: [tox] @@ -186,6 +220,8 @@ they'll be ignored (and this will not considered a failure). env: - TOXENV=... + (but not all of these forms are supported for updates) + - **appveyor.yml**: if present, it's expected to have :: environment: @@ -202,6 +238,8 @@ they'll be ignored (and this will not considered a failure). Alternatively, you can use ``TOXENV`` with the usual values (pyXY). + (``TOXENV`` is currently not supported for updates.) + - **.manylinux-install.sh**: if present, it's expected to contain a loop like :: @@ -241,13 +279,40 @@ in some of the files: - **tox.ini** may have pypy[-suffix] and pypy3[-suffix] environments -- **.travis.yml** may have pypy and pypy3 jobs +- **.travis.yml** may have pypy and pypy3 jobs with optional version suffixes + (e.g. pypy2.7-6.0.0, pypy3.5-6.0.0) - **appveyor.yml** and **.manylinux-install.sh** do not usually have pypy tests, so check-python-versions cannot recognize them. These extra Pythons are shown, but not compared for consistency. +Upcoming Python releases (such as 3.8 in setup.py or 3.8-dev in a .travis.yml) +are also shown but do not cause mismatch errors. + In addition, ``python_requires`` in setup.py usually has a lower limit, but no upper limit. check-python-versions will assume this means support up to the current Python 3.x release (3.7 at the moment). + +When you're specifying Python version ranges for --expect, --add, --drop or +--update, you can use + +- ``X.Y`` (e.g. ``--add 3.8``) +- ``X.Y-U.V`` for an inclusive range (e.g. ``--add 3.5-3.8``) +- ``X.Y-``, which means from X.Y until the latest known release from the X series + (e.g. ``--add 3.5-`` is equivalent to ``--add 3.5-3.7``) +- ``-X.Y``, which is the same as ``X.0-X.Y`` + (e.g. ``--drop -3.4`` is equivalent to ``--drop 3.0-3.4``) + +or a comma-separated list of the above (e.g. ``--expect 2.7,3.5-``, +``--drop -2.6,-3.4``). + +You may have to take extra care when using ranges with no explicit lower limit, +as they look like command-line flags, so instead of :: + + --drop -2.6 + +you may need to write :: + + --drop=-2.6 + From 31d79fa8b45512e1402916f1c2d19085fabd467a Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Wed, 17 Apr 2019 20:47:42 +0300 Subject: [PATCH 69/76] --expect with --diff makes no sense without --dry-run --- README.rst | 2 +- src/check_python_versions/cli.py | 4 ++++ tests/test_cli.py | 17 +++++++++++++++++ 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 8407651..0a7e5ca 100644 --- a/README.rst +++ b/README.rst @@ -136,7 +136,7 @@ ones, for a faster turnabout. There's also experimental support for updating supported Python versions so you can do things like :: - check-python-versions ~/projects/* --add 3.8 --dry-run + check-python-versions ~/projects/* --add 3.8 --dry-run --expect 2.7,3.5-3.8 check-python-versions ~/projects/* --drop 3.4 --diff check-python-versions ~/projects/* --update 2.7,3.4- --dry-run --diff check-python-versions ~/projects/* --add 3.8 --drop=-2.6,-3.4 diff --git a/src/check_python_versions/cli.py b/src/check_python_versions/cli.py index 08897bf..9a39e37 100644 --- a/src/check_python_versions/cli.py +++ b/src/check_python_versions/cli.py @@ -255,6 +255,10 @@ def _main(): if args.dry_run and not (args.update or args.add or args.drop): parser.error( "argument --dry-run: not allowed without --update/--add/--drop") + if args.expect and args.diff and not args.dry_run: + parser.error( + "argument --expect: not allowed with --diff," + " unless you also add --dry-run") where = args.where or ['.'] if args.skip_non_packages: diff --git a/tests/test_cli.py b/tests/test_cli.py index 6dcc14c..6be6bf0 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -429,6 +429,23 @@ def test_main_required_args(monkeypatch, tmp_path, capsys, arg): ) +def test_main_diff_and_expect_and_dry_run_oh_my(monkeypatch, tmp_path, capsys): + monkeypatch.setattr(sys, 'argv', [ + 'check-python-versions', + str(tmp_path), + '--expect', '3.6-3.7', + '--update', '3.6-3.7', + '--diff', + ]) + with pytest.raises(SystemExit): + cpv.main() + assert ( + 'argument --expect: not allowed with --diff,' + ' unless you also add --dry-run' + in capsys.readouterr().err + ) + + def test_main_here(monkeypatch, capsys): monkeypatch.setattr(sys, 'argv', [ 'check-python-versions', From ba723bf3d11cbb16c5a12ff87620cc3d1a3a85e3 Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Wed, 17 Apr 2019 21:18:12 +0300 Subject: [PATCH 70/76] Handle matrix.include better in .travis.yml "Better" means if all the build jobs are defined via matrix.include or jobs.include, then we'll attempt to modify those. Also handle environment.matrix better in appveyor.yml, and here "better" means without assuming 2-space indents. --- src/check_python_versions/parsers/appveyor.py | 6 +- src/check_python_versions/parsers/travis.py | 51 +++++++++++++---- tests/parsers/test_appveyor.py | 4 +- tests/parsers/test_travis.py | 55 +++++++++++++++++++ 4 files changed, 102 insertions(+), 14 deletions(-) diff --git a/src/check_python_versions/parsers/appveyor.py b/src/check_python_versions/parsers/appveyor.py index c6de41f..8b652f8 100644 --- a/src/check_python_versions/parsers/appveyor.py +++ b/src/check_python_versions/parsers/appveyor.py @@ -115,6 +115,8 @@ def keep_complicated(value): return True return False - new_lines = update_yaml_list(orig_lines, ' matrix', new_environments, - keep=keep_complicated) + new_lines = update_yaml_list( + orig_lines, ('environment', 'matrix'), new_environments, + keep=keep_complicated, + ) return new_lines diff --git a/src/check_python_versions/parsers/travis.py b/src/check_python_versions/parsers/travis.py index 6bdb031..eb18b9e 100644 --- a/src/check_python_versions/parsers/travis.py +++ b/src/check_python_versions/parsers/travis.py @@ -84,18 +84,30 @@ def update_travis_yml_python_versions(filename, new_versions): def keep_old(ver): return not is_important(travis_normalize_py_version(ver)) - new_lines = update_yaml_list( - new_lines, "python", new_versions, filename=fp.name, keep=keep_old, - replacements=replacements, - ) + if conf.get('python'): + new_lines = update_yaml_list( + new_lines, "python", new_versions, filename=fp.name, keep=keep_old, + replacements=replacements, + ) + else: + for toplevel in 'matrix', 'jobs': + if 'include' not in conf.get(toplevel, {}): + continue + new_jobs = [ + f'python: {ver}' + for ver in new_versions + ] + new_lines = update_yaml_list( + new_lines, (toplevel, "include"), new_jobs, filename=fp.name, + ) # If python 3.7 was enabled via matrix.include, we've just added a # second 3.7 entry directly to top-level python by the above code. # So let's drop the matrix. if ( - 'matrix' in conf - and 'include' in conf['matrix'] + conf.get('python') + and 'include' in conf.get('matrix', {}) and all( job.get('dist') == 'xenial' and set(job) <= {'python', 'dist', 'sudo'} @@ -112,12 +124,31 @@ def update_yaml_list( orig_lines, key, new_value, filename=TRAVIS_YML, keep=None, replacements=None, ): + if not isinstance(key, tuple): + key = (key,) + lines = iter(enumerate(orig_lines)) + current = 0 + indents = [0] for n, line in lines: - if line.startswith(f'{key}:'): - break + stripped = line.lstrip() + if not stripped or stripped.startswith('#'): + continue + indent = len(line) - len(stripped) + if current >= len(indents): + indents.append(indent) + elif indent > indents[current]: + continue + else: + while current > 0 and indent < indents[current]: + del indents[current] + current -= 1 + if stripped.startswith(f'{key[current]}:'): + current += 1 + if current == len(key): + break else: - warn(f'Did not find {key}: setting in {filename}') + warn(f'Did not find {".".join(key)}: setting in {filename}') return orig_lines start = n @@ -147,7 +178,7 @@ def update_yaml_list( break new_lines = orig_lines[:start] + [ - f"{key}:\n" + f"{' ' * indents[-1]}{key[-1]}:\n" ] + keep_before + [ f"{' ' * indent}- {value}\n" for value in new_value diff --git a/tests/parsers/test_appveyor.py b/tests/parsers/test_appveyor.py index 37ee076..8c23a09 100644 --- a/tests/parsers/test_appveyor.py +++ b/tests/parsers/test_appveyor.py @@ -75,14 +75,14 @@ def test_appveyor_detect_py_version_pattern(s, expected): def test_update_appveyor_yml_python_versions(): appveyor_yml = StringIO(textwrap.dedent(r""" environment: - matrix: + matrix: - PYTHON: "c:\\python27" - PYTHON: "c:\\python36" """).lstrip('\n')) result = update_appveyor_yml_python_versions(appveyor_yml, ['2.7', '3.7']) assert ''.join(result) == textwrap.dedent(r""" environment: - matrix: + matrix: - PYTHON: "c:\\python27" - PYTHON: "c:\\python37" """.lstrip('\n')) diff --git a/tests/parsers/test_travis.py b/tests/parsers/test_travis.py index 8f615b8..c96256a 100644 --- a/tests/parsers/test_travis.py +++ b/tests/parsers/test_travis.py @@ -229,6 +229,35 @@ def test_update_travis_yml_python_versions_one_to_many(): """) +def test_update_travis_yml_python_versions_matrix(): + travis_yml = StringIO(textwrap.dedent("""\ + language: python + matrix: + exclude: + - python: 2.6 + # this is where the fun begins! + include: + - python: 2.7 + - python: 3.3 + install: pip install -e . + script: pytest tests + """)) + travis_yml.name = '.travis.yml' + result = update_travis_yml_python_versions(travis_yml, ["2.7", "3.4"]) + assert "".join(result) == textwrap.dedent("""\ + language: python + matrix: + exclude: + - python: 2.6 + # this is where the fun begins! + include: + - python: 2.7 + - python: 3.4 + install: pip install -e . + script: pytest tests + """) + + def test_update_yaml_list(): source_lines = textwrap.dedent("""\ language: python @@ -291,6 +320,32 @@ def test_update_yaml_not_found(capsys): ) +def test_update_yaml_nested_keys_not_found(capsys): + source_lines = textwrap.dedent("""\ + language: python + matrix: + allow_failures: + - python: 3.8 + install: pip install -e . + script: pytest tests + """).splitlines(True) + result = update_yaml_list( + source_lines, ("matrix", "include"), ["python: 2.7"]) + assert "".join(result) == textwrap.dedent("""\ + language: python + matrix: + allow_failures: + - python: 3.8 + install: pip install -e . + script: pytest tests + """) + assert ( + "Did not find matrix.include: setting in .travis.yml" + in capsys.readouterr().err + ) + + + def test_drop_yaml_node(): source_lines = textwrap.dedent("""\ language: python From e7c1783d21556ade4021e04bea3f064ca934a48a Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Wed, 17 Apr 2019 21:23:07 +0300 Subject: [PATCH 71/76] Better dist: xenial positioning in .travis.yml --- src/check_python_versions/parsers/travis.py | 6 +++-- tests/parsers/test_travis.py | 25 +++++++++++++++++---- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/src/check_python_versions/parsers/travis.py b/src/check_python_versions/parsers/travis.py index eb18b9e..8f8caa7 100644 --- a/src/check_python_versions/parsers/travis.py +++ b/src/check_python_versions/parsers/travis.py @@ -74,7 +74,7 @@ def update_travis_yml_python_versions(filename, new_versions): if conf.get('dist') != 'xenial': new_lines = drop_yaml_node(new_lines, 'dist', filename=fp.name) new_lines = add_yaml_node(new_lines, 'dist', 'xenial', - before='python') + before=('python', 'matrix', 'jobs')) if conf.get('sudo') is False: # sudo is ignored nowadays, but in earlier times # you needed both dist: xenial and sudo: required @@ -218,9 +218,11 @@ def add_yaml_node(orig_lines, key, value, before=None): lines = iter(enumerate(orig_lines)) where = len(orig_lines) if before: + if not isinstance(before, (list, tuple, set)): + before = (before, ) lines = iter(enumerate(orig_lines)) for n, line in lines: - if line == f'{before}:\n': + if any(line == f'{key}:\n' for key in before): where = n break diff --git a/tests/parsers/test_travis.py b/tests/parsers/test_travis.py index c96256a..dcffb9f 100644 --- a/tests/parsers/test_travis.py +++ b/tests/parsers/test_travis.py @@ -345,7 +345,6 @@ def test_update_yaml_nested_keys_not_found(capsys): ) - def test_drop_yaml_node(): source_lines = textwrap.dedent("""\ language: python @@ -451,7 +450,7 @@ def test_drop_yaml_node_when_duplicate(capsys): ) -def test_add_yaml_node_before(): +def test_add_yaml_node(): source_lines = textwrap.dedent("""\ language: python python: @@ -468,7 +467,7 @@ def test_add_yaml_node_before(): """) -def test_add_yaml_node(): +def test_add_yaml_node_before(): source_lines = textwrap.dedent("""\ language: python python: @@ -485,7 +484,7 @@ def test_add_yaml_node(): """) -def test_add_yaml_node_at_end(): +def test_add_yaml_node_at_end_when_before_not_found(): source_lines = textwrap.dedent("""\ language: python python: @@ -500,3 +499,21 @@ def test_add_yaml_node_at_end(): script: pytest tests dist: xenial """) + + +def test_add_yaml_node_before_alternatives(): + source_lines = textwrap.dedent("""\ + language: python + python: + - 3.6 + script: pytest tests + """).splitlines(True) + result = add_yaml_node(source_lines, 'dist', 'xenial', + before=('sudo', 'python')) + assert "".join(result) == textwrap.dedent("""\ + language: python + dist: xenial + python: + - 3.6 + script: pytest tests + """) From 50391bc423c6fca45e6029b7f1b389d379f1e64b Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Wed, 17 Apr 2019 21:26:48 +0300 Subject: [PATCH 72/76] Keep pypy jobs in matrix.include in .travis.yml --- src/check_python_versions/parsers/travis.py | 8 ++++++++ tests/parsers/test_travis.py | 2 ++ 2 files changed, 10 insertions(+) diff --git a/src/check_python_versions/parsers/travis.py b/src/check_python_versions/parsers/travis.py index 8f8caa7..27e7328 100644 --- a/src/check_python_versions/parsers/travis.py +++ b/src/check_python_versions/parsers/travis.py @@ -84,6 +84,13 @@ def update_travis_yml_python_versions(filename, new_versions): def keep_old(ver): return not is_important(travis_normalize_py_version(ver)) + def keep_old_job(job): + if job.startswith('python:'): + ver = job[len('python:'):].strip() + return not is_important(travis_normalize_py_version(ver)) + else: + return True + if conf.get('python'): new_lines = update_yaml_list( new_lines, "python", new_versions, filename=fp.name, keep=keep_old, @@ -99,6 +106,7 @@ def keep_old(ver): ] new_lines = update_yaml_list( new_lines, (toplevel, "include"), new_jobs, filename=fp.name, + keep=keep_old_job ) # If python 3.7 was enabled via matrix.include, we've just added a diff --git a/tests/parsers/test_travis.py b/tests/parsers/test_travis.py index dcffb9f..26fdc5a 100644 --- a/tests/parsers/test_travis.py +++ b/tests/parsers/test_travis.py @@ -239,6 +239,7 @@ def test_update_travis_yml_python_versions_matrix(): include: - python: 2.7 - python: 3.3 + - python: pypy install: pip install -e . script: pytest tests """)) @@ -253,6 +254,7 @@ def test_update_travis_yml_python_versions_matrix(): include: - python: 2.7 - python: 3.4 + - python: pypy install: pip install -e . script: pytest tests """) From e50f9e25e250fc52c2b25f1850b0544282b46483 Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Wed, 17 Apr 2019 21:28:38 +0300 Subject: [PATCH 73/76] Keep non-Python jobs The logic here is very fragile and I think only the very last job will be kept intact. --- tests/parsers/test_travis.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/parsers/test_travis.py b/tests/parsers/test_travis.py index 26fdc5a..d5efe8b 100644 --- a/tests/parsers/test_travis.py +++ b/tests/parsers/test_travis.py @@ -240,6 +240,10 @@ def test_update_travis_yml_python_versions_matrix(): - python: 2.7 - python: 3.3 - python: pypy + - name: flake8 + python: 2.7 + install: pip install flake8 + script: flake8 . install: pip install -e . script: pytest tests """)) @@ -255,6 +259,10 @@ def test_update_travis_yml_python_versions_matrix(): - python: 2.7 - python: 3.4 - python: pypy + - name: flake8 + python: 2.7 + install: pip install flake8 + script: flake8 . install: pip install -e . script: pytest tests """) From 8b679aa8136309f6407beba86cb6f2885e960c11 Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Wed, 17 Apr 2019 21:31:26 +0300 Subject: [PATCH 74/76] Keep Travis jobs entire, not just the 1st line --- src/check_python_versions/parsers/travis.py | 10 ++++++++-- tests/parsers/test_travis.py | 8 ++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/src/check_python_versions/parsers/travis.py b/src/check_python_versions/parsers/travis.py index 27e7328..296753c 100644 --- a/src/check_python_versions/parsers/travis.py +++ b/src/check_python_versions/parsers/travis.py @@ -165,14 +165,17 @@ def update_yaml_list( keep_before = [] keep_after = [] lines_to_keep = keep_before + kept_last = False for n, line in lines: stripped = line.lstrip() + line_indent = len(line) - len(stripped) if stripped.startswith('- '): lines_to_keep = keep_after - indent = len(line) - len(stripped) + indent = line_indent end = n + 1 value = stripped[2:].strip() - if keep and keep(value): + kept_last = keep and keep(value) + if kept_last: if replacements and value in replacements: lines_to_keep.append( f"{' '* indent}- {replacements[value]}\n" @@ -182,6 +185,9 @@ def update_yaml_list( elif stripped.startswith('#'): lines_to_keep.append(line) end = n + 1 + elif kept_last and line_indent > indent: + lines_to_keep.append(line) + end = n + 1 if line and line[0] != ' ': break diff --git a/tests/parsers/test_travis.py b/tests/parsers/test_travis.py index d5efe8b..2afb92b 100644 --- a/tests/parsers/test_travis.py +++ b/tests/parsers/test_travis.py @@ -240,6 +240,10 @@ def test_update_travis_yml_python_versions_matrix(): - python: 2.7 - python: 3.3 - python: pypy + - name: docs + python: 2.7 + install: pip install sphinx + script: sphinx-build . - name: flake8 python: 2.7 install: pip install flake8 @@ -259,6 +263,10 @@ def test_update_travis_yml_python_versions_matrix(): - python: 2.7 - python: 3.4 - python: pypy + - name: docs + python: 2.7 + install: pip install sphinx + script: sphinx-build . - name: flake8 python: 2.7 install: pip install flake8 From 16ba194ee1f0a2412a610ed0b9a0275289b9af34 Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Wed, 17 Apr 2019 21:53:40 +0300 Subject: [PATCH 75/76] More travis.yml correctness fixes Lists nested more deeply could've confused the "parser". --- src/check_python_versions/parsers/travis.py | 16 ++++-- tests/parsers/test_travis.py | 62 ++++++++++++++++++++- 2 files changed, 72 insertions(+), 6 deletions(-) diff --git a/src/check_python_versions/parsers/travis.py b/src/check_python_versions/parsers/travis.py index 296753c..677e756 100644 --- a/src/check_python_versions/parsers/travis.py +++ b/src/check_python_versions/parsers/travis.py @@ -162,6 +162,7 @@ def update_yaml_list( start = n end = n + 1 indent = 2 + list_indent = None keep_before = [] keep_after = [] lines_to_keep = keep_before @@ -169,7 +170,9 @@ def update_yaml_list( for n, line in lines: stripped = line.lstrip() line_indent = len(line) - len(stripped) - if stripped.startswith('- '): + if list_indent is None and stripped.startswith('- '): + list_indent = line_indent + if stripped.startswith('- ') and line_indent == list_indent: lines_to_keep = keep_after indent = line_indent end = n + 1 @@ -185,10 +188,15 @@ def update_yaml_list( elif stripped.startswith('#'): lines_to_keep.append(line) end = n + 1 - elif kept_last and line_indent > indent: - lines_to_keep.append(line) + elif line_indent > indent: + if kept_last: + lines_to_keep.append(line) end = n + 1 - if line and line[0] != ' ': + elif line == '\n': + continue + elif line[0] != ' ': + break + elif list_indent is not None and line_indent < list_indent: break new_lines = orig_lines[:start] + [ diff --git a/tests/parsers/test_travis.py b/tests/parsers/test_travis.py index 2afb92b..09e00d2 100644 --- a/tests/parsers/test_travis.py +++ b/tests/parsers/test_travis.py @@ -320,7 +320,7 @@ def test_update_yaml_list_keep_indent_comments_and_pypy(): """) -def test_update_yaml_not_found(capsys): +def test_update_yaml_list_not_found(capsys): source_lines = textwrap.dedent("""\ language: python install: pip install -e . @@ -338,7 +338,7 @@ def test_update_yaml_not_found(capsys): ) -def test_update_yaml_nested_keys_not_found(capsys): +def test_update_yaml_list_nested_keys_not_found(capsys): source_lines = textwrap.dedent("""\ language: python matrix: @@ -363,6 +363,64 @@ def test_update_yaml_nested_keys_not_found(capsys): ) +def test_update_yaml_list_nesting_does_not_confuse(): + source_lines = textwrap.dedent("""\ + language: python + matrix: + include: + + - name: flake8 + script: + - flake8 + + - python: 2.7 + env: + - PURE_PYTHON: 1 + allow_failures: + - python: 3.8 + install: pip install -e . + script: pytest tests + """).splitlines(True) + result = update_yaml_list( + source_lines, ("matrix", "include"), [], + keep=lambda job: job.startswith('python:')) + assert "".join(result) == textwrap.dedent("""\ + language: python + matrix: + include: + - python: 2.7 + env: + - PURE_PYTHON: 1 + allow_failures: + - python: 3.8 + install: pip install -e . + script: pytest tests + """) + + +def test_update_yaml_list_nesting_some_garbage(): + source_lines = textwrap.dedent("""\ + language: python + matrix: + include: + - python: 2.7 + garbage + install: pip install -e . + script: pytest tests + """).splitlines(True) + result = update_yaml_list( + source_lines, ("matrix", "include"), ['python: 2.7']) + assert "".join(result) == textwrap.dedent("""\ + language: python + matrix: + include: + - python: 2.7 + garbage + install: pip install -e . + script: pytest tests + """) + + def test_drop_yaml_node(): source_lines = textwrap.dedent("""\ language: python From 6d4fd3d9e2d943485d36e8ddb5fb5ffbfc5a4195 Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Wed, 17 Apr 2019 22:29:54 +0300 Subject: [PATCH 76/76] Windows is special, as usual --- tests/test_cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index 6be6bf0..e29c5a1 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -308,8 +308,8 @@ def test_update_versions_diff(tmp_path, capsys): ) """) assert ( - capsys.readouterr().out.replace(str(tmp_path), 'tmp').expandtabs() - ) == textwrap.dedent("""\ + capsys.readouterr().out.replace(str(tmp_path) + os.path.sep, 'tmp/') + ).expandtabs() == textwrap.dedent("""\ --- tmp/setup.py (original) +++ tmp/setup.py (updated) @@ -4,5 +4,6 @@