diff --git a/.travis.yml b/.travis.yml index 876a061..1e3b7a3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,198 +1,216 @@ env: global: - - PIP_URL="https://bootstrap.pypa.io/get-pip.py" - - PIP="python -m pip" + - PIP_URL="https://bootstrap.pypa.io/get-pip.py" + - PIP="python -m pip" matrix: fast_finish: true include: - - os: osx - language: generic - env: - - NAME="Python 2.6" - - PIP_URL="https://bootstrap.pypa.io/2.6/get-pip.py" - - PIP=pip - before_install: - - brew update - - brew upgrade pyenv || brew install pyenv || true - - pyenv install 2.6.9 - - pyenv local 2.6.9 - - eval "$(pyenv init -)" - - pip install virtualenv - - python2.6 -m virtualenv env - - pyenv local --unset - - source env/bin/activate - - os: osx - language: generic - env: NAME="Python 2.7" - before_install: - - brew update - - brew upgrade python2 || true - - python2.7 -m pip install virtualenv - - python2.7 -m virtualenv env - - source env/bin/activate - - os: osx - language: generic - env: NAME="Python pypy" - before_install: - - brew update - - brew upgrade pypy || brew install pypy || true - - pypy -m pip install virtualenv - - pypy -m virtualenv env - - source env/bin/activate - - os: osx - language: generic - env: NAME="Python 3.4" - before_install: - - brew update - - brew upgrade pyenv || brew install pyenv || true - - pyenv install 3.4.9 - - pyenv local 3.4.9 - - eval "$(pyenv init -)" - - python3.4 -m venv env - - pyenv local --unset - - source env/bin/activate - - os: osx - language: generic - env: NAME="Python 3.5" - before_install: - - brew update - - brew upgrade pyenv || brew install pyenv || true - - pyenv install 3.5.6 - - pyenv local 3.5.6 - - eval "$(pyenv init -)" - - python3.5 -m venv env - - pyenv local --unset - - source env/bin/activate - - os: osx - language: generic - env: NAME="Python 3.6" - before_install: - - brew update - - brew upgrade pyenv || brew install pyenv || true - - pyenv install 3.6.6 - - pyenv local 3.6.6 - - eval "$(pyenv init -)" - - python3.6 -m venv env - - pyenv local --unset - - source env/bin/activate - - os: osx - language: generic - env: NAME="Python 3.7" - before_install: - - brew update - - brew upgrade python || true - - python3.7 -m venv env - - source env/bin/activate - - os: osx - language: generic - env: NAME="Python HEAD" - before_install: - - brew update - - brew unlink python - - brew install python --HEAD - - python3 -m venv env - - source env/bin/activate - - os: osx - language: generic - env: NAME="Python pypy3" - before_install: - - brew update - - brew upgrade pypy3 || brew install pypy3 || true - - pypy3 -m venv env - - source env/bin/activate - - os: linux - env: - - PIP_URL="https://bootstrap.pypa.io/2.6/get-pip.py" - - PIP=env/bin/pip - language: python - python: 2.6 - before_install: - - pip install virtualenv - - python2.6 -m virtualenv env - - source env/bin/activate - - os: linux - language: python - python: 2.7 - before_install: - - pip install virtualenv - - python2.7 -m virtualenv env - - source env/bin/activate - - os: linux - language: python - python: pypy - before_install: - - pip install virtualenv - - pypy -m virtualenv env - - source env/bin/activate - - os: linux - language: python - python: 3.4 - before_install: - - python3.4 -m venv env - - source env/bin/activate - - os: linux - language: python - python: 3.5 - before_install: - - python3.5 -m venv env - - source env/bin/activate - - os: linux - language: python - python: 3.6 - before_install: - - python3.6 -m venv env - - source env/bin/activate - - os: linux - language: python - python: &python_major_ver 3.7 - dist: xenial - sudo: true - before_install: - - python3.7 -m venv env - - source env/bin/activate - - os: linux - language: python - python: 3.6-dev - before_install: - - python3.6 -m venv env - - source env/bin/activate - - os: linux - language: python - python: 3.7-dev - before_install: - - python3.7 -m venv env - - source env/bin/activate - - os: linux - language: python - python: nightly - before_install: - - python3 -m venv env - - source env/bin/activate - - os: linux - language: python - python: pypy3 - before_install: - - pypy3 -m venv env - - source env/bin/activate + - os: osx + language: generic + env: + - NAME="Python 2.6" + - PIP_URL="https://bootstrap.pypa.io/2.6/get-pip.py" + - PIP=pip + before_install: + - brew update + - brew upgrade pyenv || brew install pyenv || true + - pyenv install 2.6.9 + - pyenv local 2.6.9 + - eval "$(pyenv init -)" + - pip install virtualenv + - python2.6 -m virtualenv env + - pyenv local --unset + - source env/bin/activate + - os: osx + language: generic + env: NAME="Python 2.7" + before_install: + - brew update + - brew upgrade python2 || true + - python2.7 -m pip install virtualenv + - python2.7 -m virtualenv env + - source env/bin/activate + - os: osx + language: generic + env: NAME="Python pypy" + before_install: + - brew update + - brew upgrade pypy || brew install pypy || true + - pypy -m pip install virtualenv + - pypy -m virtualenv env + - source env/bin/activate + - os: osx + language: generic + env: NAME="Python 3.4" + before_install: + - brew update + - brew upgrade pyenv || brew install pyenv || true + - pyenv install 3.4.9 + - pyenv local 3.4.9 + - eval "$(pyenv init -)" + - python3.4 -m venv env + - pyenv local --unset + - source env/bin/activate + - os: osx + language: generic + env: NAME="Python 3.5" + before_install: + - brew update + - brew upgrade pyenv || brew install pyenv || true + - pyenv install 3.5.6 + - pyenv local 3.5.6 + - eval "$(pyenv init -)" + - python3.5 -m venv env + - pyenv local --unset + - source env/bin/activate + - os: osx + language: generic + env: NAME="Python 3.6" + before_install: + - brew update + - brew upgrade pyenv || brew install pyenv || true + - pyenv install 3.6.6 + - pyenv local 3.6.6 + - eval "$(pyenv init -)" + - python3.6 -m venv env + - pyenv local --unset + - source env/bin/activate + - os: osx + language: generic + env: NAME="Python 3.7" + before_install: + - brew update + - brew upgrade python || true + - python3.7 -m venv env + - source env/bin/activate + - os: osx + language: generic + env: NAME="Python HEAD" + before_install: + - brew update + - brew unlink python + - brew install python --HEAD + - python3 -m venv env + - source env/bin/activate + - os: osx + language: generic + env: NAME="Python pypy3" + before_install: + - brew update + - brew upgrade pypy3 || brew install pypy3 || true + - pypy3 -m venv env + - source env/bin/activate + - os: linux + env: + - PIP_URL="https://bootstrap.pypa.io/2.6/get-pip.py" + - PIP=env/bin/pip + language: python + python: 2.6 + before_install: + - pip install virtualenv + - python2.6 -m virtualenv env + - source env/bin/activate + - os: linux + language: python + python: 2.7 + before_install: + - pip install virtualenv + - python2.7 -m virtualenv env + - source env/bin/activate + - os: linux + language: python + python: pypy + before_install: + - pip install virtualenv + - pypy -m virtualenv env + - source env/bin/activate + - os: linux + language: python + python: 3.4 + before_install: + - python3.4 -m venv env + - source env/bin/activate + - os: linux + language: python + python: 3.5 + before_install: + - python3.5 -m venv env + - source env/bin/activate + - os: linux + language: python + python: 3.6 + before_install: + - python3.6 -m venv env + - source env/bin/activate + - os: linux + language: python + python: 3.7 + dist: xenial + sudo: true + before_install: + - python3.7 -m venv env + - source env/bin/activate + - os: linux + language: python + python: 3.6-dev + before_install: + - python3.6 -m venv env + - source env/bin/activate + - os: linux + language: python + python: 3.7-dev + before_install: + - python3.7 -m venv env + - source env/bin/activate + - os: linux + language: python + python: nightly + before_install: + - python3 -m venv env + - source env/bin/activate + - os: linux + language: python + python: pypy3 + before_install: + - pypy3 -m venv env + - source env/bin/activate allow_failures: - - os: osx - env: NAME="Python HEAD" - - os: linux - python: 3.6-dev - - os: linux - python: 3.7-dev - - os: linux - python: nightly + - os: osx + env: NAME="Python HEAD" + - os: linux + python: 3.6-dev + - os: linux + python: 3.7-dev + - os: linux + python: nightly install: - - which python - - ls env/bin - - curl ${PIP_URL} | python - - ${PIP} install -r requirements.txt +- curl ${PIP_URL} | python +- ${PIP} install -r requirements.txt script: - - python setup.py test +- make test after_success: - - python -m codecov +- python -m codecov + +deploy: +- provider: pypi + user: Ilya.Kulakov + password: + secure: "VlZYEwVXqGknNEfz6vCo26JWtcbXfBY0ihG+Co6PO7JQn2jEoogKEsMhi3f/rs22nlsH124AQa7bugmH1TkgBn8ODsKLZWGAQKSm9tQlYD2idsTYqCDk5bCAWpjX/RJqZZeMnDmL9BFJkaubrTfnFWdMxI1xXi/G9wC4NaSXSJc=" + distributions: "sdist bdist_wheel" + on: + tags: true + branch: master + python: 3.7 +- provider: releases + api_key: + secure: "QHn7vzWo7rbgemP37qdNU4h+q7Xb2CQ7HxPFfa7yTsxFd8V4+sQLVrnaQtzYTM8dJWvRgi8PVHVGl2VGnQAiRM4Nd/NE/3HL9aHQIfWRtZ6XHfNVQ55bxJzLfZZy2M+32b8W268ELj3ty4C3Mo7TuOTv4svQoRDrLzGozJCpu+w=" + file_glob: true + file: dist/* + on: + tags: true + branch: master + python: 3.7 diff --git a/CHANGES.rst b/CHANGES.rst new file mode 100644 index 0000000..a0aea8d --- /dev/null +++ b/CHANGES.rst @@ -0,0 +1,10 @@ +CHANGES +======= + +1.18.0 (2018-08-xx) +------------------- + +- Add **CHANGES.rst** to track further changes +- Add tests +- Use `git check-attr` to test against export-ignore +- Better support for complicated file names (with unicode or diff --git a/Makefile b/Makefile index 9e4c319..a866e2b 100644 --- a/Makefile +++ b/Makefile @@ -8,10 +8,8 @@ all: @echo " make uninstall" @echo " test" -test: test_pep - -test_pep: - pep8 --max-line-length=240 git_archive_all.py +test: + python setup.py test install: install -d -m 0755 $(TARGET_DIR) diff --git a/git_archive_all.py b/git_archive_all.py index 0edf480..9dbadc8 100755 --- a/git_archive_all.py +++ b/git_archive_all.py @@ -167,17 +167,21 @@ def is_file_excluded(self, repo_abspath, repo_file_path): @param repo_abspath: Absolute path to the git repository. @type repo_abspath: str - @param repo_file_path: Path to a file within repo_abspath. + @param repo_file_path: Path to a file relative to repo_abspath. @type repo_file_path: str @return: True if file should be excluded. Otherwise False. @rtype: bool """ out = self.run_git_shell( - 'git check-attr -a -- %s' % repo_file_path, + 'git check-attr -z export-ignore -- %s' % repo_file_path, cwd=repo_abspath - ) - return 'export-ignore: set' in out + ).split('\0') + + try: + return out[2] == 'set' + except IndexError: + return False def archive_all_files(self, archiver): """ @@ -210,13 +214,11 @@ def walk_git_files(self, repo_path=''): """ repo_abspath = path.join(self.main_repo_abspath, repo_path) repo_file_paths = self.run_git_shell( - "git ls-files --cached --full-name --no-empty-directory", + 'git ls-files -z --cached --full-name --no-empty-directory', repo_abspath - ).splitlines() + ).split('\0')[:-1] for repo_file_path in repo_file_paths: - # Git puts path in quotes if file path has unicode characters. - repo_file_path = repo_file_path.strip('"') # file path relative to current repo repo_file_abspath = path.join(repo_abspath, repo_file_path) # absolute file path main_repo_file_path = path.join(repo_path, repo_file_path) # file path relative to the main repo @@ -230,8 +232,8 @@ def walk_git_files(self, repo_path=''): yield main_repo_file_path if self.force_sub: - self.run_git_shell("git submodule init", repo_abspath) - self.run_git_shell("git submodule update", repo_abspath) + self.run_git_shell('git submodule init', repo_abspath) + self.run_git_shell('git submodule update', repo_abspath) try: repo_gitmodules_abspath = path.join(repo_abspath, ".gitmodules") @@ -240,21 +242,21 @@ def walk_git_files(self, repo_path=''): lines = f.readlines() for l in lines: - m = re.match("^\s*path\s*=\s*(.*)\s*$", l) + m = re.match("^\\s*path\\s*=\\s*(.*)\\s*$", l) if m: - submodule_path = m.group(1) - submodule_abspath = path.join(repo_path, submodule_path) + repo_submodule_path = m.group(1) # relative to repo_path + main_repo_submodule_path = path.join(repo_path, repo_submodule_path) # relative to main_repo_abspath - if self.is_file_excluded(repo_abspath, submodule_path): + if self.is_file_excluded(repo_abspath, repo_submodule_path): continue - for submodule_file_path in self.walk_git_files(submodule_abspath): - rel_file_path = submodule_file_path.replace(repo_path, "", 1).strip("/") - if self.is_file_excluded(repo_abspath, rel_file_path): + for main_repo_submodule_file_path in self.walk_git_files(main_repo_submodule_path): + repo_submodule_file_path = main_repo_submodule_file_path.replace(repo_path, "", 1).strip("/") + if self.is_file_excluded(repo_abspath, repo_submodule_file_path): continue - yield submodule_file_path + yield main_repo_submodule_file_path except IOError: pass @@ -349,7 +351,7 @@ def main(): output_name = path.basename(output_file_path) output_name = re.sub( - '(\.zip|\.tar|\.tgz|\.txz|\.gz|\.bz2|\.xz|\.tar\.gz|\.tar\.bz2|\.tar\.xz)$', + '(\\.zip|\\.tar|\\.tgz|\\.txz|\\.gz|\\.bz2|\\.xz|\\.tar\\.gz|\\.tar\\.bz2|\\.tar\\.xz)$', '', output_name ) or "Archive" diff --git a/requirements.txt b/requirements.txt index 1695968..a77637b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,4 @@ pytest==3.7.1; python_version>"2.6" pytest==3.2.5; python_version<="2.6" pytest-cov==2.5.1 codecov==2.0.15 +pycodestyle==2.4.0 diff --git a/test_git_archive_all.py b/test_git_archive_all.py index 5214a3d..16d8d18 100644 --- a/test_git_archive_all.py +++ b/test_git_archive_all.py @@ -1,10 +1,17 @@ +# -*- coding: utf-8 -*- + +from __future__ import print_function +from __future__ import unicode_literals + from copy import deepcopy import errno from functools import partial import os from subprocess import check_call -from tarfile import TarFile +import sys +from tarfile import TarFile, PAX_FORMAT +import pycodestyle import pytest from git_archive_all import GitArchiver @@ -18,13 +25,6 @@ def makedirs(p): raise -def pytest_param(*values, **kwargs): - try: - return pytest.param(*values, **kwargs) - except AttributeError: - return values - - @pytest.fixture def git_env(tmpdir_factory): """ @@ -140,50 +140,78 @@ def archive(self, path): }) } +base_quoted = deepcopy(base) +base_quoted['data'] = DirRecord({ + '\"hello world.dat\"': FileRecord('Special cases aren\'t special enough to break the rules.'), + '\'hello world.dat\'': FileRecord('Although practicality beats purity.') +}) + ignore_in_root = deepcopy(base) -ignore_in_root['.gitattributes'] = FileRecord('\ntests/__init__.py export-ignore') +ignore_in_root['.gitattributes'] = FileRecord('tests/__init__.py export-ignore') ignore_in_root['tests'] = DirRecord({ '__init__.py': FileRecord('#Complex is better than complicated.', excluded=True) }) ignore_in_submodule = deepcopy(base) -ignore_in_submodule['lib']['.gitattributes'] = FileRecord('\ntests/__init__.py export-ignore') +ignore_in_submodule['lib']['.gitattributes'] = FileRecord('tests/__init__.py export-ignore') ignore_in_submodule['lib']['tests'] = DirRecord({ - '__init__.py': FileRecord('#Flat is better than nested.', excluded=True) + '__init__.py': FileRecord('#Complex is better than complicated.', excluded=True) }) ignore_in_nested_submodule = deepcopy(base) -ignore_in_nested_submodule['lib']['extra']['.gitattributes'] = FileRecord('\ntests/__init__.py export-ignore') +ignore_in_nested_submodule['lib']['extra']['.gitattributes'] = FileRecord('tests/__init__.py export-ignore') ignore_in_nested_submodule['lib']['extra']['tests'] = DirRecord({ - '__init__.py': FileRecord('#Sparse is better than dense.', excluded=True) + '__init__.py': FileRecord('#Complex is better than complicated.', excluded=True) }) ignore_in_submodule_from_root = deepcopy(base) -ignore_in_submodule_from_root['.gitattributes'] = FileRecord('\nlib/tests/__init__.py export-ignore') +ignore_in_submodule_from_root['.gitattributes'] = FileRecord('lib/tests/__init__.py export-ignore') ignore_in_submodule_from_root['lib']['tests'] = DirRecord({ - '__init__.py': FileRecord('#Readability counts.', excluded=True) + '__init__.py': FileRecord('#Complex is better than complicated.', excluded=True) }) ignore_in_nested_submodule_from_root = deepcopy(base) -ignore_in_nested_submodule_from_root['.gitattributes'] = FileRecord('\nlib/extra/tests/__init__.py export-ignore') +ignore_in_nested_submodule_from_root['.gitattributes'] = FileRecord('lib/extra/tests/__init__.py export-ignore') ignore_in_nested_submodule_from_root['lib']['extra']['tests'] = DirRecord({ - '__init__.py': FileRecord('#Special cases aren\'t special enough to break the rules.', excluded=True) + '__init__.py': FileRecord('#Complex is better than complicated.', excluded=True) }) ignore_in_nested_submodule_from_submodule = deepcopy(base) -ignore_in_nested_submodule_from_submodule['lib']['.gitattributes'] = FileRecord('\nextra/tests/__init__.py export-ignore') +ignore_in_nested_submodule_from_submodule['lib']['.gitattributes'] = FileRecord('extra/tests/__init__.py export-ignore') ignore_in_nested_submodule_from_submodule['lib']['extra']['tests'] = DirRecord({ - '__init__.py': FileRecord('#Although practicality beats purity.', excluded=True) + '__init__.py': FileRecord('#Complex is better than complicated.', excluded=True) +}) + +unset_export_ignore = deepcopy(base) +unset_export_ignore['.gitattributes'] = FileRecord('.* export-ignore\n*.htaccess -export-ignore', excluded=True) +unset_export_ignore['.a'] = FileRecord('Flat is better than nested.', excluded=True) +unset_export_ignore['.b'] = FileRecord('Sparse is better than dense.', excluded=True) +unset_export_ignore['.htaccess'] = FileRecord('Readability counts.') + +unicode_base = deepcopy(base) +unicode_base['data'] = DirRecord({ + 'مرحبا بالعالم.dat': FileRecord('Special cases aren\'t special enough to break the rules.') +}) + +unicode_quoted = deepcopy(base) +unicode_quoted['data'] = DirRecord({ + '\"مرحبا بالعالم.dat\"': FileRecord('Special cases aren\'t special enough to break the rules.'), + '\'привет мир.dat\'': FileRecord('Although practicality beats purity.') }) @pytest.mark.parametrize('contents', [ - pytest_param(base, id='No Ignore'), - pytest_param(ignore_in_root, id='Ignore in Root'), - pytest_param(ignore_in_submodule, id='Ignore in Submodule'), - pytest_param(ignore_in_nested_submodule, id='Ignore in Nested Submodule'), - pytest_param(ignore_in_submodule_from_root, id='Ignore in Submodule from Root'), - pytest_param(ignore_in_nested_submodule_from_root, id='Ignore in Nested Submodule from Root'), + pytest.param(base, id='No Ignore'), + pytest.param(base_quoted, id='No Ignore (Quoted)'), + pytest.param(ignore_in_root, id='Ignore in Root'), + pytest.param(ignore_in_submodule, id='Ignore in Submodule'), + pytest.param(ignore_in_nested_submodule, id='Ignore in Nested Submodule'), + pytest.param(ignore_in_submodule_from_root, id='Ignore in Submodule from Root'), + pytest.param(ignore_in_nested_submodule_from_root, id='Ignore in Nested Submodule from Root'), + pytest.param(ignore_in_nested_submodule_from_submodule, id='Ignore in Nested Submodule from Submodule'), + pytest.param(unset_export_ignore, id='-export-ignore'), + pytest.param(unicode_base, id='No Ignore (Unicode)'), + pytest.param(unicode_quoted, id='No Ignore (Quoted Unicode)') ]) def test_ignore(contents, tmpdir, git_env): """ @@ -197,7 +225,7 @@ def test_ignore(contents, tmpdir, git_env): repo_tar_path = os.path.join(str(tmpdir), 'repo.tar') repo.archive(repo_tar_path) - repo_tar = TarFile(repo_tar_path) + repo_tar = TarFile(repo_tar_path, format=PAX_FORMAT, encoding='utf-8') def make_expected(contents): e = {} @@ -216,7 +244,12 @@ def make_actual(tar_file): for m in tar_file.getmembers(): if m.isfile(): - a[m.name] = tar_file.extractfile(m).read().decode() + name = m.name + + if sys.version_info < (3,): + name = m.name.decode('utf-8') + + a[name] = tar_file.extractfile(m).read().decode() else: raise NotImplementedError @@ -226,3 +259,9 @@ def make_actual(tar_file): actual = make_actual(repo_tar) assert actual == expected + + +def test_pycodestyle(): + style = pycodestyle.StyleGuide(repeat=True, max_line_length=240) + report = style.check_files(['git_archive_all.py']) + assert report.total_errors == 0, "Found code style errors (and warnings)."