From 28df74b7c9ce9d104fa803032322d4bb10bda7dc Mon Sep 17 00:00:00 2001 From: Ilya Kulakov Date: Fri, 4 Dec 2020 18:33:51 +0600 Subject: [PATCH 1/2] Use git to get list of submodules Parsing .gitmodules may yield paths that are ignored by git. Refs #85 --- git_archive_all.py | 55 ++++++++++++++++++++++++++++----------------- git_archive_all.pyi | 5 ++++- 2 files changed, 39 insertions(+), 21 deletions(-) diff --git a/git_archive_all.py b/git_archive_all.py index 5ace362..8e58744 100755 --- a/git_archive_all.py +++ b/git_archive_all.py @@ -345,30 +345,19 @@ def walk_git_files(self, repo_path=fspath('')): self.run_git_shell('git submodule init', repo_abspath) self.run_git_shell('git submodule update', repo_abspath) - try: - repo_gitmodules_abspath = path.join(repo_abspath, fspath(".gitmodules")) - - with open(repo_gitmodules_abspath) as f: - lines = f.readlines() - - for l in lines: - m = re.match("^\\s*path\\s*=\\s*(.*)\\s*$", l) + for repo_submodule_path in self.list_repo_submodules(repo_abspath): # relative to repo_path + if self.is_file_excluded(repo_abspath, repo_submodule_path): + continue - if m: - repo_submodule_path = fspath(m.group(1)) # relative to repo_path - main_repo_submodule_path = path.join(repo_path, repo_submodule_path) # relative to main_repo_abspath + main_repo_submodule_path = path.join(repo_path, repo_submodule_path) # relative to main_repo_abspath - if self.is_file_excluded(repo_abspath, repo_submodule_path): - continue + for main_repo_submodule_file_path in self.walk_git_files(main_repo_submodule_path): + repo_submodule_file_path = path.relpath(main_repo_submodule_file_path, repo_path) # relative to repo_path - for main_repo_submodule_file_path in self.walk_git_files(main_repo_submodule_path): - repo_submodule_file_path = path.relpath(main_repo_submodule_file_path, repo_path) # relative to repo_path - if self.is_file_excluded(repo_abspath, repo_submodule_file_path): - continue + if self.is_file_excluded(repo_abspath, repo_submodule_file_path): + continue - yield main_repo_submodule_file_path - except IOError: - pass + yield main_repo_submodule_file_path finally: self._check_attr_gens[repo_abspath].close() del self._check_attr_gens[repo_abspath] @@ -538,6 +527,9 @@ def get_git_version(cls): @classmethod def list_repo_files(cls, repo_abspath): + """ + Return a list of all files as seen by git in a given repo. + """ repo_file_paths = cls.run_git_shell( 'git ls-files -z --cached --full-name --no-empty-directory', cwd=repo_abspath @@ -551,6 +543,29 @@ def list_repo_files(cls, repo_abspath): return repo_file_paths + @classmethod + def list_repo_submodules(cls, repo_abspath): + """ + Return a list of all direct submodules as seen by git in a given repo. + """ + if sys.platform.startswith('win32'): + shell_command = 'git submodule foreach --quiet "\\"{0}\\" -c \\"from __future__ import print_function; print(\'"$sm_path"\', end=chr(0))\\""' + else: + shell_command = 'git submodule foreach --quiet \'"{0}" -c "from __future__ import print_function; print(\\"$sm_path\\", end=chr(0))"\'' + + python_exe = sys.executable or 'python' + shell_command = shell_command.format(python_exe) + + repo_submodule_paths = cls.run_git_shell(shell_command, cwd=repo_abspath) + repo_submodule_paths = repo_submodule_paths.split(b'\0')[:-1] + + if sys.platform.startswith('win32'): + repo_submodule_paths = (git_fspath(p.replace(b'/', b'\\')) for p in repo_submodule_paths) + else: + repo_submodule_paths = map(git_fspath, repo_submodule_paths) + + return repo_submodule_paths + def main(argv=None): if argv is None: diff --git a/git_archive_all.pyi b/git_archive_all.pyi index 28733b9..710c1ea 100644 --- a/git_archive_all.pyi +++ b/git_archive_all.pyi @@ -64,4 +64,7 @@ class GitArchiver(object): def get_git_version(cls) -> Optional[Tuple[int]]: ... @classmethod - def list_repo_files(cls, repo_abspath: PathStr) -> Generator[PathStr, None, None]: ... \ No newline at end of file + def list_repo_files(cls, repo_abspath: PathStr) -> Generator[PathStr, None, None]: ... + + @classmethod + def list_repo_submodules(cls, repo_abspath: PathStr) -> Generator[PathStr, None, None]: ... From e063c8b67b4a0b7a7d40010a78e223077de02dc8 Mon Sep 17 00:00:00 2001 From: Ilya Kulakov Date: Sat, 5 Dec 2020 01:21:33 +0600 Subject: [PATCH 2/2] Fix errors in tests on Windows. --- appveyor-requirements.txt | 4 +- test_git_archive_all.py | 169 +++++++++++++++++++------------------- travis-requirements.txt | 4 +- 3 files changed, 90 insertions(+), 87 deletions(-) diff --git a/appveyor-requirements.txt b/appveyor-requirements.txt index e3e2f77..61faa3d 100644 --- a/appveyor-requirements.txt +++ b/appveyor-requirements.txt @@ -1,3 +1,3 @@ -tox==3.14.3 -codecov==2.0.15 +tox==3.20.1 +codecov==2.1.10 tox-venv==0.4.0 diff --git a/test_git_archive_all.py b/test_git_archive_all.py index 957ee24..bcaa015 100644 --- a/test_git_archive_all.py +++ b/test_git_archive_all.py @@ -55,13 +55,13 @@ def git_env(tmpdir_factory): 'HOME': tmpdir_factory.getbasetemp().strpath } - with tmpdir_factory.getbasetemp().join('.gitconfig').open('w+') as f: + with tmpdir_factory.getbasetemp().join('.gitconfig').open('wb+') as f: f.writelines([ - '[core]\n', - 'attributesfile = {0}\n'.format(as_posix(tmpdir_factory.getbasetemp().join('.gitattributes').strpath)), - '[user]\n', - 'name = git-archive-all\n', - 'email = git-archive-all@example.com\n', + b'[core]\n', + 'attributesfile = {0}\n'.format(as_posix(tmpdir_factory.getbasetemp().join('.gitattributes').strpath)).encode(), + b'[user]\n', + b'name = git-archive-all\n', + b'email = git-archive-all@example.com\n', ]) # .gitmodules's content is dynamic and is maintained by git. @@ -69,9 +69,9 @@ def git_env(tmpdir_factory): # # If test is run with the --no-exclude CLI option (or its exclude=False API equivalent) # then the file itself is included while its content is discarded for the same reason. - with tmpdir_factory.getbasetemp().join('.gitattributes').open('w+') as f: + with tmpdir_factory.getbasetemp().join('.gitattributes').open('wb+') as f: f.writelines([ - '.gitmodules export-ignore' + b'.gitmodules export-ignore\n' ]) return e @@ -116,7 +116,7 @@ def add(self, rel_path, record): def add_file(self, rel_path, contents): file_path = os_path_join(self.path, rel_path) - with open(file_path, 'w') as f: + with open(file_path, 'wb') as f: f.write(contents) check_call(['git', 'add', as_posix(os.path.normpath(file_path))], cwd=self.path) @@ -176,7 +176,7 @@ def make_actual_tree(tar_file): # See the comment in git_env. if not name.endswith(fspath('.gitmodules')): - a[name] = tar_file.extractfile(m).read().decode() + a[name] = tar_file.extractfile(m).read() else: a[name] = None else: @@ -187,164 +187,164 @@ def make_actual_tree(tar_file): base = { 'app': DirRecord({ - '__init__.py': FileRecord('#Beautiful is better than ugly.'), + '__init__.py': FileRecord(b'#Beautiful is better than ugly.'), }), 'lib': SubmoduleRecord({ - '__init__.py': FileRecord('#Explicit is better than implicit.'), + '__init__.py': FileRecord(b'#Explicit is better than implicit.'), 'extra': SubmoduleRecord({ - '__init__.py': FileRecord('#Simple is better than complex.'), + '__init__.py': FileRecord(b'#Simple is better than complex.'), }) }) } base_quoted = deepcopy(base) base_quoted['data'] = DirRecord({ - '\"hello world.dat\"': FileRecord('Special cases aren\'t special enough to break the rules.'), - '\'hello world.dat\'': FileRecord('Although practicality beats purity.') + '\"hello world.dat\"': FileRecord(b'Special cases aren\'t special enough to break the rules.'), + '\'hello world.dat\'': FileRecord(b'Although practicality beats purity.') }) ignore_in_root = deepcopy(base) -ignore_in_root['.gitattributes'] = FileRecord('tests/__init__.py export-ignore') +ignore_in_root['.gitattributes'] = FileRecord(b'tests/__init__.py export-ignore') ignore_in_root['tests'] = DirRecord({ - '__init__.py': FileRecord('#Complex is better than complicated.', excluded=True) + '__init__.py': FileRecord(b'#Complex is better than complicated.', excluded=True) }) ignore_in_submodule = deepcopy(base) -ignore_in_submodule['lib']['.gitattributes'] = FileRecord('tests/__init__.py export-ignore') +ignore_in_submodule['lib']['.gitattributes'] = FileRecord(b'tests/__init__.py export-ignore') ignore_in_submodule['lib']['tests'] = DirRecord({ - '__init__.py': FileRecord('#Complex is better than complicated.', excluded=True) + '__init__.py': FileRecord(b'#Complex is better than complicated.', excluded=True) }) ignore_in_nested_submodule = deepcopy(base) -ignore_in_nested_submodule['lib']['extra']['.gitattributes'] = FileRecord('tests/__init__.py export-ignore') +ignore_in_nested_submodule['lib']['extra']['.gitattributes'] = FileRecord(b'tests/__init__.py export-ignore') ignore_in_nested_submodule['lib']['extra']['tests'] = DirRecord({ - '__init__.py': FileRecord('#Complex is better than complicated.', excluded=True) + '__init__.py': FileRecord(b'#Complex is better than complicated.', excluded=True) }) ignore_in_submodule_from_root = deepcopy(base) -ignore_in_submodule_from_root['.gitattributes'] = FileRecord('lib/tests/__init__.py export-ignore') +ignore_in_submodule_from_root['.gitattributes'] = FileRecord(b'lib/tests/__init__.py export-ignore') ignore_in_submodule_from_root['lib']['tests'] = DirRecord({ - '__init__.py': FileRecord('#Complex is better than complicated.', excluded=True) + '__init__.py': FileRecord(b'#Complex is better than complicated.', excluded=True) }) ignore_in_nested_submodule_from_root = deepcopy(base) -ignore_in_nested_submodule_from_root['.gitattributes'] = FileRecord('lib/extra/tests/__init__.py export-ignore') +ignore_in_nested_submodule_from_root['.gitattributes'] = FileRecord(b'lib/extra/tests/__init__.py export-ignore') ignore_in_nested_submodule_from_root['lib']['extra']['tests'] = DirRecord({ - '__init__.py': FileRecord('#Complex is better than complicated.', excluded=True) + '__init__.py': FileRecord(b'#Complex is better than complicated.', excluded=True) }) ignore_in_nested_submodule_from_submodule = deepcopy(base) -ignore_in_nested_submodule_from_submodule['lib']['.gitattributes'] = FileRecord('extra/tests/__init__.py export-ignore') +ignore_in_nested_submodule_from_submodule['lib']['.gitattributes'] = FileRecord(b'extra/tests/__init__.py export-ignore') ignore_in_nested_submodule_from_submodule['lib']['extra']['tests'] = DirRecord({ - '__init__.py': FileRecord('#Complex is better than complicated.', excluded=True) + '__init__.py': FileRecord(b'#Complex is better than complicated.', excluded=True) }) unset_export_ignore = deepcopy(base) -unset_export_ignore['.gitattributes'] = FileRecord('.* export-ignore\n*.htaccess -export-ignore', excluded=True) -unset_export_ignore['.a'] = FileRecord('Flat is better than nested.', excluded=True) -unset_export_ignore['.b'] = FileRecord('Sparse is better than dense.', excluded=True) -unset_export_ignore['.htaccess'] = FileRecord('Readability counts.') +unset_export_ignore['.gitattributes'] = FileRecord(b'.* export-ignore\n*.htaccess -export-ignore', excluded=True) +unset_export_ignore['.a'] = FileRecord(b'Flat is better than nested.', excluded=True) +unset_export_ignore['.b'] = FileRecord(b'Sparse is better than dense.', excluded=True) +unset_export_ignore['.htaccess'] = FileRecord(b'Readability counts.') unicode_base = deepcopy(base) unicode_base['data'] = DirRecord({ - 'مرحبا بالعالم.dat': FileRecord('Special cases aren\'t special enough to break the rules.') + 'مرحبا بالعالم.dat': FileRecord(b'Special cases aren\'t special enough to break the rules.') }) unicode_quoted = deepcopy(base) unicode_quoted['data'] = DirRecord({ - '\"مرحبا بالعالم.dat\"': FileRecord('Special cases aren\'t special enough to break the rules.'), - '\'привет мир.dat\'': FileRecord('Although practicality beats purity.') + '\"مرحبا بالعالم.dat\"': FileRecord(b'Special cases aren\'t special enough to break the rules.'), + '\'привет мир.dat\'': FileRecord(b'Although practicality beats purity.') }) brackets_base = deepcopy(base) brackets_base['data'] = DirRecord({ - '[.dat': FileRecord('Special cases aren\'t special enough to break the rules.'), - '(.dat': FileRecord('Although practicality beats purity.'), - '{.dat': FileRecord('Errors should never pass silently.'), - '].dat': FileRecord('Unless explicitly silenced.'), - ').dat': FileRecord('In the face of ambiguity, refuse the temptation to guess.'), - '}.dat': FileRecord('There should be one-- and preferably only one --obvious way to do it.'), - '[].dat': FileRecord('Although that way may not be obvious at first unless you\'re Dutch.'), - '().dat': FileRecord('Now is better than never.'), - '{}.dat': FileRecord('Although never is often better than *right* now.'), + '[.dat': FileRecord(b'Special cases aren\'t special enough to break the rules.'), + '(.dat': FileRecord(b'Although practicality beats purity.'), + '{.dat': FileRecord(b'Errors should never pass silently.'), + '].dat': FileRecord(b'Unless explicitly silenced.'), + ').dat': FileRecord(b'In the face of ambiguity, refuse the temptation to guess.'), + '}.dat': FileRecord(b'There should be one-- and preferably only one --obvious way to do it.'), + '[].dat': FileRecord(b'Although that way may not be obvious at first unless you\'re Dutch.'), + '().dat': FileRecord(b'Now is better than never.'), + '{}.dat': FileRecord(b'Although never is often better than *right* now.'), }) brackets_quoted = deepcopy(base) brackets_quoted['data'] = DirRecord({ - '\"[.dat\"': FileRecord('Special cases aren\'t special enough to break the rules.'), - '\'[.dat\'': FileRecord('Special cases aren\'t special enough to break the rules.'), - '\"(.dat\"': FileRecord('Although practicality beats purity.'), - '\'(.dat\'': FileRecord('Although practicality beats purity.'), - '\"{.dat\"': FileRecord('Errors should never pass silently.'), - '\'{.dat\'': FileRecord('Errors should never pass silently.'), - '\"].dat\"': FileRecord('Unless explicitly silenced.'), - '\'].dat\'': FileRecord('Unless explicitly silenced.'), - '\").dat\"': FileRecord('In the face of ambiguity, refuse the temptation to guess.'), - '\').dat\'': FileRecord('In the face of ambiguity, refuse the temptation to guess.'), - '\"}.dat\"': FileRecord('There should be one-- and preferably only one --obvious way to do it.'), - '\'}.dat\'': FileRecord('There should be one-- and preferably only one --obvious way to do it.'), - '\"[].dat\"': FileRecord('Although that way may not be obvious at first unless you\'re Dutch.'), - '\'[].dat\'': FileRecord('Although that way may not be obvious at first unless you\'re Dutch.'), - '\"().dat\"': FileRecord('Now is better than never.'), - '\'().dat\'': FileRecord('Now is better than never.'), - '\"{}.dat\"': FileRecord('Although never is often better than *right* now.'), - '\'{}.dat\'': FileRecord('Although never is often better than *right* now.'), + '\"[.dat\"': FileRecord(b'Special cases aren\'t special enough to break the rules.'), + '\'[.dat\'': FileRecord(b'Special cases aren\'t special enough to break the rules.'), + '\"(.dat\"': FileRecord(b'Although practicality beats purity.'), + '\'(.dat\'': FileRecord(b'Although practicality beats purity.'), + '\"{.dat\"': FileRecord(b'Errors should never pass silently.'), + '\'{.dat\'': FileRecord(b'Errors should never pass silently.'), + '\"].dat\"': FileRecord(b'Unless explicitly silenced.'), + '\'].dat\'': FileRecord(b'Unless explicitly silenced.'), + '\").dat\"': FileRecord(b'In the face of ambiguity, refuse the temptation to guess.'), + '\').dat\'': FileRecord(b'In the face of ambiguity, refuse the temptation to guess.'), + '\"}.dat\"': FileRecord(b'There should be one-- and preferably only one --obvious way to do it.'), + '\'}.dat\'': FileRecord(b'There should be one-- and preferably only one --obvious way to do it.'), + '\"[].dat\"': FileRecord(b'Although that way may not be obvious at first unless you\'re Dutch.'), + '\'[].dat\'': FileRecord(b'Although that way may not be obvious at first unless you\'re Dutch.'), + '\"().dat\"': FileRecord(b'Now is better than never.'), + '\'().dat\'': FileRecord(b'Now is better than never.'), + '\"{}.dat\"': FileRecord(b'Although never is often better than *right* now.'), + '\'{}.dat\'': FileRecord(b'Although never is often better than *right* now.'), }) quote_base = deepcopy(base) quote_base['data'] = DirRecord({ - '\'.dat': FileRecord('Special cases aren\'t special enough to break the rules.'), - '\".dat': FileRecord('Although practicality beats purity.'), + '\'.dat': FileRecord(b'Special cases aren\'t special enough to break the rules.'), + '\".dat': FileRecord(b'Although practicality beats purity.'), }) quote_quoted = deepcopy(base) quote_quoted['data'] = DirRecord({ - '\"\'.dat\"': FileRecord('Special cases aren\'t special enough to break the rules.'), - '\'\'.dat\'': FileRecord('Special cases aren\'t special enough to break the rules.'), - '\"\".dat\"': FileRecord('Although practicality beats purity.'), - '\'\".dat\'': FileRecord('Although practicality beats purity.'), + '\"\'.dat\"': FileRecord(b'Special cases aren\'t special enough to break the rules.'), + '\'\'.dat\'': FileRecord(b'Special cases aren\'t special enough to break the rules.'), + '\"\".dat\"': FileRecord(b'Although practicality beats purity.'), + '\'\".dat\'': FileRecord(b'Although practicality beats purity.'), }) nonunicode_base = deepcopy(base) nonunicode_base['data'] = DirRecord({ - b'test.\xc2': FileRecord('Special cases aren\'t special enough to break the rules.'), + b'test.\xc2': FileRecord(b'Special cases aren\'t special enough to break the rules.'), }) nonunicode_quoted = deepcopy(base) nonunicode_quoted['data'] = DirRecord({ - b'\'test.\xc2\'': FileRecord('Special cases aren\'t special enough to break the rules.'), - b'\"test.\xc2\"': FileRecord('Although practicality beats purity.'), + b'\'test.\xc2\'': FileRecord(b'Special cases aren\'t special enough to break the rules.'), + b'\"test.\xc2\"': FileRecord(b'Although practicality beats purity.'), }) backslash_base = deepcopy(base) backslash_base['data'] = DirRecord({ - '\\.dat': FileRecord('Special cases aren\'t special enough to break the rules.'), + '\\.dat': FileRecord(b'Special cases aren\'t special enough to break the rules.'), }) backslash_quoted = deepcopy(base) backslash_quoted['data'] = DirRecord({ - '\'\\.dat\'': FileRecord('Special cases aren\'t special enough to break the rules.'), - '\"\\.dat\"': FileRecord('Although practicality beats purity.') + '\'\\.dat\'': FileRecord(b'Special cases aren\'t special enough to break the rules.'), + '\"\\.dat\"': FileRecord(b'Although practicality beats purity.') }) non_unicode_backslash_base = deepcopy(base) non_unicode_backslash_base['data'] = DirRecord({ - b'\\\xc2.dat': FileRecord('Special cases aren\'t special enough to break the rules.'), + b'\\\xc2.dat': FileRecord(b'Special cases aren\'t special enough to break the rules.'), }) non_unicode_backslash_quoted = deepcopy(base) non_unicode_backslash_quoted['data'] = DirRecord({ - b'\'\\\xc2.dat\'': FileRecord('Special cases aren\'t special enough to break the rules.'), - b'\"\\\xc2.dat\"': FileRecord('Although practicality beats purity.') + b'\'\\\xc2.dat\'': FileRecord(b'Special cases aren\'t special enough to break the rules.'), + b'\"\\\xc2.dat\"': FileRecord(b'Although practicality beats purity.') }) ignore_dir = { - '.gitattributes': FileRecord('.gitattributes export-ignore\n**/src export-ignore\ndata/src/__main__.py -export-ignore', excluded=True), - '__init__.py': FileRecord('#Beautiful is better than ugly.'), + '.gitattributes': FileRecord(b'.gitattributes export-ignore\n**/src export-ignore\ndata/src/__main__.py -export-ignore', excluded=True), + '__init__.py': FileRecord(b'#Beautiful is better than ugly.'), 'data': DirRecord({ 'src': DirRecord({ - '__init__.py': FileRecord('#Explicit is better than implicit.', excluded=True), - '__main__.py': FileRecord('#Simple is better than complex.') + '__init__.py': FileRecord(b'#Explicit is better than implicit.', excluded=True), + '__main__.py': FileRecord(b'#Simple is better than complex.') }) }) } @@ -385,7 +385,6 @@ def test_ignore(contents, exclude, tmpdir, git_env, monkeypatch): """ Ensure that GitArchiver respects export-ignore. """ - # On Python 2.7 contained code raises pytest.PytestWarning warning for no good reason. with warnings.catch_warnings(): warnings.simplefilter("ignore") @@ -412,8 +411,12 @@ def test_ignore(contents, exclude, tmpdir, git_env, monkeypatch): def test_cli(tmpdir, git_env, monkeypatch): contents = base - for name, value in git_env.items(): - monkeypatch.setenv(name, value) + # On Python 2.7 contained code raises pytest.PytestWarning warning for no good reason. + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + + for name, value in git_env.items(): + monkeypatch.setenv(name, value) repo_path = os_path_join(tmpdir.strpath, 'repo') repo = Repo(repo_path) diff --git a/travis-requirements.txt b/travis-requirements.txt index 9d50b2f..676401c 100644 --- a/travis-requirements.txt +++ b/travis-requirements.txt @@ -1,4 +1,4 @@ -tox==3.14.3 -codecov==2.0.15 +tox==3.20.1 +codecov==2.1.10 tox-pyenv==1.1.0 virtualenv==15.2.0