Fix reading attributes on Git < 1.8.5

Fixes #65

CHANGES.rst

@@ -1,6 +1,13 @@
 CHANGES
 =======
 
+1.19.3 (2018-xx-yy)
+-------------------
+
+- Add the git_version parameter to GitArchiver and the get_git_version class method
+- If git version (initialized or guessed) is less than 1.6.1, exception is raised
+- Properly read non-nul separated output of check-attr if git version is less than 1.8.5. See #65
+
 1.19.2 (2018-11-13)
 -------------------
 

git_archive_all.py

@@ -76,7 +76,7 @@ class GitArchiver(object):
 
     LOG = logging.getLogger('GitArchiver')
 
-    def __init__(self, prefix='', exclude=True, force_sub=False, extra=None, main_repo_abspath=None):
+    def __init__(self, prefix='', exclude=True, force_sub=False, extra=None, main_repo_abspath=None, git_version=None):
         """
         @param prefix: Prefix used to prepend all paths in the resulting archive.
             Extra file paths are only prefixed if they are not relative.
@@ -101,7 +101,17 @@ def __init__(self, prefix='', exclude=True, force_sub=False, extra=None, main_re
             with abspath to top-level directory of the repository.
             If None, current cwd is used.
         @type main_repo_abspath: str
+
+        @param git_version: Version of Git that determines whether various workarounds are on.
+            If None, tries to resolve via Git's CLI.
+        @type git_version: tuple or None
         """
+        if git_version is None:
+            git_version = self.get_git_version()
+
+        if git_version is not None and git_version < (1, 6, 1):
+            raise ValueError("git of version 1.6.1 and higher is required")
+
         if extra is None:
             extra = []
 
@@ -120,6 +130,7 @@ def __init__(self, prefix='', exclude=True, force_sub=False, extra=None, main_re
         self.extra = extra
         self.force_sub = force_sub
         self.main_repo_abspath = main_repo_abspath
+        self.git_version = git_version
 
         self._check_attr_gens = {}
 
@@ -298,47 +309,7 @@ def walk_git_files(self, repo_path=''):
             self._check_attr_gens[repo_abspath].close()
             del self._check_attr_gens[repo_abspath]
 
-    @classmethod
-    def decode_git_output(cls, output):
-        """
-        Decode Git's binary output handeling the way it escapes unicode characters.
-
-        @type output: bytes
-
-        @rtype: str
-        """
-        return output.decode('unicode_escape').encode('raw_unicode_escape').decode('utf-8')
-
-    @classmethod
-    def run_git_shell(cls, cmd, cwd=None):
-        """
-        Runs git shell command, reads output and decodes it into unicode string.
-
-        @param cmd: Command to be executed.
-        @type cmd: str
-
-        @type cwd: str
-        @param cwd: Working directory.
-
-        @rtype: str
-        @return: Output of the command.
-
-        @raise CalledProcessError:  Raises exception if return code of the command is non-zero.
-        """
-        p = Popen(cmd, shell=True, stdout=PIPE, cwd=cwd)
-        output, _ = p.communicate()
-        output = cls.decode_git_output(output)
-
-        if p.returncode:
-            if sys.version_info > (2, 6):
-                raise CalledProcessError(returncode=p.returncode, cmd=cmd, output=output)
-            else:
-                raise CalledProcessError(returncode=p.returncode, cmd=cmd)
-
-        return output
-
-    @classmethod
-    def check_attr(cls, repo_abspath, attrs):
+    def check_attr(self, repo_abspath, attrs):
         """
         Generator that returns attributes for given paths relative to repo_abspath.
 
@@ -378,7 +349,7 @@ def read_attrs(process, repo_file_path):
                     nuls_count += 1
 
                     if nuls_count % 3 == 0:
-                        yield map(cls.decode_git_output, (path, attr, info))
+                        yield map(self.decode_git_output, (path, attr, info))
                         path, attr, info = b'', b'', b''
                 elif nuls_count % 3 == 0:
                     path += b
@@ -387,6 +358,38 @@ def read_attrs(process, repo_file_path):
                 elif nuls_count % 3 == 2:
                     info += b
 
+        def read_attrs_old(process, repo_file_path):
+            """
+            Compatibility with versions 1.8.5 and below that do not recognize -z for output.
+            """
+            process.stdin.write(repo_file_path.encode('utf-8') + b'\0')
+            process.stdin.flush()
+
+            # For every attribute check-attr will output: <path>: <attribute>: <info>\n
+            # where <path> is c-quoted
+
+            path, attr, info = b'', b'', b''
+            lines_count = 0
+            lines_expected = len(attrs)
+
+            while lines_count != lines_expected:
+                line = process.stdout.readline()
+
+                info_start = line.rfind(b': ')
+                if info_start == -1:
+                    raise RuntimeError("unexpected output of check-attr: {0}".format(line))
+
+                attr_start = line.rfind(b': ', 0, info_start)
+                if attr_start == -1:
+                    raise RuntimeError("unexpected output of check-attr: {0}".format(line))
+
+                info = line[info_start + 2:len(line) - 1]  # trim leading ": " and trailing \n
+                attr = line[attr_start + 2:info_start]  # trim leading ": "
+                path = line[:attr_start]
+
+                yield map(self.decode_git_output, (path, attr, info))
+                lines_count += 1
+
         if not attrs:
             return
 
@@ -397,14 +400,85 @@ def read_attrs(process, repo_file_path):
                 repo_file_path = yield
                 repo_file_attrs = {}
 
-                for path, attr, value in read_attrs(process, repo_file_path):
+                if self.git_version is None or self.git_version > (1, 8, 5):
+                    reader = read_attrs
+                else:
+                    reader = read_attrs_old
+
+                for path, attr, value in reader(process, repo_file_path):
                     repo_file_attrs[attr] = value
 
                 yield repo_file_attrs
         finally:
             process.stdin.close()
             process.wait()
 
+    @classmethod
+    def decode_git_output(cls, output):
+        """
+        Decode Git's binary output handeling the way it escapes unicode characters.
+
+        @type output: bytes
+
+        @rtype: str
+        """
+        return output.decode('unicode_escape').encode('raw_unicode_escape').decode('utf-8')
+
+    @classmethod
+    def run_git_shell(cls, cmd, cwd=None):
+        """
+        Runs git shell command, reads output and decodes it into unicode string.
+
+        @param cmd: Command to be executed.
+        @type cmd: str
+
+        @type cwd: str
+        @param cwd: Working directory.
+
+        @rtype: str
+        @return: Output of the command.
+
+        @raise CalledProcessError:  Raises exception if return code of the command is non-zero.
+        """
+        p = Popen(cmd, shell=True, stdout=PIPE, cwd=cwd)
+        output, _ = p.communicate()
+        output = cls.decode_git_output(output)
+
+        if p.returncode:
+            if sys.version_info > (2, 6):
+                raise CalledProcessError(returncode=p.returncode, cmd=cmd, output=output)
+            else:
+                raise CalledProcessError(returncode=p.returncode, cmd=cmd)
+
+        return output
+
+    @classmethod
+    def get_git_version(cls):
+        """
+        Return version of git current shell points to.
+
+        If version cannot be parsed None is returned.
+
+        @rtype: tuple or None
+        """
+        try:
+            output = cls.run_git_shell('git version')
+        except CalledProcessError:
+            cls.LOG.warning("Unable to get Git version.")
+            return None
+
+        try:
+            version = output.split()[-1]
+        except IndexError:
+            cls.LOG.warning("Unable to parse Git version \"%s\".", output)
+            return None
+
+        try:
+            return tuple(int(v) for v in version.split('.'))
+        except ValueError:
+            cls.LOG.warning("Unable to parse Git version \"%s\".", version)
+            return None
+
 
 def main():
     from optparse import OptionParser, SUPPRESS_HELP