Merge pull request #69 from swinkels/on-update-copy-changed-files-onl…

…y-pr

On update, only copy changed files

CHANGES.md

@@ -4,6 +4,10 @@ Changelog for git-cc
 1.0.1 (unreleased)
 ------------------
 
+- Various fixes to work towards Python 3 compatibility.
+- Various improvements to the update command:
+  - only copies files that have changed;
+  - copied files have the same date and time as the original files.
 - Supports [tox] [tox] to package and run tests in virtualenvs.
 
 [tox]: http://tox.readthedocs.io/en/latest/

git_cc/cache.py

@@ -1,5 +1,5 @@
 from os.path import join, exists
-from common import *
+from .common import *
 
 FILE = '.gitcc'
 

git_cc/checkin.py

@@ -1,12 +1,13 @@
 """Checkin new git changesets to Clearcase"""
 
-from common import *
-from clearcase import cc
-from status import Modify, Add, Delete, Rename, SymLink
+from .common import *
+from .clearcase import cc
+from .status import Modify, Add, Delete, Rename, SymLink
 import filecmp
 from os import listdir
 from os.path import isdir
-import cache, reset
+from . import cache
+from . import reset
 
 IGNORE_CONFLICTS=False
 LOG_FORMAT = '%H%x01%B'

git_cc/clearcase.py

@@ -1,4 +1,4 @@
-from common import *
+from .common import *
 
 class Clearcase:
     def rebase(self):

git_cc/gitcc.py

@@ -4,14 +4,14 @@
 
 from optparse import OptionParser
 
-import checkin
-import init
-import rebase
-import reset
-import sync
-import tag
-import update
-import version
+from . import checkin
+from . import init
+from . import rebase
+from . import reset
+from . import sync
+from . import tag
+from . import update
+from . import version
 
 commands = [
     init, rebase, checkin, sync, reset, tag, update, version

git_cc/init.py

@@ -1,6 +1,6 @@
 """Initialise gitcc with a clearcase directory"""
 
-from common import *
+from .common import *
 from os import open
 from os.path import join, exists
 

git_cc/rebase.py

@@ -2,11 +2,11 @@
 
 from os.path import join, dirname, exists, isdir
 import os, stat
-from common import *
+from .common import *
 from datetime import datetime, timedelta
 from fnmatch import fnmatch
-from clearcase import cc
-from cache import getCache, CCFile
+from .clearcase import cc
+from .cache import getCache, CCFile
 from re import search
 
 """

git_cc/reset.py

@@ -1,6 +1,6 @@
 """Reset hard to a specific changeset"""
 
-from common import *
+from .common import *
 
 def main(commit):
     git_exec(['branch', '-f', CC_TAG, commit])

git_cc/status.py

@@ -1,4 +1,4 @@
-from common import *
+from .common import *
 from os.path import join, dirname
 
 class Status:

git_cc/sync.py

@@ -1,7 +1,9 @@
 """Copy files from Clearcase to Git manually"""
 
-from common import *
-from cache import *
+import filecmp
+
+from .common import *
+from .cache import *
 import os, shutil, stat
 from os.path import join, abspath, isdir
 from fnmatch import fnmatch
@@ -16,32 +18,45 @@ def main(cache=False):
         return syncCache()
     glob = '*'
     base = abspath(CC_DIR)
+    copied_file_count = 0
     for i in cfg.getInclude():
         for (dirpath, dirnames, filenames) in os.walk(join(CC_DIR, i)):
             reldir = dirpath[len(base)+1:]
             if fnmatch(reldir, './lost+found'):
                 continue
             for file in filenames:
                 if fnmatch(file, glob):
-                    copy(join(reldir, file))
+                    if copy(join(reldir, file)):
+                        copied_file_count += 1
+    return copied_file_count
+
 
-def copy(file):
-    newFile = join(GIT_DIR, file)
-    debug('Copying %s' % newFile)
-    mkdirs(newFile)
-    shutil.copy(join(CC_DIR, file), newFile)
-    os.chmod(newFile, stat.S_IREAD | stat.S_IWRITE)
+def copy(file, src_dir=CC_DIR, dst_dir=GIT_DIR):
+    src_file = join(src_dir, file)
+    dst_file = join(dst_dir, file)
+    skip_file = os.path.exists(dst_file) and \
+        filecmp.cmp(src_file, dst_file, shallow=False)
+    if not skip_file:
+        debug('Copying to %s' % dst_file)
+        mkdirs(dst_file)
+        shutil.copy2(src_file, dst_file)
+        os.chmod(dst_file, stat.S_IREAD | stat.S_IWRITE)
+        return True
+    return False
 
 def syncCache():
     cache1 = Cache(GIT_DIR)
     cache1.start()
-    
+
     cache2 = Cache(GIT_DIR)
     cache2.initial()
-
+
+    copied_file_count = 0
     for path in cache2.list():
         if not cache1.contains(path):
             cache1.update(path)
             if not isdir(join(CC_DIR, path.file)):
-                copy(path.file)
+                if copy(path.file):
+                    copied_file_count += 1
     cache1.write()
+    return copied_file_count

git_cc/tag.py

@@ -1,6 +1,6 @@
 """Tag a particular commit as gitcc start point"""
 
-from common import *
+from .common import *
 
 def main(commit):
     tag(CI_TAG, commit)

git_cc/update.py

@@ -1,11 +1,17 @@
 """Update the git repository with Clearcase manually, ignoring history"""
 
-from common import *
-import sync, reset
+from __future__ import print_function
+
+from .common import *
+from . import reset
+from . import sync
+
 
 def main(message):
     cc_exec(['update', '.'], errors=False)
-    sync.main()
-    git_exec(['add', '.'])
-    git_exec(['commit', '-m', message])
-    reset.main('HEAD')
+    if sync.main():
+        git_exec(['add', '.'])
+        git_exec(['commit', '-m', message])
+        reset.main('HEAD')
+    else:
+        print("No files have changed, nothing to commit.")

tests/copy-data/a.txt

@@ -0,0 +1 @@
+This is a document about the letter "a".

tests/test_imports.py

@@ -0,0 +1,54 @@
+import os
+import imp
+import pkgutil
+import sys
+import unittest
+
+
+class ImportTestSuite(unittest.TestCase):
+
+    def test_import_of_each_module(self):
+        """Import each module of package git_cc.
+
+        This test was added because the import Python 3 requires you to use the
+        relative package import syntax "from . import <module name>" to import
+        packages from the same package, whereas Python 2 you could just use
+        "import <module name>".
+
+        """
+        import git_cc
+        package_dir = os.path.dirname(os.path.abspath(git_cc.__file__))
+        for _, module_name, _ in pkgutil.iter_modules([package_dir]):
+            self.try_module_import(package_dir, module_name)
+
+    def try_module_import(self, package_dir, module_name):
+        """Try to import the given module from the given directory.
+
+        If the import fails, this method throws an exception.
+
+        Note that when the import was successful, this method immediately
+        removes it from the list of loaded modules.
+
+        """
+
+        module_path = os.path.join(package_dir, module_name) + ".py"
+
+        # The call to imp.load_source fails when it indirectly imports a
+        # module that was already loaded by a previous call to
+        # imp.load_source under the same name.
+        #
+        # To give an example, say you have the following pseudo-code:
+        #
+        #    imp.load_source("package.module-a", ... )
+        #    imp.load_source("package.module-b", ... )
+        #
+        # and module-b has an import of module a, then the second call
+        # throws an ImportError with the message that it cannot import name
+        # "module-b".
+        #
+        # To avoid this issue, we remove the module from the list of imported
+        # modules once the import has succeeded.
+
+        full_module_name = "git_cc." + module_name
+        imp.load_source(full_module_name, module_path)
+        del sys.modules[full_module_name]

tests/test_sync.py

@@ -0,0 +1,101 @@
+import filecmp
+import os
+import shutil
+import stat
+import unittest
+
+
+from git_cc.sync import copy
+
+
+class SyncTestSuite(unittest.TestCase):
+
+    def setUp(self):
+
+        self.clear_filecmp_cache()
+
+        current_dir = os.path.dirname(os.path.abspath(__file__))
+        self.src_dir = os.path.join(current_dir, "copy-data")
+        self.dst_dir = os.path.join(current_dir, "sandbox")
+
+        if os.path.exists(self.dst_dir):
+            shutil.rmtree(self.dst_dir)
+
+        os.mkdir(self.dst_dir)
+
+    def tearDown(self):
+        shutil.rmtree(self.dst_dir)
+
+    def test_copy_creates_new_file(self):
+
+        fileName = "a.txt"
+
+        copyIsDone = copy(fileName, src_dir=self.src_dir, dst_dir=self.dst_dir)
+        self.assertTrue(copyIsDone)
+        src_path = os.path.join(self.src_dir, fileName)
+        dst_path = os.path.join(self.dst_dir, fileName)
+        self.files_are_equal(src_path, dst_path)
+
+    def test_copy_overwrites_existing_different_file(self):
+
+        fileName = "a.txt"
+
+        src_path = os.path.join(self.src_dir, fileName)
+        with open(src_path, "r") as f:
+            lines = f.readlines()
+        lines[0] = lines[0].replace('e', 'f')
+
+        dst_path = os.path.join(self.dst_dir, fileName)
+        with open(dst_path, "w") as f:
+            f.writelines(lines)
+
+        # to make it more difficult, we give the destination file the same
+        # file stats
+        shutil.copystat(src_path, dst_path)
+
+        copyIsDone = copy(fileName, src_dir=self.src_dir, dst_dir=self.dst_dir)
+        self.assertTrue(copyIsDone)
+        self.files_are_equal(src_path, dst_path)
+
+    def test_copy_does_not_overwrite_equal_file(self):
+
+        fileName = "a.txt"
+
+        src_path = os.path.join(self.src_dir, fileName)
+        dst_path = os.path.join(self.dst_dir, fileName)
+
+        shutil.copyfile(src_path, dst_path)
+        self.assertTrue(os.path.exists(dst_path))
+
+        # We make the destination file read-only. If the copy statement throws
+        # an exception, it did not recognize that the destination file was the
+        # same and tried to copy it.
+        os.chmod(dst_path, stat.S_IREAD)
+
+        copyIsDone = copy(fileName, src_dir=self.src_dir, dst_dir=self.dst_dir)
+        self.assertFalse(copyIsDone)
+
+    def files_are_equal(self, src_path, dst_path):
+
+        self.clear_filecmp_cache()
+
+        self.assertTrue(filecmp.cmp(src_path, dst_path))
+
+        src_stats = os.stat(src_path)
+        dst_stats = os.stat(dst_path)
+        self.assertAlmostEqual(src_stats.st_mtime, dst_stats.st_mtime,
+                               places=2)
+
+    def clear_filecmp_cache(self):
+        """Clear the cache of module filecmp to trigger new file comparisons.
+
+        Module filecmp keeps a cache of file comparisons so it does not have to
+        recompare files whose stats have not changed. This disrupts tests in
+        this suite which compare files with the same paths and stats but with
+        different contents. For that reason, we can clear the cache.
+
+        Do note that this function uses an internal variable of filecmp and can
+        break when that variable is removed, renamed etc.
+
+        """
+        filecmp._cache = {}