Some code clean-up. (#39)

* Moved utils.py from client to package-root.

Also merged 2 functions that did exactly the same thing.

* Removed unused function duplicate and renamed utils to not be private.

Author: radeusgd

filetracker/__init__.py

@@ -113,4 +113,4 @@
 # use new paths.
 
 from filetracker.client import Client, dummy
-from filetracker.client.utils import split_name
+from filetracker.utils import split_name

filetracker/client/client.py

@@ -12,7 +12,7 @@
 from filetracker.client.local_data_store import LocalDataStore
 from filetracker.client.lock_manager import FcntlLockManager, NoOpLockManager
 from filetracker.client.remote_data_store import RemoteDataStore
-from filetracker.client.utils import split_name, versioned_name, _check_name
+from filetracker.utils import split_name, versioned_name, check_name
 
 logger = logging.getLogger('filetracker')
 
@@ -272,7 +272,7 @@ def put_file(self,
             raise ValueError("Neither to_local_store nor to_remote_store set "
                              "in a call to filetracker.Client.put_file")
 
-        _check_name(name)
+        check_name(name)
 
         lock = None
         if self.local_store:

filetracker/client/data_store.py

@@ -4,7 +4,7 @@
 import os
 import shutil
 
-from filetracker.client.utils import split_name, _mkdir
+from filetracker.utils import split_name, mkdir
 
 
 class DataStore(object):
@@ -90,7 +90,7 @@ def get_file(self, name, filename):
 
         dir_path = os.path.dirname(filename)
         if dir_path:
-            _mkdir(dir_path)
+            mkdir(dir_path)
 
         with open(filename, 'wb') as f:
             shutil.copyfileobj(stream, f)

filetracker/client/dummy.py

@@ -11,7 +11,7 @@
 
 from filetracker.client import Client
 from filetracker.client.data_store import DataStore
-from filetracker.client.utils import split_name, versioned_name, _check_name
+from filetracker.utils import split_name, versioned_name, check_name
 
 
 class DummyDataStore(DataStore):
@@ -26,7 +26,7 @@ def __init__(self):
         self.versions = defaultdict(int)
 
     def _parse_name(self, name):
-        _check_name(name)
+        check_name(name)
         key, version = split_name(name)
         return key, version
 

filetracker/client/local_data_store.py

@@ -5,7 +5,7 @@
 
 from filetracker.client import FiletrackerError
 from filetracker.client.data_store import DataStore
-from filetracker.client.utils import split_name, versioned_name, _check_name, _mkdir
+from filetracker.utils import split_name, versioned_name, check_name, mkdir
 
 
 class LocalDataStore(DataStore):
@@ -17,10 +17,10 @@ class LocalDataStore(DataStore):
 
     def __init__(self, dir):
         self.dir = os.path.join(dir, 'files')
-        _mkdir(self.dir)
+        mkdir(self.dir)
 
     def _parse_name(self, name):
-        _check_name(name)
+        check_name(name)
         name, version = split_name(name)
         path = self.dir + name
         return path, version
@@ -96,7 +96,7 @@ def list_files(self):
 def _save_stream(path, stream, version=None):
     dir = os.path.dirname(path)
     if dir:
-        _mkdir(dir)
+        mkdir(dir)
     if os.path.exists(path) and version is not None \
             and _file_version(path) >= version:
         return version

filetracker/client/lock_manager.py

@@ -3,7 +3,7 @@
 import fcntl
 import os
 
-from filetracker.client.utils import split_name, _check_name, _mkdir
+from filetracker.utils import split_name, check_name, mkdir
 
 
 class LockManager(object):
@@ -76,14 +76,14 @@ def __del__(self):
 
     def __init__(self, dir):
         self.dir = dir
-        _mkdir(dir)
+        mkdir(dir)
 
     def lock_for(self, name):
-        _check_name(name)
+        check_name(name)
         name, version = split_name(name)
         path = self.dir + name
         dir = os.path.dirname(path)
-        _mkdir(dir)
+        mkdir(dir)
         return self.FcntlLock(path)
 
 
@@ -105,5 +105,5 @@ def unlock(self):
             pass
 
     def lock_for(self, name):
-        _check_name(name)
+        check_name(name)
         return self.NoOpLock()

filetracker/client/remote_data_store.py

@@ -14,8 +14,8 @@
 
 from filetracker.client import FiletrackerError
 from filetracker.client.data_store import DataStore
-from filetracker.client.utils import (split_name, versioned_name, _check_name,
-                                      _compute_checksum)
+from filetracker.utils import (split_name, versioned_name, check_name,
+                               file_digest)
 
 logger = logging.getLogger('filetracker')
 
@@ -57,7 +57,7 @@ def __init__(self, base_url):
         self.base_url = base_url
 
     def _parse_name(self, name):
-        _check_name(name)
+        check_name(name)
         name, version = split_name(name)
         url = self.base_url + pathname2url(name)
         return url, version
@@ -90,7 +90,7 @@ def _put_file(self, url, version, f, headers):
     def add_file(self, name, filename, compress_hint=True):
         url, version = self._parse_name(name)
 
-        sha = _compute_checksum(filename)
+        sha = file_digest(filename)
 
         headers = {
             'SHA256-Checksum': sha

filetracker/client/shell.py

@@ -143,5 +143,6 @@ def main():
     client = Client(remote_url=options.remote_url, cache_dir=options.cache_dir)
     cmd(client, *args[1:])
 
+
 if __name__ == '__main__':
     main()

filetracker/servers/files.py

@@ -126,25 +126,5 @@ def handle_DELETE(self, environ, start_response):
         return [b'OK']
 
 
-_BUFFER_SIZE = 64 * 1024
-
-
-def _copy_stream(src, dest, length):
-    """Similar to shutil.copyfileobj, but supports limiting data size.
-
-    As for why this is required, refer to
-    https://www.python.org/dev/peps/pep-0333/#input-and-error-streams
-
-    Yes, there are WSGI implementations which do not support EOFs, and
-    believe me, you don't want to debug this.
-    """
-    bytes_left = length
-    while bytes_left > 0:
-        buf_size = min(_BUFFER_SIZE, bytes_left)
-        buf = src.read(buf_size)
-        dest.write(buf)
-        bytes_left -= buf_size
-
-
 if __name__ == '__main__':
     base.main(FiletrackerServer())

filetracker/servers/storage.py

@@ -29,7 +29,6 @@
 import email.utils
 import fcntl
 import gzip
-import hashlib
 import os
 import shutil
 import subprocess
@@ -38,6 +37,8 @@
 import bsddb3
 import six
 
+from filetracker.utils import file_digest
+
 
 class FiletrackerFileNotFoundError(Exception):
     pass
@@ -127,9 +128,9 @@ def store(self, name, data, version, size=0, compressed=False, digest=None):
                     # If data was already compressed, we have to decompress it
                     # before calculating the digest.
                     with gzip.open(temp_file_path, 'rb') as compressed_file:
-                        digest = _file_digest(compressed_file)
+                        digest = file_digest(compressed_file)
                 else:
-                    digest = _file_digest(temp_file_path)
+                    digest = file_digest(temp_file_path)
 
             blob_path = self._blob_path(digest)
 
@@ -303,29 +304,6 @@ def _path_exists(path):
     return os.path.exists(path) or os.path.islink(path)
 
 
-def _file_digest(source):
-    """Calculates SHA256 digest of a file.
-
-    Args:
-        source: either a file-like object or a path to file
-    """
-    hash_sha256 = hashlib.sha256()
-
-    should_close = False
-
-    if isinstance(source, six.string_types):
-        should_close = True
-        source = open(source, 'rb')
-
-    for chunk in iter(lambda: source.read(_BUFFER_SIZE), b''):
-        hash_sha256.update(chunk)
-
-    if should_close:
-        source.close()
-
-    return hash_sha256.hexdigest()
-
-
 def _file_version(path):
     return os.lstat(path).st_mtime