Separated server into /files and /list endpoints. (#40)

* Separated server into /files and /list endpoints.

Also implemented /list endpoint and added tests.

* Increased wait time for server in migration tests.

* Refactored URL handling code.

Author: kodek16

MIGRATING.md

@@ -45,16 +45,17 @@ it's more involved.
 You should have read and become comfortable with the section above, this section only
 describes the differences.
 
-The trick is based on the new `--fallback-url` parameter for the filetracker server. After upgrading the filetracker server, you should configure two servers: the main server which
+The trick is based on the new `--fallback-url` parameter for the filetracker server.
+After upgrading the filetracker server, you should configure two servers: the main server which
 uses the new filetracker code, and the fallback server which is plain lighttpd.
 
 Before starting the new server and the rest of the SIO2 infrastructure, start a lighttpd
-server with a simple configuration for serving static files from `ft_root/files`
+server with a simple configuration for serving static files from `ft_root`
 (port and log path are arbitrary):
 
 ```
 server.tag = "filetracker-old"
-server.document-root = "/path/to/ft_root/files"
+server.document-root = "/path/to/ft_root"
 server.port = 59999
 server.bind = "0.0.0.0"
 server.modules = ( "mod_accesslog", "mod_status" )

filetracker/client/remote_data_store.py

@@ -59,7 +59,7 @@ def __init__(self, base_url):
     def _parse_name(self, name):
         check_name(name)
         name, version = split_name(name)
-        url = self.base_url + pathname2url(name)
+        url = self.base_url + '/files' + pathname2url(name)
         return url, version
 
     def _parse_last_modified(self, response):
@@ -155,7 +155,7 @@ def exists(self, name):
 
     @_verbose_http_errors
     def file_version(self, name):
-        url, version = self._parse_name(name)
+        url, _ = self._parse_name(name)
         response = requests.head(url, allow_redirects=True)
         response.raise_for_status()
         return self._parse_last_modified(response)

filetracker/migration_test.py

@@ -37,7 +37,7 @@ def setUpClass(cls):
             args=(cls.server_dir, fallback_url))
         cls.server_process.start()
 
-        time.sleep(1)  # give servers some time to start
+        time.sleep(2)  # give servers some time to start
 
         cls.client = Client(
             cache_dir=cls.cache_dir1,
@@ -105,6 +105,17 @@ def test_migration_server_should_redirect_to_fallback(self):
         f = self.client.get_stream('/fallback.txt')[0]
         self.assertEqual(f.read(), b'remote hello')
 
+    def test_file_version_should_return_version_from_fallback(self):
+        temp_file = os.path.join(self.temp_dir, 'fallback_version.txt')
+        with open(temp_file, 'w') as tf:
+            tf.write('fallback version')
+
+        timestamp = int(time.time())
+        self.fallback_client.put_file('/fallback_version.txt', temp_file)
+
+        self.assertGreaterEqual(
+                self.client.file_version('/fallback_version.txt'), timestamp)
+
     def test_file_version_of_not_existent_file_should_return_404(self):
         with self.assertRaisesRegexp(FiletrackerError, "404"):
             self.client.get_stream('/nonexistent.txt')

filetracker/protocol_test.py

@@ -0,0 +1,95 @@
+"""Similar to interaction tests, but tests the HTTP API itself."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from multiprocessing import Process
+import os
+import shutil
+import tempfile
+import time
+import unittest
+
+import requests
+import six
+
+from filetracker.client import Client
+from filetracker.servers.run import main as server_main
+
+_TEST_PORT_NUMBER = 45775
+
+
+class ProtocolTest(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.cache_dir = tempfile.mkdtemp()
+        cls.server_dir = tempfile.mkdtemp()
+        cls.temp_dir = tempfile.mkdtemp()
+
+        cls.server_process = Process(
+                target=_start_server, args=(cls.server_dir,))
+        cls.server_process.start()
+        time.sleep(1)   # give server some time to start
+
+        # We use a client to set up test environments.
+        cls.client = Client(
+                cache_dir=cls.cache_dir,
+                remote_url='http://127.0.0.1:{}'.format(_TEST_PORT_NUMBER))
+
+    @classmethod
+    def tearDownClass(cls):
+        cls.server_process.terminate()
+        shutil.rmtree(cls.cache_dir)
+        shutil.rmtree(cls.server_dir)
+        shutil.rmtree(cls.temp_dir)
+
+    def setUp(self):
+        # Shortcuts for convenience
+        self.cache_dir = ProtocolTest.cache_dir
+        self.server_dir = ProtocolTest.server_dir
+        self.temp_dir = ProtocolTest.temp_dir
+        self.client = ProtocolTest.client
+
+    def test_list_files_in_root_should_work(self):
+        src_file = os.path.join(self.temp_dir, 'list.txt')
+        with open(src_file, 'wb') as sf:
+            sf.write(b'hello list')
+
+        self.client.put_file('/list_a.txt', src_file)
+        self.client.put_file('/list_b.txt', src_file)
+
+        res = requests.get(
+                'http://127.0.0.1:{}/list/'.format(_TEST_PORT_NUMBER))
+        self.assertEqual(res.status_code, 200)
+
+        lines = [l for l in res.text.split('\n') if l]
+
+        self.assertEqual(lines.count('list_a.txt'), 1)
+        self.assertEqual(lines.count('list_b.txt'), 1)
+
+    def test_list_files_in_subdirectory_should_work(self):
+        src_file = os.path.join(self.temp_dir, 'list_sub.txt')
+        with open(src_file, 'wb') as sf:
+            sf.write(b'hello list sub')
+
+        self.client.put_file('/sub/direct/ory/list_a.txt', src_file)
+        self.client.put_file('/sub/direct/ory/list_b.txt', src_file)
+        self.client.put_file('/should_not_be_listed', src_file)
+
+        res = requests.get(
+                'http://127.0.0.1:{}/list/sub/direct/'
+                .format(_TEST_PORT_NUMBER))
+        self.assertEqual(res.status_code, 200)
+
+        lines = [l for l in res.text.split('\n') if l]
+        expected = [
+            'ory/list_a.txt',
+            'ory/list_b.txt',
+        ]
+        six.assertCountEqual(self, lines, expected)
+
+
+def _start_server(server_dir):
+    server_main(['-p', str(_TEST_PORT_NUMBER), '-d', server_dir, '-D',
+                 '--workers', '4'])

filetracker/servers/base.py

@@ -1,30 +1,89 @@
+"""This module contains an utility superclass for creating WSGI servers."""
+
 from __future__ import absolute_import
+from __future__ import division
 from __future__ import print_function
+
 import socket
 import errno
 import os
 import sys
 import traceback
 
 
+class HttpError(Exception):
+    def __init__(self, status, description):
+        # status should be a string of form '404 Not Found'
+        self.status = status
+        self.description = description
+
+
 class Server(object):
     """A base WSGI-compatible class, which delegates request handling to
        ``handle_<HTTP-method-name>`` methods."""
 
     def __call__(self, environ, start_response):
         try:
-            return getattr(self, 'handle_' + environ['REQUEST_METHOD']) \
-                    (environ, start_response)
+            if environ['REQUEST_METHOD'] == 'HEAD':
+                environ['REQUEST_METHOD'] = 'GET'
+                _body_iter = self.__call__(environ, start_response)
+                # Server implementations should return closeable iterators
+                # from handle_GET to avoid resource leaks.
+                _body_iter.close()
+
+                return []
+            else:
+                handler = getattr(
+                        self, 'handle_{}'.format(environ['REQUEST_METHOD']))
+                return handler(environ, start_response)
+
+        except HttpError as e:
+            response_headers = [
+                ('Content-Type', 'text/plain'),
+                ('X-Exception', e.description)
+            ]
+            start_response(e.status, response_headers, sys.exc_info())
+            return [traceback.format_exc().encode()]
         except Exception as e:
             status = '500 Oops'
             response_headers = [
-                    ('Content-Type', 'text/plain'),
-                    ('X-Exception', str(e))
-                ]
+                ('Content-Type', 'text/plain'),
+                ('X-Exception', str(e))
+            ]
             start_response(status, response_headers, sys.exc_info())
             return [traceback.format_exc().encode()]
 
 
+def get_endpoint_and_path(environ):
+    """Extracts "endpoint" and "path" from the request URL.
+
+    Endpoint is the first path component, and path is the rest. Both
+    of them are without leading slashes.
+    """
+    path = environ['PATH_INFO']
+    if '..' in path:
+        raise HttpError('400 Bad Request', 'Path cannot contain "..".')
+
+    components = path.split('/')
+
+    # Strip closing slash
+    if components and components[-1] == '':
+        components.pop()
+
+    # If path contained '//', get the segment after the last occurence
+    try:
+        first = _rindex(components, '') + 1
+    except ValueError:
+        first = 0
+
+    components = components[first:]
+
+    if len(components) == 0:
+        return '', ''
+    else:
+        return components[0], '/'.join(components[1:])
+
+
 def start_cgi(server):
     from flup.server.cgi import WSGIServer
     WSGIServer(server).run()
@@ -59,3 +118,8 @@ def main(server):
             start_fcgi(server)
 
     start_standalone(server)
+
+
+def _rindex(l, value):
+    """Same as str.rindex, but for lists."""
+    return len(l) - l[::-1].index(value) - 1