Skip to content

Commit 926b42a

Browse files
radeusgdaccek
authored andcommitted
Replaced lighttpd with gunicorn. (#37)
* Replaced lighttd with gunicorn. * Rewritten migration_test to use new server backend. * Added workers count parameter. * Fixed file_version not working with migrations (HEAD was not using redirects) Also added a test that checks if file_version works with migration server.
1 parent f2829b3 commit 926b42a

File tree

6 files changed

+117
-114
lines changed

6 files changed

+117
-114
lines changed

filetracker/client/remote_data_store.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -142,7 +142,7 @@ def get_stream(self, name):
142142

143143
def exists(self, name):
144144
url, version = self._parse_name(name)
145-
response = requests.head(url)
145+
response = requests.head(url, allow_redirects=True)
146146
if response.status_code == 404:
147147
return False
148148

@@ -156,14 +156,14 @@ def exists(self, name):
156156
@_verbose_http_errors
157157
def file_version(self, name):
158158
url, version = self._parse_name(name)
159-
response = requests.head(url)
159+
response = requests.head(url, allow_redirects=True)
160160
response.raise_for_status()
161161
return self._parse_last_modified(response)
162162

163163
@_verbose_http_errors
164164
def file_size(self, name):
165165
url, version = self._parse_name(name)
166-
response = requests.head(url)
166+
response = requests.head(url, allow_redirects=True)
167167
response.raise_for_status()
168168
return int(response.headers.get('content-length', 0))
169169

filetracker/interaction_test.py

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
import unittest
1313

1414
from filetracker.client import Client, FiletrackerError
15-
from filetracker.servers.run import main as lighttpd_main
15+
from filetracker.servers.run import main as server_main
1616

1717
_TEST_PORT_NUMBER = 45735
1818

@@ -184,4 +184,5 @@ def test_delete_should_remove_file(self):
184184

185185

186186
def _start_server(server_dir):
187-
lighttpd_main(['-p', str(_TEST_PORT_NUMBER), '-d', server_dir, '-D'])
187+
server_main(['-p', str(_TEST_PORT_NUMBER), '-d', server_dir, '-D',
188+
'--workers', '4'])

filetracker/migration_test.py

Lines changed: 29 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -4,20 +4,18 @@
44
from __future__ import division
55
from __future__ import print_function
66

7+
from multiprocessing import Process
78
import os
89
import shutil
9-
import signal
1010
import tempfile
1111
import time
1212
import unittest
13-
from wsgiref.simple_server import make_server
13+
from filetracker.servers.run import main as server_main
1414

15-
from filetracker.client import Client
16-
from filetracker.servers.migration import MigrationFiletrackerServer
17-
from filetracker.servers.files import FiletrackerServer
15+
from filetracker.client import Client, FiletrackerError
1816

19-
_TEST_PRIMARY_PORT_NUMBER = 45735
20-
_TEST_FALLBACK_PORT_NUMBER = 45745
17+
_TEST_PRIMARY_PORT_NUMBER = 45755
18+
_TEST_FALLBACK_PORT_NUMBER = 45765
2119

2220

2321
class MigrationTest(unittest.TestCase):
@@ -29,13 +27,17 @@ def setUpClass(cls):
2927
cls.fallback_server_dir = tempfile.mkdtemp()
3028
cls.temp_dir = tempfile.mkdtemp()
3129

32-
cls.fallback_server = FiletrackerServer(cls.fallback_server_dir)
33-
cls.fallback_server_pid = _fork_to_server(cls.fallback_server,
34-
_TEST_FALLBACK_PORT_NUMBER)
30+
cls.fallback_server_process = Process(
31+
target=_start_fallback_server, args=(cls.fallback_server_dir,))
32+
cls.fallback_server_process.start()
3533

3634
fallback_url = 'http://127.0.0.1:{}'.format(_TEST_FALLBACK_PORT_NUMBER)
37-
cls.server = MigrationFiletrackerServer(fallback_url, cls.server_dir)
38-
cls.server_pid = _fork_to_server(cls.server, _TEST_PRIMARY_PORT_NUMBER)
35+
cls.server_process = Process(
36+
target=_start_migration_server,
37+
args=(cls.server_dir, fallback_url))
38+
cls.server_process.start()
39+
40+
time.sleep(1) # give servers some time to start
3941

4042
cls.client = Client(
4143
cache_dir=cls.cache_dir1,
@@ -47,8 +49,9 @@ def setUpClass(cls):
4749

4850
@classmethod
4951
def tearDownClass(cls):
50-
os.kill(cls.server_pid, signal.SIGKILL)
51-
os.kill(cls.fallback_server_pid, signal.SIGKILL)
52+
cls.fallback_server_process.terminate()
53+
cls.server_process.terminate()
54+
5255
shutil.rmtree(cls.cache_dir1)
5356
shutil.rmtree(cls.cache_dir2)
5457
shutil.rmtree(cls.fallback_server_dir)
@@ -102,14 +105,16 @@ def test_migration_server_should_redirect_to_fallback(self):
102105
f = self.client.get_stream('/fallback.txt')[0]
103106
self.assertEqual(f.read(), b'remote hello')
104107

108+
def test_file_version_of_not_existent_file_should_return_404(self):
109+
with self.assertRaisesRegexp(FiletrackerError, "404"):
110+
self.client.get_stream('/nonexistent.txt')
111+
112+
113+
def _start_fallback_server(server_dir):
114+
server_main(['-p', str(_TEST_FALLBACK_PORT_NUMBER), '-d', server_dir, '-D',
115+
'--workers', '4'])
116+
105117

106-
def _fork_to_server(server, port):
107-
"""Returns child server process PID."""
108-
pid = os.fork()
109-
if pid > 0:
110-
time.sleep(1) # give server some time to start
111-
return pid
112-
else:
113-
httpd = make_server('', port, server)
114-
print('Serving on port %d' % port)
115-
httpd.serve_forever()
118+
def _start_migration_server(server_dir, fallback_url):
119+
server_main(['-p', str(_TEST_PRIMARY_PORT_NUMBER), '-d', server_dir, '-D',
120+
'--fallback-url', fallback_url, '--workers', '4'])

filetracker/parallel_test.py

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
from six import BytesIO
1515

1616
from filetracker.client import Client
17-
from filetracker.servers.run import main as lighttpd_main
17+
from filetracker.servers.run import main as server_main
1818

1919
# _CLIENT_WAIT_S and _FILE_SIZE should be picked in a way that the time
2020
# between spawning a client and this client sending a request is
@@ -92,4 +92,5 @@ def test_only_last_parallel_upload_of_same_file_should_succeed(self):
9292

9393

9494
def _start_server(server_dir):
95-
lighttpd_main(['-p', str(_TEST_PORT_NUMBER), '-d', server_dir, '-D'])
95+
server_main(['-p', str(_TEST_PORT_NUMBER), '-d', server_dir, '-D',
96+
'--workers', '6'])

filetracker/servers/run.py

Lines changed: 78 additions & 83 deletions
Original file line numberDiff line numberDiff line change
@@ -1,31 +1,33 @@
11
#!/usr/bin/env python
22

3-
"""A script for starting filetracker server using lighttpd."""
3+
"""A script for starting filetracker server using gunicorn."""
44

55
from __future__ import absolute_import
6+
7+
import multiprocessing
68
import os
7-
import os.path
9+
import re
10+
import signal
11+
import subprocess
812
import sys
9-
from optparse import OptionParser
1013
import tempfile
11-
import subprocess
12-
import signal
14+
from optparse import OptionParser
1315

14-
import filetracker.servers.files
15-
import filetracker.servers.base
16+
from filetracker.servers.files import FiletrackerServer
1617
from filetracker.servers.migration import MigrationFiletrackerServer
1718

1819
# Clients may use this as a sensible default port to connect to.
1920
DEFAULT_PORT = 9999
2021

2122

23+
def strip_margin(text):
24+
return re.sub('\n[ \t]*\|', '\n', text)
25+
26+
2227
def main(args=None):
23-
epilog = "If LIGHTTPD_DIR is set in environment, it is assumed that " \
24-
"the lighttpd binary resides in that directory together with " \
25-
"the required modules: mod_fastcgi, mod_setenv and mod_status."
26-
parser = OptionParser(epilog=epilog)
28+
parser = OptionParser()
2729
parser.add_option('-p', '--port', dest='port', default=DEFAULT_PORT,
28-
type="int",
30+
type='int',
2931
help="Listen on specified port number")
3032
parser.add_option('-l', '--listen-on', dest='listen_on',
3133
default='127.0.0.1',
@@ -34,17 +36,19 @@ def main(args=None):
3436
help="Specify Filetracker dir (taken from FILETRACKER_DIR "
3537
"environment variable if not present)")
3638
parser.add_option('-L', '--log', dest='log', default=None,
37-
help="Log file location (no log by default)")
39+
help="Error log file location (no log by default)")
40+
parser.add_option('--access-log', dest='access_log', default=None,
41+
help="Access log file location (no log by default)")
3842
parser.add_option('-D', '--no-daemon', dest='daemonize',
3943
action='store_false', default=True,
4044
help="Do not daemonize, stay in foreground")
41-
parser.add_option('--lighttpd-bin', dest='lighttpd_bin',
42-
default='lighttpd',
43-
help="Specify the lighttpd binary to use")
4445
parser.add_option('--fallback-url', dest='fallback_url',
4546
default=None,
4647
help="Turns on migration mode "
4748
"and redirects requests to nonexistent files to the remote")
49+
parser.add_option('--workers', dest='workers', type='int',
50+
default=2 * multiprocessing.cpu_count(),
51+
help="Specifies the amount of worker processes to use")
4852
options, args = parser.parse_args(args)
4953
if args:
5054
parser.error("Unrecognized arguments: " + ' '.join(args))
@@ -59,80 +63,51 @@ def main(args=None):
5963
if not os.path.exists(docroot):
6064
os.makedirs(docroot, 0o700)
6165

62-
if options.fallback_url is not None:
63-
run_migration_server(options)
64-
os.exit(0)
65-
66-
LIGHTHTTPD_CONF = """
67-
server.tag = "filetracker"
68-
server.document-root = "%(docroot)s"
69-
server.port = %(port)d
70-
server.bind = "%(listen_on)s"
71-
server.modules = ( "mod_fastcgi", "mod_status", "mod_setenv" )
72-
status.status-url = "/status"
73-
#debug.log-response-header = "enable"
74-
#debug.log-request-header = "enable"
75-
#debug.log-request-handling = "enable"
76-
#debug.log-condition-handling = "enable"
77-
fastcgi.debug = 1
78-
mimetype.assign = (
79-
"" => "application/octet-stream"
80-
)
81-
fastcgi.server += (
82-
"" =>
83-
(( "bin-path" => "%(interpreter)s %(files_script)s",
84-
"bin-environment" => (
85-
"FILETRACKER_DIR" => "%(filetracker_dir)s"
86-
),
87-
"socket" => "%(tempdir)s/filetracker-files.%(pid)d",
88-
"check-local" => "disable",
89-
"fix-root-scriptname" => "enable"
90-
))
91-
)
92-
""" % dict(
93-
filetracker_dir=filetracker_dir,
94-
docroot=docroot,
95-
port=options.port,
96-
listen_on=options.listen_on,
97-
interpreter=sys.executable,
98-
files_script=filetracker.servers.files.__file__,
99-
pid=os.getpid(),
100-
tempdir=tempfile.gettempdir())
66+
gunicorn_settings = strip_margin("""
67+
|bind = ['{listen_on}:{port}']
68+
|daemon = {daemonize}
69+
|workers = {workers}
70+
|raw_env = ['FILETRACKER_DIR={filetracker_dir}',
71+
| 'FILETRACKER_FALLBACK_URL={fallback_url}']
72+
""".format(
73+
listen_on=options.listen_on,
74+
port=options.port,
75+
daemonize=options.daemonize,
76+
workers=options.workers,
77+
filetracker_dir=options.dir,
78+
fallback_url=options.fallback_url
79+
))
10180

10281
if options.log:
103-
LIGHTHTTPD_CONF += """
104-
server.modules += ( "mod_accesslog" )
105-
accesslog.filename = "%(log)s"
106-
""" % dict(log=os.path.abspath(options.log))
82+
gunicorn_settings += strip_margin("""
83+
|errorlog = '{errorlog}'
84+
|capture_output = True
85+
""".format(
86+
errorlog=options.log,
87+
))
88+
if options.access_log:
89+
gunicorn_settings += strip_margin("""
90+
|accesslog = '{accesslog}'
91+
""".format(
92+
accesslog=options.access_log,
93+
))
10794

10895
conf_fd, conf_path = tempfile.mkstemp(text=True)
10996
try:
11097
conf_file = os.fdopen(conf_fd, 'w')
111-
conf_file.write(LIGHTHTTPD_CONF)
98+
conf_file.write(gunicorn_settings)
11299
conf_file.close()
113100

114-
env = os.environ.copy()
115-
if sys.platform == 'darwin' or not options.daemonize:
116-
# setsid(1) is not available on Mac
117-
args = []
118-
else:
119-
args = ['setsid']
120-
if 'LIGHTTPD_DIR' in os.environ:
121-
server_dir = os.environ['LIGHTTPD_DIR']
122-
args += [os.path.join(server_dir, 'lighttpd'),
123-
'-f', conf_path, '-m', server_dir]
124-
env['LD_LIBRARY_PATH'] = server_dir + ':' \
125-
+ env.get('LD_LIBRARY_PATH', '')
101+
args = ['gunicorn', '-c', conf_path]
102+
if options.fallback_url is not None:
103+
args.append('filetracker.servers.run:gunicorn_entry_migration')
126104
else:
127-
args += [options.lighttpd_bin, '-f', conf_path]
128-
129-
if not options.daemonize:
130-
args.append('-D')
105+
args.append('filetracker.servers.run:gunicorn_entry')
131106

132107
try:
133-
popen = subprocess.Popen(args, env=env)
108+
popen = subprocess.Popen(args)
134109
except OSError as e:
135-
raise RuntimeError("Cannot run lighttpd:\n%s" % e)
110+
raise RuntimeError("Cannot run gunicorn:\n%s" % e)
136111

137112
signal.signal(signal.SIGINT, lambda signum, frame: popen.terminate())
138113
signal.signal(signal.SIGTERM, lambda signum, frame: popen.terminate())
@@ -141,16 +116,36 @@ def main(args=None):
141116
if not options.daemonize:
142117
sys.exit(retval)
143118
if retval:
144-
raise RuntimeError("Lighttpd exited with code %d" % retval)
119+
raise RuntimeError("gunicorn exited with code %d" % retval)
145120
finally:
146-
# At this point lighttpd does not need the configuration file, so it
121+
# At this point gunicorn does not need the configuration file, so it
147122
# can be safely deleted.
148123
os.unlink(conf_path)
149124

150125

151-
def run_migration_server(options):
152-
server = MigrationFiletrackerServer(options.fallback_url, options.dir)
153-
filetracker.servers.base.start_standalone(server, options.port)
126+
# This filetracker_instance is cached between requests within one WSGI process.
127+
# There are no threading problems though,
128+
# because each process is set to use 1 thread.
129+
filetracker_instance = None
130+
131+
132+
def gunicorn_entry(env, start_response):
133+
global filetracker_instance
134+
if filetracker_instance is None:
135+
filetracker_instance = FiletrackerServer()
136+
return filetracker_instance(env, start_response)
137+
138+
139+
def gunicorn_entry_migration(env, start_response):
140+
global filetracker_instance
141+
if filetracker_instance is None:
142+
fallback = os.environ.get('FILETRACKER_FALLBACK_URL', None)
143+
if not fallback:
144+
raise RuntimeError("Configuration error. Fallback url not set.")
145+
filetracker_instance = MigrationFiletrackerServer(
146+
redirect_url=fallback
147+
)
148+
return filetracker_instance(env, start_response)
154149

155150

156151
if __name__ == '__main__':

setup.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
install_requires = [
1414
'bsddb3',
1515
'flup6',
16+
'gunicorn',
1617
'progressbar2',
1718
'requests',
1819
'six',

0 commit comments

Comments
 (0)