From 3ca1d98b40650017a667cc6815186f19b0272096 Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Tue, 16 Mar 2021 10:46:00 +0100 Subject: [PATCH 1/5] Slightly improve test layer setup by invoking the CrateLayer at runtime This is the foundation to improve the situation later by adding support for running CrateDB through Docker or other means. --- src/crate/client/tests.py | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/src/crate/client/tests.py b/src/crate/client/tests.py index 2a4cc61c..4c9636df 100644 --- a/src/crate/client/tests.py +++ b/src/crate/client/tests.py @@ -108,17 +108,23 @@ def setUpMocked(test): crate_port = 44209 crate_transport_port = 44309 local = '127.0.0.1' -crate_layer = CrateLayer('crate', - crate_home=crate_path(), - port=crate_port, - host=local, - transport_port=crate_transport_port, - settings=settings) - crate_host = "{host}:{port}".format(host=local, port=crate_port) crate_uri = "http://%s" % crate_host +def ensure_cratedb_layer(): + global crate_layer + + if crate_layer is None: + crate_layer = CrateLayer('crate', + crate_home=crate_path(), + port=crate_port, + host=local, + transport_port=crate_transport_port, + settings=settings) + return crate_layer + + def refresh(table): with connect(crate_host) as conn: cursor = conn.cursor() @@ -348,7 +354,7 @@ def test_suite(): optionflags=flags, encoding='utf-8' ) - s.layer = crate_layer + s.layer = ensure_cratedb_layer() suite.addTest(s) s = doctest.DocFileSuite( @@ -362,7 +368,7 @@ def test_suite(): optionflags=flags, encoding='utf-8' ) - s.layer = crate_layer + s.layer = ensure_cratedb_layer() suite.addTest(s) s = doctest.DocFileSuite( @@ -372,7 +378,7 @@ def test_suite(): optionflags=flags, encoding='utf-8' ) - s.layer = crate_layer + s.layer = ensure_cratedb_layer() suite.addTest(s) return suite From e4f03a05b612e73b3e303e1ed7dea6d164493eb9 Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Tue, 16 Mar 2021 11:42:36 +0100 Subject: [PATCH 2/5] Slightly improve HttpsTestServerLayer - Send Content-Length header based on the encoded response body - Reduce sleep time after starting the server thread --- src/crate/client/tests.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/crate/client/tests.py b/src/crate/client/tests.py index 4c9636df..a0203e7c 100644 --- a/src/crate/client/tests.py +++ b/src/crate/client/tests.py @@ -110,6 +110,7 @@ def setUpMocked(test): local = '127.0.0.1' crate_host = "{host}:{port}".format(host=local, port=crate_port) crate_uri = "http://%s" % crate_host +crate_layer = None def ensure_cratedb_layer(): @@ -213,7 +214,7 @@ class Location(Base): test.globs['CrateDialect'] = CrateDialect -class HttpsTestServerLayer(object): +class HttpsTestServerLayer: PORT = 65534 HOST = "localhost" CERT_FILE = os.path.abspath(os.path.join(os.path.dirname(__file__), @@ -241,11 +242,11 @@ class HttpsHandler(BaseHTTPRequestHandler): def do_GET(self): self.send_response(200) - self.send_header("Content-Length", len(self.payload)) + payload = self.payload.encode('UTF-8') + self.send_header("Content-Length", len(payload)) self.send_header("Content-Type", "application/json; charset=UTF-8") self.end_headers() - self.wfile.write(self.payload.encode('UTF-8')) - return + self.wfile.write(payload) def __init__(self): self.server = self.HttpsServer( @@ -257,7 +258,7 @@ def setUp(self): thread = threading.Thread(target=self.serve_forever) thread.daemon = True # quit interpreter when only thread exists thread.start() - time.sleep(1) + time.sleep(0.5) def serve_forever(self): print("listening on", self.HOST, self.PORT) From 3cce9b68ac88993a47a6497f0726b8c4f4ee794b Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Tue, 16 Mar 2021 12:53:35 +0100 Subject: [PATCH 3/5] Use correct logger method ("warning" instead of "warn") --- src/crate/client/http.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/crate/client/http.py b/src/crate/client/http.py index 1f81c631..02aa80bd 100644 --- a/src/crate/client/http.py +++ b/src/crate/client/http.py @@ -544,8 +544,8 @@ def _get_server(self): (ts, server, message)) else: self._active_servers.append(server) - logger.warn("Restored server %s into active pool", - server) + logger.warning("Restored server %s into active pool", + server) # if none is old enough, use oldest if not self._active_servers: From f9c492631f2f1ff13687e6db7f4e06b930d170f2 Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Thu, 25 Mar 2021 17:31:41 +0100 Subject: [PATCH 4/5] Tests: Remove arbitrary sleeping for waiting for the webserver to start --- setup.py | 5 +++-- src/crate/client/tests.py | 28 +++++++++++++++++++++++++++- versions.cfg | 1 + 3 files changed, 31 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index cdd445ef..cea61f20 100644 --- a/setup.py +++ b/setup.py @@ -64,8 +64,9 @@ def read(path): ] }, extras_require=dict( - test=['zope.testing', - 'zc.customdoctests>=1.0.1'], + test=['zope.testing>=4,<5', + 'zc.customdoctests>=1.0.1,<2', + 'stopit>=1.1.2,<2'], sqlalchemy=['sqlalchemy>=1.0,<1.4', 'geojson>=2.5.0'] ), python_requires='>=3.4', diff --git a/src/crate/client/tests.py b/src/crate/client/tests.py index a0203e7c..58db6cb7 100644 --- a/src/crate/client/tests.py +++ b/src/crate/client/tests.py @@ -23,6 +23,7 @@ import json import os +import socket import unittest import doctest from pprint import pprint @@ -33,6 +34,8 @@ import threading import logging +import stopit + from crate.testing.layer import CrateLayer from crate.testing.tests import crate_path, docs_path from crate.client import connect @@ -258,7 +261,7 @@ def setUp(self): thread = threading.Thread(target=self.serve_forever) thread.daemon = True # quit interpreter when only thread exists thread.start() - time.sleep(0.5) + self.waitForServer() def serve_forever(self): print("listening on", self.HOST, self.PORT) @@ -268,6 +271,29 @@ def serve_forever(self): def tearDown(self): self.server.shutdown() + def isUp(self): + """ + Test if a host is up. + """ + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + ex = s.connect_ex((self.HOST, self.PORT)) + s.close() + return ex == 0 + + def waitForServer(self, timeout=5): + """ + Wait for the host to be available. + """ + with stopit.ThreadingTimeout(timeout) as to_ctx_mgr: + while True: + if self.isUp(): + break + time.sleep(0.001) + + if not to_ctx_mgr: + raise TimeoutError("Could not properly start embedded webserver " + "within {} seconds".format(timeout)) + def setUpWithHttps(test): test.globs['HttpClient'] = http.Client diff --git a/versions.cfg b/versions.cfg index 1ff59bf6..cd8759f6 100644 --- a/versions.cfg +++ b/versions.cfg @@ -24,6 +24,7 @@ zc.customdoctests = 1.0.1 zc.recipe.egg = 2.0.7 zc.recipe.testrunner = 2.2 zope.testing = 4.9 +stopit = 1.1.2 # Required by: # clint==0.5.1 From 3335961f298555776a1ac64e0d024ad69ab74472 Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Thu, 25 Mar 2021 18:29:19 +0100 Subject: [PATCH 5/5] Add rationale for `ensure_cratedb_layer` --- src/crate/client/tests.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/crate/client/tests.py b/src/crate/client/tests.py index 58db6cb7..574f1a7e 100644 --- a/src/crate/client/tests.py +++ b/src/crate/client/tests.py @@ -117,6 +117,19 @@ def setUpMocked(test): def ensure_cratedb_layer(): + """ + In order to skip individual tests by manually disabling them within + `def test_suite()`, it is crucial make the test layer not run on each + and every occasion. So, things like this will be possible:: + + ./bin/test -vvvv --ignore_dir=testing + + TODO: Through a subsequent patch, the possibility to individually + unselect specific tests might be added to `def test_suite()` + on behalf of environment variables. + A blueprint for this kind of logic can be found at + https://github.com/crate/crate/commit/414cd833. + """ global crate_layer if crate_layer is None: