Merge pull request #146 from dstufft/fix-shared-cache

Correctly handle cached shared between Py2 and Py3

Author: ionrock

cachecontrol/compat.py

@@ -24,6 +24,6 @@
 
 # Replicate some six behaviour
 try:
-    text_type = (unicode,)
+    text_type = unicode
 except NameError:
-    text_type = (str,)
+    text_type = str

cachecontrol/serialize.py

@@ -6,7 +6,7 @@
 import msgpack
 from requests.structures import CaseInsensitiveDict
 
-from .compat import HTTPResponse, pickle
+from .compat import HTTPResponse, pickle, text_type
 
 
 def _b64_decode_bytes(b):
@@ -37,27 +37,40 @@ def dumps(self, request, response, body=None):
             #       `Serializer.dump`.
             response._fp = io.BytesIO(body)
 
+        # NOTE: This is all a bit weird, but it's really important that on
+        #       Python 2.x these objects are unicode and not str, even when
+        #       they contain only ascii. The problem here is that msgpack
+        #       understands the difference between unicode and bytes and we
+        #       have it set to differentiate between them, however Python 2
+        #       doesn't know the difference. Forcing these to unicode will be
+        #       enough to have msgpack know the difference.
         data = {
-            "response": {
-                "body": body,
-                "headers": dict(response.headers),
-                "status": response.status,
-                "version": response.version,
-                "reason": response.reason,
-                "strict": response.strict,
-                "decode_content": response.decode_content,
+            u"response": {
+                u"body": body,
+                u"headers": dict(
+                    (text_type(k), text_type(v))
+                    for k, v in response.headers.items()
+                ),
+                u"status": response.status,
+                u"version": response.version,
+                u"reason": text_type(response.reason),
+                u"strict": response.strict,
+                u"decode_content": response.decode_content,
             },
         }
 
         # Construct our vary headers
-        data["vary"] = {}
-        if "vary" in response_headers:
-            varied_headers = response_headers['vary'].split(',')
+        data[u"vary"] = {}
+        if u"vary" in response_headers:
+            varied_headers = response_headers[u'vary'].split(',')
             for header in varied_headers:
                 header = header.strip()
-                data["vary"][header] = request.headers.get(header, None)
+                header_value = request.headers.get(header, None)
+                if header_value is not None:
+                    header_value = text_type(header_value)
+                data[u"vary"][header] = header_value
 
-        return b",".join([b"cc=3", msgpack.dumps(data, use_bin_type=True)])
+        return b",".join([b"cc=4", msgpack.dumps(data, use_bin_type=True)])
 
     def loads(self, request, data):
         # Short circuit if we've been given an empty set of data
@@ -167,6 +180,12 @@ def _loads_v2(self, request, data):
         return self.prepare_response(request, cached)
 
     def _loads_v3(self, request, data):
+        # Due to Python 2 encoding issues, it's impossible to know for sure
+        # exactly how to load v3 entries, thus we'll treat these as a miss so
+        # that they get rewritten out as v4 entries.
+        return
+
+    def _loads_v4(self, request, data):
         try:
             cached = msgpack.loads(data, encoding='utf-8')
         except ValueError:

tests/test_serialization.py

@@ -12,20 +12,20 @@ class TestSerializer(object):
     def setup(self):
         self.serializer = Serializer()
         self.response_data = {
-            'response': {
+            u'response': {
                 # Encode the body as bytes b/c it will eventually be
                 # converted back into a BytesIO object.
-                'body': 'Hello World'.encode('utf-8'),
-                'headers': {
-                    'Content-Type': 'text/plain',
-                    'Expires': '87654',
-                    'Cache-Control': 'public',
+                u'body': 'Hello World'.encode('utf-8'),
+                u'headers': {
+                    u'Content-Type': u'text/plain',
+                    u'Expires': u'87654',
+                    u'Cache-Control': u'public',
                 },
-                'status': 200,
-                'version': '2',
-                'reason': '',
-                'strict': '',
-                'decode_content': True,
+                u'status': 200,
+                u'version': 11,
+                u'reason': u'',
+                u'strict': True,
+                u'decode_content': True,
             },
         }
 
@@ -48,9 +48,15 @@ def test_read_version_v2(self):
         # We have to decode our urllib3 data back into a unicode string.
         assert resp.data == 'Hello World'.encode('utf-8')
 
-    def test_read_version_v3(self):
+    def test_load_by_version_v3(self):
+        data = b'cc=3,somedata'
         req = Mock()
-        resp = self.serializer._loads_v3(req, msgpack.dumps(self.response_data))
+        resp = self.serializer.loads(req, data)
+        assert resp is None
+
+    def test_read_version_v4(self):
+        req = Mock()
+        resp = self.serializer._loads_v4(req, msgpack.dumps(self.response_data))
         # We have to decode our urllib3 data back into a unicode string.
         assert resp.data == 'Hello World'.encode('utf-8')
 
@@ -120,4 +126,3 @@ def test_no_vary_header(self, url):
                 body=data
             )
         )
-