fix xmlrpc error when transport non-utf8 content binux#15

fix callback not work bug in handler
add test for fetcher save

fetcher/tornado_fetcher.py

@@ -265,14 +265,20 @@ def quit(self):
 
     def xmlrpc_run(self, port=24444, bind='127.0.0.1', logRequests=False):
         from SimpleXMLRPCServer import SimpleXMLRPCServer
+        from xmlrpclib import Binary
+        import cPickle as pickle
 
         server = SimpleXMLRPCServer((bind, port), allow_none=True, logRequests=logRequests)
         server.register_introspection_functions()
         server.register_multicall_functions()
 
         server.register_function(self.quit, '_quit')
         server.register_function(self.size)
-        server.register_function(self.sync_fetch, 'fetch')
+        def sync_fetch(task):
+            result = self.sync_fetch(task)
+            result = Binary(pickle.dumps(result))
+            return result
+        server.register_function(sync_fetch, 'fetch')
         def dump_counter(_time, _type):
             return self._cnt[_time].to_dict(_type)
         server.register_function(dump_counter, 'counter')

libs/base_handler.py

@@ -185,7 +185,7 @@ def _crawl(self, url, **kwargs):
             task['fetch'] = fetch
 
         process = {}
-        for key in ('callback'):
+        for key in ('callback', ):
             if key in kwargs and kwargs[key] is not None:
                 process[key] = kwargs[key]
         if process:

run.py

@@ -55,11 +55,12 @@ def run_processor():
 
 def run_webui():
     import xmlrpclib
+    import cPickle as pickle
     scheduler_rpc = xmlrpclib.ServerProxy('http://localhost:%d' % scheduler_xmlrpc_port)
     fetch_rpc = xmlrpclib.ServerProxy('http://localhost:%d' % fetcher_xmlrpc_port)
 
     from webui.app import app
-    app.config['fetch'] = lambda task: fetch_rpc.fetch(task)
+    app.config['fetch'] = lambda task: pickle.loads(fetch_rpc.fetch(task).data)
     app.config['projectdb'] = get_projectdb
     app.config['scheduler_rpc'] = scheduler_rpc
     app.run()

test/test_fetcher.py

@@ -24,6 +24,7 @@ class TestTaskDB(unittest.TestCase):
                     'a': 'b'
                     },
                 'timeout': 60,
+                'save': 'abc',
                 },
             'process': {
                 'callback': 'callback',
@@ -44,6 +45,7 @@ def test_http_get(self):
         result = self.fetcher.sync_fetch(self.sample_task_http)
         self.assertEqual(result['status_code'], 200)
         self.assertEqual(result['orig_url'], self.sample_task_http['url'])
+        self.assertEqual(result['save'], self.sample_task_http['fetch']['save'])
         self.assertIn('content', result)
         content = json.loads(result['content'])
         self.assertIn('headers', content)