Refactored for Python3 compatibility

Author: candale

scrapy_jsonrpc/jsonrpc.py

@@ -58,10 +58,11 @@ def jsonrpc_server_call(target, jsonrpc_request, json_decoder=None):
         json_decoder = ScrapyJSONDecoder()
 
     try:
-        req = json_decoder.decode(jsonrpc_request)
+        req = json_decoder.decode(jsonrpc_request.decode('utf-8'))
     except Exception as e:
-        return jsonrpc_error(None, jsonrpc_errors.PARSE_ERROR, 'Parse error',
-                             traceback.format_exc())
+        return jsonrpc_error(
+            None, jsonrpc_errors.PARSE_ERROR, 'Parse error',
+            traceback.format_exc())
 
     try:
         id, methname = req['id'], req['method']

scrapy_jsonrpc/txweb.py

@@ -2,19 +2,24 @@
 
 from twisted.web import resource
 
-class JsonResource(resource.Resource):
+
+class JsonResource(resource.Resource, object):
 
     json_encoder = json.JSONEncoder()
 
+    def __init__(self):
+        super(JsonResource, self).__init__()
+
     def render(self, txrequest):
-        r = resource.Resource.render(self, txrequest)
+        r = super(JsonResource, self).render(txrequest)
         return self.render_object(r, txrequest)
 
     def render_object(self, obj, txrequest):
-        r = self.json_encoder.encode(obj) + "\n"
+        r = (self.json_encoder.encode(obj) + "\n").encode()
+
         txrequest.setHeader('Content-Type', 'application/json')
         txrequest.setHeader('Access-Control-Allow-Origin', '*')
         txrequest.setHeader('Access-Control-Allow-Methods', 'GET, POST, PATCH, PUT, DELETE')
-        txrequest.setHeader('Access-Control-Allow-Headers',' X-Requested-With')
+        txrequest.setHeader('Access-Control-Allow-Headers', 'X-Requested-With')
         txrequest.setHeader('Content-Length', len(r))
         return r

scrapy_jsonrpc/webservice.py

@@ -1,6 +1,8 @@
 import logging
+import six
 
 from twisted.web import server, resource
+from twisted.python.compat import _PY3, nativeString
 
 from scrapy.exceptions import NotConfigured
 from scrapy import signals
@@ -17,27 +19,34 @@
 class JsonResource(JsonResource_):
 
     def __init__(self, crawler, target=None):
-        JsonResource_.__init__(self)
+        super(JsonResource, self).__init__()
+
         self.crawler = crawler
         self.json_encoder = ScrapyJSONEncoder(crawler=crawler)
 
+    def getChildWithDefault(self, path, request):
+        path = path.decode('UTF-8')
+        return super(JsonResource, self).getChildWithDefault(path, request)
+
+
 class JsonRpcResource(JsonResource):
 
     def __init__(self, crawler, target=None):
-        JsonResource.__init__(self, crawler, target)
+        super(JsonRpcResource, self).__init__(crawler, target)
+
         self.json_decoder = ScrapyJSONDecoder(crawler=crawler)
         self.crawler = crawler
         self._target = target
 
-    def render_GET(self, txrequest):
+    def render_GET(self, request):
         return self.get_target()
 
-    def render_POST(self, txrequest):
-        reqstr = txrequest.content.getvalue()
+    def render_POST(self, request):
+        reqstr = request.content.getvalue()
         target = self.get_target()
         return jsonrpc_server_call(target, reqstr, self.json_decoder)
 
-    def getChild(self, name, txrequest):
+    def getChild(self, name, request):
         target = self.get_target()
         try:
             newtarget = getattr(target, name)
@@ -54,33 +63,37 @@ class CrawlerResource(JsonRpcResource):
     ws_name = 'crawler'
 
     def __init__(self, crawler):
-        JsonRpcResource.__init__(self, crawler, crawler)
+        super(CrawlerResource, self).__init__(crawler, target=crawler)
 
 
 class RootResource(JsonResource):
 
-    def render_GET(self, txrequest):
-        return {'resources': self.children.keys()}
+    def render_GET(self, request):
+        return {'resources': list(self.children.keys())}
 
-    def getChild(self, name, txrequest):
+    def getChild(self, name, request):
         if name == '':
             return self
-        return JsonResource.getChild(self, name, txrequest)
+        return JsonResource.getChild(self, name, request)
 
 
-class WebService(server.Site):
+class WebService(server.Site, object):
 
     def __init__(self, crawler):
         if not crawler.settings.getbool('JSONRPC_ENABLED'):
             raise NotConfigured
-        self.crawler = crawler
+
         logfile = crawler.settings['JSONRPC_LOGFILE']
+        self.crawler = crawler
         self.portrange = [int(x) for x in crawler.settings.getlist('JSONRPC_PORT', [6023, 6073])]
         self.host = crawler.settings.get('JSONRPC_HOST', '127.0.0.1')
+        self.noisy = False
+
         root = RootResource(crawler)
         root.putChild('crawler', CrawlerResource(self.crawler))
-        server.Site.__init__(self, root, logPath=logfile)
-        self.noisy = False
+
+        super(WebService, self).__init__(root, logPath=logfile)
+
         crawler.signals.connect(self.start_listening, signals.engine_started)
         crawler.signals.connect(self.stop_listening, signals.engine_stopped)
 
@@ -90,10 +103,30 @@ def from_crawler(cls, crawler):
 
     def start_listening(self):
         self.port = listen_tcp(self.portrange, self.host, self)
+
         logger.debug(
             'Web service listening on {host.host:s}:{host.port:d}'.format(
                 host=self.port.getHost()))
 
     def stop_listening(self):
         self.port.stopListening()
 
+    def log(self, request):
+        """
+        Write a line representing C{request} to the access log file.
+
+        @param request: The request object about which to log.
+        @type request: L{Request}
+        """
+        try:
+            logFile = self.logFile
+        except AttributeError:
+            pass
+        else:
+            line = self._logFormatter(self._logDateTime, request) + u"\n"
+            if self._nativeize:
+                line = nativeString(line)
+            # If we're on Python3, we don't need to encode it
+            elif _PY3 is False:
+                line = line.encode("utf-8")
+            logFile.write(line)