Coverage for src/pyicap.py : 74%
Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
|
############################################################################### # # vim:ts=3:sw=3:expandtab # # Open source pyicap module retrofitted with tornado based socket read/write # # Copyright (C) 2017, Menlo Security, Inc. # # All rights reserved. # # Summary: Open source pyicap module (https://github.com/netom/pyicap) uses # multi-threading for handling icap transactions. This did not scale well. In # this implementation, we use tornado based socket io operations and leverage # pyicap for icap message parsing. The changes made here are minimal so that any # future changes/bugfixes to pyicap can be easily ported to this implementation ###############################################################################
"""Dynamic cached mapping of header names to Http-Header-Case.
Implemented as a dict subclass so that cache hits are as fast as a normal dict lookup, without the overhead of a python function call.
This is a modified version of tornado.httputil's implementation to define default mappings. Defaults can be specified by assigning a lower case version of the header to the re-cased version.
>>> normalized_headers = NormalizedHeaderCache(10) >>> normalized_headers["coNtent-TYPE"] 'Content-Type' >>> normalized_headers["istag"] = 'ISTag' >>> normalized_headers["iStAg"] 'ISTag' """
normalized = self[key_l] else: # Limit the size of the cache. LRU would be better, but this # simpler approach should be fine. In Python 2.7+ we could # use OrderedDict (or in 3.2+, @functools.lru_cache). old_key = self.queue.popleft() del self[old_key]
# Defined specific header to re-cased version
"""Signals a protocol error""" if message is None: message = BaseICAPRequestHandler._responses[code] self.message = message super(ICAPError, self).__init__(message) self.code = code
"""ICAP request handler base class.
You have to subclass it and provide methods for each service endpoint. Every endpoint MUST have an _OPTION method, and either a _REQMOD or a _RESPMOD method. """
# The version of the ICAP protocol we support.
# Table mapping response codes to messages; entries have the # form {code: (shortmessage, longmessage)}. # See RFC 2616 and RFC 3507 100: (b'Continue', b'Request received, please continue'), 101: (b'Switching Protocols', b'Switching to new protocol; obey Upgrade header'),
200: (b'OK', b'Request fulfilled, document follows'), 201: (b'Created', b'Document created, URL follows'), 202: (b'Accepted', b'Request accepted, processing continues off-line'), 203: (b'Non-Authoritative Information', b'Request fulfilled from cache'), 204: (b'No Content', b'Request fulfilled, nothing follows'), 205: (b'Reset Content', b'Clear input form for further input.'), 206: (b'Partial Content', b'Partial content follows.'),
300: (b'Multiple Choices', b'Object has several resources -- see URI list'), 301: (b'Moved Permanently', b'Object moved permanently -- see URI list'), 302: (b'Found', b'Object moved temporarily -- see URI list'), 303: (b'See Other', b'Object moved -- see Method and URL list'), 304: (b'Not Modified', b'Document has not changed since given time'), 305: (b'Use Proxy', b'You must use proxy specified in Location to access this ' b'resource.'), 307: (b'Temporary Redirect', b'Object moved temporarily -- see URI list'),
400: (b'Bad Request', b'Bad request syntax or unsupported method'), 401: (b'Unauthorized', b'No permission -- see authorization schemes'), 402: (b'Payment Required', b'No payment -- see charging schemes'), 403: (b'Forbidden', b'Request forbidden -- authorization will not help'), 404: (b'Not Found', b'Nothing matches the given URI'), 405: (b'Method Not Allowed', b'Specified method is invalid for this resource.'), 406: (b'Not Acceptable', b'URI not available in preferred format.'), 407: (b'Proxy Authentication Required', b'You must authenticate with ' b'this proxy before proceeding.'), 408: (b'Request Timeout', b'Request timed out; try again later.'), 409: (b'Conflict', b'Request conflict.'), 410: (b'Gone', b'URI no longer exists and has been permanently removed.'), 411: (b'Length Required', b'Client must specify Content-Length.'), 412: (b'Precondition Failed', b'Precondition in headers is false.'), 413: (b'Request Entity Too Large', b'Entity is too large.'), 414: (b'Request-URI Too Long', b'URI is too long.'), 415: (b'Unsupported Media Type', b'Entity body in unsupported format.'), 416: (b'Requested Range Not Satisfiable', b'Cannot satisfy request range.'), 417: (b'Expectation Failed', b'Expect condition could not be satisfied.'),
500: (b'Internal Server Error', b'Server got itself in trouble'), 501: (b'Not Implemented', b'Server does not support this operation'), 502: (b'Bad Gateway', b'Invalid responses from another server/proxy.'), 503: (b'Service Unavailable', b'The server cannot process the request due to a high load'), 504: (b'Gateway Timeout', b'The gateway server did not receive a timely response'), 505: (b'Protocol Version Not Supported', b'Cannot fulfill request.'),
}
# The Python system version, truncated to its first component.
# The server software version. You may want to override this. # The format is multiple whitespace-separated strings, # where each string is of the form name[/version].
b'Aug', b'Sep', b'Oct', b'Nov', b'Dec']
# Initialize handler state 'OPTIONS': self.icap_options, 'REQMOD': self.icap_reqmod, 'RESPMOD': self.icap_respmod, }
"""Read a HTTP or ICAP request line from input stream"""
"""Read a sequence of header lines"""
def read_chunk(self): """Read a HTTP chunk
Also handles the ieof chunk extension defined by the ICAP protocol by setting the ieof variable to True. It returns an empty line if the last chunk is read. Reading after the last chunks will return empty strings. """
# Don't try to read when there's no body
# Connection was probably closed self.eob = True raise tornado.gen.Return(b'')
except ValueError: raise ICAPError(400, 'Protocol error, could not read chunk')
# Look for ieof chunk extension
# +2 is to the line terminators - b'\r\n'
"""Write a chunk of data
When finished writing, an empty chunk with data='' must be written. """
def cont(self): """Send a 100 continue reply
Useful when the client sends a preview request, and we have to read the entire message body. After this command, read_chunk can safely be called again. """ if self.ieof: raise ICAPError(500, 'Tried to continue on ieof condition')
yield self.icap_connection.write_chunk(b'ICAP/1.0 100 Continue\r\n\r\n') self.eob = False
"""Set encapsulated status in response
ICAP responses can only contain one encapsulated header section. Such section is either an encapsulated HTTP request, or a response. This method can be called to set encapsulated HTTP response's status line. """ # TODO: some semantic checking might be OK self.enc_status = status
"""Set encapsulated request line in response
ICAP responses can only contain one encapsulated header section. Such section is either an encapsulated HTTP request, or a response. This method can be called to set encapsulated HTTP request's request line. """ # TODO: some semantic checking might be OK self.enc_request = request
"""Set an encapsulated header to the given value
Multiple sets will cause the header to be sent multiple times. """
"""Sets the ICAP response's status line and response code""" (message if message else self._responses[code][0])
"""Set an ICAP header to the given value
Multiple sets will cause the header to be sent multiple times. """
"""Send ICAP and encapsulated headers
Assembles the Encapsulated header, so it's need the information of wether an encapsulated message body is present. """ enc_header = b'req-hdr=0' enc_body = b'req-body=' enc_req_stat = self.enc_request + b'\r\n'
self.set_icap_header(b'connection', b'close')
self.close_connection = True self.close_connection = False
icap_header_str + enc_header_str)
def parse_request(self, raw_lines): # pylint: disable=too-many-branches """Parse a request (internal)."""
raise ICAPError(400, "Bad request syntax (%r)" % self.requestline)
raise ICAPError(400, "Bad request protocol, only accepting ICAP")
raise ICAPError(501, "command %r is not implemented" % command)
# RFC 2145 section 3.1 says there can be only one "." and # - major and minor numbers MUST be treated as # separate integers; # - ICAP/2.4 is a lower version than ICAP/2.13, which in # turn is lower than ICAP/12.3; # - Leading zeros MUST be ignored by recipients. raise ValueError except (ValueError, IndexError): raise ICAPError(400, "Bad request version (%r)" % version)
raise ICAPError( 505, "Invalid ICAP Version (%s)" % base_version_number )
command, request_uri, version
# Examine the headers and look for a Connection directive
self.close_connection = True
# TODO: raise ICAPError if Encapsulated is malformed or empty
x.strip() for x in self.headers.get(b'allow', [b''])[0].split(b',') ] b'x-client-ip', [b'No X-Client-IP header'])[0]
self.has_body = True # Else: OPTIONS. No encapsulation.
# Parse service name
def handle_request(self): """Handle a single ICAP request.
You normally don't need to override this method; see the class __doc__ string for information on how to handle specific HTTP commands such as GET and POST.
""" self.close_connection = True return
self.log_error("Request timed out: %r", e) self.close_connection = True self.log_error("ICAP error: %r", e) self.send_error(e.code, e.message[0]) # except: # self.send_error(500)
"""Send and log an error reply.
Arguments are the error code, and a detailed message. The detailed message defaults to the short entry matching the response code.
This sends an error response (so it must be called before any output has been generated), logs the error, and finally sends a piece of HTML explaining the error to the user.
"""
if message is None: message = self._responses[code][0] self.log_error("code %d, message %s", code, message)
# No encapsulation self.enc_req = None self.enc_res_stats = None
self.set_icap_response(code, message=message) self.set_icap_header(b'Connection', b'close') # TODO: why? yield self.send_headers()
contenttype='text/html'): """Send an encapsulated error reply.
Arguments are the error code, and a detailed message. The detailed message defaults to the short entry matching the response code.
This sends an encapsulated error response (so it must be called before any output has been generated), logs the error, and finally sends a piece of HTML explaining the error to the user. """
# No encapsulation self.enc_req = None
self.set_icap_response(200, message=message) self.set_enc_status('HTTP/1.1 %s %s' % (str(code).encode('utf-8'), message)) self.set_enc_header('Content-Type', contenttype) self.set_enc_header('Content-Length', str(len(body)).encode('utf-8')) yield self.send_headers(has_body=True) if body: yield self.write_chunk(body) yield self.write_chunk('')
"""Log an error.
This is called when a request cannot be fulfilled. """ logger.error({'msg': '%s' % (format_str % args)}, event='pyicap_error')
"""Return the server software version string."""
"""Return the current date and time formatted for a message header.""" self._weekdayname[wd], day, self._monthname[month], year, hh, mm, ss)
"""Return the current time formatted for logging.""" now = time.time() year, month, day, hh, mm, ss, _, _, _ = time.localtime(now) s = "%02d/%3s/%04d %02d:%02d:%02d" % ( day, self._monthname[month], year, hh, mm, ss) return s
def no_adaptation_required(self): """Tells the client to leave the message unaltered
If the client allows 204, or this is a preview request than a 204 preview response is sent. Otherwise a copy of the message is returned to the client. """ # We MUST read everything the client sent us else: # We have to copy everything, # but it's sure there's no preview self.set_icap_response(200)
if self.enc_res_status is not None: self.set_enc_status(b' '.join(self.enc_res_status)) for h in self.enc_res_headers: for v in self.enc_res_headers[h]: self.set_enc_header(h, v)
if not self.has_body: yield self.send_headers(False) return
yield self.send_headers(True) while True: chunk = yield self.read_chunk() yield self.write_chunk(chunk) if chunk == b'': break
def icap_options(self): """Called on OPTIONS icap meesage""" raise NotImplementedError()
def icap_reqmod(self): """Called on REQMOD icap meesage""" raise NotImplementedError()
def icap_respmod(self): """Called on RESPMOD icap meesage""" raise NotImplementedError() |