Viewing file:  http.py (58.05 KB) -rw-r--r--
Select action/file-type:
 (+) |  (+) |  (+) | Code (+) | Session (+) |  (+) | SDB (+) |  (+) |  (+) |  (+) |  (+) |  (+) |
# -*- test-case-name: twisted.web.test.test_http -*-
# Copyright (c) 2001-2010 Twisted Matrix Laboratories.
# See LICENSE for details.
"""
HyperText Transfer Protocol implementation.
This is used by twisted.web.
Future Plans:
- HTTP client support will at some point be refactored to support HTTP/1.1.
- Accept chunked data from clients in server.
- Other missing HTTP features from the RFC.
Maintainer: Itamar Shtull-Trauring
"""
# system imports
from cStringIO import StringIO
import tempfile
import base64, binascii
import cgi
import socket
import math
import time
import calendar
import warnings
import os
from urlparse import urlparse as _urlparse
from zope.interface import implements
# twisted imports
from twisted.internet import interfaces, reactor, protocol, address
from twisted.internet.defer import Deferred
from twisted.protocols import policies, basic
from twisted.python import log
try: # try importing the fast, C version
from twisted.protocols._c_urlarg import unquote
except ImportError:
from urllib import unquote
from twisted.web.http_headers import _DictHeaders, Headers
protocol_version = "HTTP/1.1"
_CONTINUE = 100
SWITCHING = 101
OK = 200
CREATED = 201
ACCEPTED = 202
NON_AUTHORITATIVE_INFORMATION = 203
NO_CONTENT = 204
RESET_CONTENT = 205
PARTIAL_CONTENT = 206
MULTI_STATUS = 207
MULTIPLE_CHOICE = 300
MOVED_PERMANENTLY = 301
FOUND = 302
SEE_OTHER = 303
NOT_MODIFIED = 304
USE_PROXY = 305
TEMPORARY_REDIRECT = 307
BAD_REQUEST = 400
UNAUTHORIZED = 401
PAYMENT_REQUIRED = 402
FORBIDDEN = 403
NOT_FOUND = 404
NOT_ALLOWED = 405
NOT_ACCEPTABLE = 406
PROXY_AUTH_REQUIRED = 407
REQUEST_TIMEOUT = 408
CONFLICT = 409
GONE = 410
LENGTH_REQUIRED = 411
PRECONDITION_FAILED = 412
REQUEST_ENTITY_TOO_LARGE = 413
REQUEST_URI_TOO_LONG = 414
UNSUPPORTED_MEDIA_TYPE = 415
REQUESTED_RANGE_NOT_SATISFIABLE = 416
EXPECTATION_FAILED = 417
INTERNAL_SERVER_ERROR = 500
NOT_IMPLEMENTED = 501
BAD_GATEWAY = 502
SERVICE_UNAVAILABLE = 503
GATEWAY_TIMEOUT = 504
HTTP_VERSION_NOT_SUPPORTED = 505
INSUFFICIENT_STORAGE_SPACE = 507
NOT_EXTENDED = 510
RESPONSES = {
# 100
_CONTINUE: "Continue",
SWITCHING: "Switching Protocols",
# 200
OK: "OK",
CREATED: "Created",
ACCEPTED: "Accepted",
NON_AUTHORITATIVE_INFORMATION: "Non-Authoritative Information",
NO_CONTENT: "No Content",
RESET_CONTENT: "Reset Content.",
PARTIAL_CONTENT: "Partial Content",
MULTI_STATUS: "Multi-Status",
# 300
MULTIPLE_CHOICE: "Multiple Choices",
MOVED_PERMANENTLY: "Moved Permanently",
FOUND: "Found",
SEE_OTHER: "See Other",
NOT_MODIFIED: "Not Modified",
USE_PROXY: "Use Proxy",
# 306 not defined??
TEMPORARY_REDIRECT: "Temporary Redirect",
# 400
BAD_REQUEST: "Bad Request",
UNAUTHORIZED: "Unauthorized",
PAYMENT_REQUIRED: "Payment Required",
FORBIDDEN: "Forbidden",
NOT_FOUND: "Not Found",
NOT_ALLOWED: "Method Not Allowed",
NOT_ACCEPTABLE: "Not Acceptable",
PROXY_AUTH_REQUIRED: "Proxy Authentication Required",
REQUEST_TIMEOUT: "Request Time-out",
CONFLICT: "Conflict",
GONE: "Gone",
LENGTH_REQUIRED: "Length Required",
PRECONDITION_FAILED: "Precondition Failed",
REQUEST_ENTITY_TOO_LARGE: "Request Entity Too Large",
REQUEST_URI_TOO_LONG: "Request-URI Too Long",
UNSUPPORTED_MEDIA_TYPE: "Unsupported Media Type",
REQUESTED_RANGE_NOT_SATISFIABLE: "Requested Range not satisfiable",
EXPECTATION_FAILED: "Expectation Failed",
# 500
INTERNAL_SERVER_ERROR: "Internal Server Error",
NOT_IMPLEMENTED: "Not Implemented",
BAD_GATEWAY: "Bad Gateway",
SERVICE_UNAVAILABLE: "Service Unavailable",
GATEWAY_TIMEOUT: "Gateway Time-out",
HTTP_VERSION_NOT_SUPPORTED: "HTTP Version not supported",
INSUFFICIENT_STORAGE_SPACE: "Insufficient Storage Space",
NOT_EXTENDED: "Not Extended"
}
CACHED = """Magic constant returned by http.Request methods to set cache
validation headers when the request is conditional and the value fails
the condition."""
# backwards compatability
responses = RESPONSES
# datetime parsing and formatting
weekdayname = ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun']
monthname = [None,
'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun',
'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']
weekdayname_lower = [name.lower() for name in weekdayname]
monthname_lower = [name and name.lower() for name in monthname]
def urlparse(url):
"""
Parse an URL into six components.
This is similar to L{urlparse.urlparse}, but rejects C{unicode} input
and always produces C{str} output.
@type url: C{str}
@raise TypeError: The given url was a C{unicode} string instead of a
C{str}.
@rtype: six-tuple of str
@return: The scheme, net location, path, params, query string, and fragment
of the URL.
"""
if isinstance(url, unicode):
raise TypeError("url must be str, not unicode")
scheme, netloc, path, params, query, fragment = _urlparse(url)
if isinstance(scheme, unicode):
scheme = scheme.encode('ascii')
netloc = netloc.encode('ascii')
path = path.encode('ascii')
query = query.encode('ascii')
fragment = fragment.encode('ascii')
return scheme, netloc, path, params, query, fragment
def parse_qs(qs, keep_blank_values=0, strict_parsing=0, unquote=unquote):
"""
like cgi.parse_qs, only with custom unquote function
"""
d = {}
items = [s2 for s1 in qs.split("&") for s2 in s1.split(";")]
for item in items:
try:
k, v = item.split("=", 1)
except ValueError:
if strict_parsing:
raise
continue
if v or keep_blank_values:
k = unquote(k.replace("+", " "))
v = unquote(v.replace("+", " "))
if k in d:
d[k].append(v)
else:
d[k] = [v]
return d
def datetimeToString(msSinceEpoch=None):
"""
Convert seconds since epoch to HTTP datetime string.
"""
if msSinceEpoch == None:
msSinceEpoch = time.time()
year, month, day, hh, mm, ss, wd, y, z = time.gmtime(msSinceEpoch)
s = "%s, %02d %3s %4d %02d:%02d:%02d GMT" % (
weekdayname[wd],
day, monthname[month], year,
hh, mm, ss)
return s
def datetimeToLogString(msSinceEpoch=None):
"""
Convert seconds since epoch to log datetime string.
"""
if msSinceEpoch == None:
msSinceEpoch = time.time()
year, month, day, hh, mm, ss, wd, y, z = time.gmtime(msSinceEpoch)
s = "[%02d/%3s/%4d:%02d:%02d:%02d +0000]" % (
day, monthname[month], year,
hh, mm, ss)
return s
# a hack so we don't need to recalculate log datetime every hit,
# at the price of a small, unimportant, inaccuracy.
_logDateTime = None
_logDateTimeUsers = 0
_resetLogDateTimeID = None
def _resetLogDateTime():
global _logDateTime
global _resetLogDateTime
global _resetLogDateTimeID
_logDateTime = datetimeToLogString()
_resetLogDateTimeID = reactor.callLater(1, _resetLogDateTime)
def _logDateTimeStart():
global _logDateTimeUsers
if not _logDateTimeUsers:
_resetLogDateTime()
_logDateTimeUsers += 1
def _logDateTimeStop():
global _logDateTimeUsers
_logDateTimeUsers -= 1;
if (not _logDateTimeUsers and _resetLogDateTimeID
and _resetLogDateTimeID.active()):
_resetLogDateTimeID.cancel()
def timegm(year, month, day, hour, minute, second):
"""
Convert time tuple in GMT to seconds since epoch, GMT
"""
EPOCH = 1970
if year < EPOCH:
raise ValueError("Years prior to %d not supported" % (EPOCH,))
assert 1 <= month <= 12
days = 365*(year-EPOCH) + calendar.leapdays(EPOCH, year)
for i in range(1, month):
days = days + calendar.mdays[i]
if month > 2 and calendar.isleap(year):
days = days + 1
days = days + day - 1
hours = days*24 + hour
minutes = hours*60 + minute
seconds = minutes*60 + second
return seconds
def stringToDatetime(dateString):
"""
Convert an HTTP date string (one of three formats) to seconds since epoch.
"""
parts = dateString.split()
if not parts[0][0:3].lower() in weekdayname_lower:
# Weekday is stupid. Might have been omitted.
try:
return stringToDatetime("Sun, "+dateString)
except ValueError:
# Guess not.
pass
partlen = len(parts)
if (partlen == 5 or partlen == 6) and parts[1].isdigit():
# 1st date format: Sun, 06 Nov 1994 08:49:37 GMT
# (Note: "GMT" is literal, not a variable timezone)
# (also handles without "GMT")
# This is the normal format
day = parts[1]
month = parts[2]
year = parts[3]
time = parts[4]
elif (partlen == 3 or partlen == 4) and parts[1].find('-') != -1:
# 2nd date format: Sunday, 06-Nov-94 08:49:37 GMT
# (Note: "GMT" is literal, not a variable timezone)
# (also handles without without "GMT")
# Two digit year, yucko.
day, month, year = parts[1].split('-')
time = parts[2]
year=int(year)
if year < 69:
year = year + 2000
elif year < 100:
year = year + 1900
elif len(parts) == 5:
# 3rd date format: Sun Nov 6 08:49:37 1994
# ANSI C asctime() format.
day = parts[2]
month = parts[1]
year = parts[4]
time = parts[3]
else:
raise ValueError("Unknown datetime format %r" % dateString)
day = int(day)
month = int(monthname_lower.index(month.lower()))
year = int(year)
hour, min, sec = map(int, time.split(':'))
return int(timegm(year, month, day, hour, min, sec))
def toChunk(data):
"""
Convert string to a chunk.
@returns: a tuple of strings representing the chunked encoding of data
"""
return ("%x\r\n" % len(data), data, "\r\n")
def fromChunk(data):
"""
Convert chunk to string.
@returns: tuple (result, remaining), may raise ValueError.
"""
prefix, rest = data.split('\r\n', 1)
length = int(prefix, 16)
if length < 0:
raise ValueError("Chunk length must be >= 0, not %d" % (length,))
if not rest[length:length + 2] == '\r\n':
raise ValueError, "chunk must end with CRLF"
return rest[:length], rest[length + 2:]
def parseContentRange(header):
"""
Parse a content-range header into (start, end, realLength).
realLength might be None if real length is not known ('*').
"""
kind, other = header.strip().split()
if kind.lower() != "bytes":
raise ValueError, "a range of type %r is not supported"
startend, realLength = other.split("/")
start, end = map(int, startend.split("-"))
if realLength == "*":
realLength = None
else:
realLength = int(realLength)
return (start, end, realLength)
class StringTransport:
"""
I am a StringIO wrapper that conforms for the transport API. I support
the `writeSequence' method.
"""
def __init__(self):
self.s = StringIO()
def writeSequence(self, seq):
self.s.write(''.join(seq))
def __getattr__(self, attr):
return getattr(self.__dict__['s'], attr)
class HTTPClient(basic.LineReceiver):
"""
A client for HTTP 1.0.
Notes:
You probably want to send a 'Host' header with the name of the site you're
connecting to, in order to not break name based virtual hosting.
@ivar length: The length of the request body in bytes.
@type length: C{int}
@ivar firstLine: Are we waiting for the first header line?
@type firstLine: C{bool}
@ivar __buffer: The buffer that stores the response to the HTTP request.
@type __buffer: A C{StringIO} object.
@ivar _header: Part or all of an HTTP request header.
@type _header: C{str}
"""
length = None
firstLine = True
__buffer = None
_header = ""
def sendCommand(self, command, path):
self.transport.write('%s %s HTTP/1.0\r\n' % (command, path))
def sendHeader(self, name, value):
self.transport.write('%s: %s\r\n' % (name, value))
def endHeaders(self):
self.transport.write('\r\n')
def extractHeader(self, header):
"""
Given a complete HTTP header, extract the field name and value and
process the header.
@param header: a complete HTTP request header of the form
'field-name: value'.
@type header: C{str}
"""
key, val = header.split(':', 1)
val = val.lstrip()
self.handleHeader(key, val)
if key.lower() == 'content-length':
self.length = int(val)
def lineReceived(self, line):
"""
Parse the status line and headers for an HTTP request.
@param line: Part of an HTTP request header. Request bodies are parsed
in L{rawDataReceived}.
@type line: C{str}
"""
if self.firstLine:
self.firstLine = False
l = line.split(None, 2)
version = l[0]
status = l[1]
try:
message = l[2]
except IndexError:
# sometimes there is no message
message = ""
self.handleStatus(version, status, message)
return
if not line:
if self._header != "":
# Only extract headers if there are any
self.extractHeader(self._header)
self.__buffer = StringIO()
self.handleEndHeaders()
self.setRawMode()
return
if line.startswith('\t') or line.startswith(' '):
# This line is part of a multiline header. According to RFC 822, in
# "unfolding" multiline headers you do not strip the leading
# whitespace on the continuing line.
self._header = self._header + line
elif self._header:
# This line starts a new header, so process the previous one.
self.extractHeader(self._header)
self._header = line
else: # First header
self._header = line
def connectionLost(self, reason):
self.handleResponseEnd()
def handleResponseEnd(self):
"""
The response has been completely received.
This callback may be invoked more than once per request.
"""
if self.__buffer is not None:
b = self.__buffer.getvalue()
self.__buffer = None
self.handleResponse(b)
def handleResponsePart(self, data):
self.__buffer.write(data)
def connectionMade(self):
pass
def handleStatus(self, version, status, message):
"""
Called when the status-line is received.
@param version: e.g. 'HTTP/1.0'
@param status: e.g. '200'
@type status: C{str}
@param message: e.g. 'OK'
"""
def handleHeader(self, key, val):
"""
Called every time a header is received.
"""
def handleEndHeaders(self):
"""
Called when all headers have been received.
"""
def rawDataReceived(self, data):
if self.length is not None:
data, rest = data[:self.length], data[self.length:]
self.length -= len(data)
else:
rest = ''
self.handleResponsePart(data)
if self.length == 0:
self.handleResponseEnd()
self.setLineMode(rest)
# response codes that must have empty bodies
NO_BODY_CODES = (204, 304)
class Request:
"""
A HTTP request.
Subclasses should override the process() method to determine how
the request will be processed.
@ivar method: The HTTP method that was used.
@ivar uri: The full URI that was requested (includes arguments).
@ivar path: The path only (arguments not included).
@ivar args: All of the arguments, including URL and POST arguments.
@type args: A mapping of strings (the argument names) to lists of values.
i.e., ?foo=bar&foo=baz&quux=spam results in
{'foo': ['bar', 'baz'], 'quux': ['spam']}.
@type requestHeaders: L{http_headers.Headers}
@ivar requestHeaders: All received HTTP request headers.
@ivar received_headers: Backwards-compatibility access to
C{requestHeaders}. Use C{requestHeaders} instead. C{received_headers}
behaves mostly like a C{dict} and does not provide access to all header
values.
@type responseHeaders: L{http_headers.Headers}
@ivar responseHeaders: All HTTP response headers to be sent.
@ivar headers: Backwards-compatibility access to C{responseHeaders}. Use
C{responseHeaders} instead. C{headers} behaves mostly like a C{dict}
and does not provide access to all header values nor does it allow
multiple values for one header to be set.
@ivar notifications: A C{list} of L{Deferred}s which are waiting for
notification that the response to this request has been finished
(successfully or with an error). Don't use this attribute directly,
instead use the L{Request.notifyFinish} method.
@ivar _disconnected: A flag which is C{False} until the connection over
which this request was received is closed and which is C{True} after
that.
@type _disconnected: C{bool}
"""
implements(interfaces.IConsumer)
producer = None
finished = 0
code = OK
code_message = RESPONSES[OK]
method = "(no method yet)"
clientproto = "(no clientproto yet)"
uri = "(no uri yet)"
startedWriting = 0
chunked = 0
sentLength = 0 # content-length of response, or total bytes sent via chunking
etag = None
lastModified = None
args = None
path = None
content = None
_forceSSL = 0
_disconnected = False
def __init__(self, channel, queued):
"""
@param channel: the channel we're connected to.
@param queued: are we in the request queue, or can we start writing to
the transport?
"""
self.notifications = []
self.channel = channel
self.queued = queued
self.requestHeaders = Headers()
self.received_cookies = {}
self.responseHeaders = Headers()
self.cookies = [] # outgoing cookies
if queued:
self.transport = StringTransport()
else:
self.transport = self.channel.transport
def __setattr__(self, name, value):
"""
Support assignment of C{dict} instances to C{received_headers} for
backwards-compatibility.
"""
if name == 'received_headers':
# A property would be nice, but Request is classic.
self.requestHeaders = headers = Headers()
for k, v in value.iteritems():
headers.setRawHeaders(k, [v])
elif name == 'requestHeaders':
self.__dict__[name] = value
self.__dict__['received_headers'] = _DictHeaders(value)
elif name == 'headers':
self.responseHeaders = headers = Headers()
for k, v in value.iteritems():
headers.setRawHeaders(k, [v])
elif name == 'responseHeaders':
self.__dict__[name] = value
self.__dict__['headers'] = _DictHeaders(value)
else:
self.__dict__[name] = value
def _cleanup(self):
"""
Called when have finished responding and are no longer queued.
"""
if self.producer:
log.err(RuntimeError("Producer was not unregistered for %s" % self.uri))
self.unregisterProducer()
self.channel.requestDone(self)
del self.channel
try:
self.content.close()
except OSError:
# win32 suckiness, no idea why it does this
pass
del self.content
for d in self.notifications:
d.callback(None)
self.notifications = []
# methods for channel - end users should not use these
def noLongerQueued(self):
"""
Notify the object that it is no longer queued.
We start writing whatever data we have to the transport, etc.
This method is not intended for users.
"""
if not self.queued:
raise RuntimeError, "noLongerQueued() got called unnecessarily."
self.queued = 0
# set transport to real one and send any buffer data
data = self.transport.getvalue()
self.transport = self.channel.transport
if data:
self.transport.write(data)
# if we have producer, register it with transport
if (self.producer is not None) and not self.finished:
self.transport.registerProducer(self.producer, self.streamingProducer)
# if we're finished, clean up
if self.finished:
self._cleanup()
def gotLength(self, length):
"""
Called when HTTP channel got length of content in this request.
This method is not intended for users.
@param length: The length of the request body, as indicated by the
request headers. C{None} if the request headers do not indicate a
length.
"""
if length is not None and length < 100000:
self.content = StringIO()
else:
self.content = tempfile.TemporaryFile()
def parseCookies(self):
"""
Parse cookie headers.
This method is not intended for users.
"""
cookieheaders = self.requestHeaders.getRawHeaders("cookie")
if cookieheaders is None:
return
for cookietxt in cookieheaders:
if cookietxt:
for cook in cookietxt.split(';'):
cook = cook.lstrip()
try:
k, v = cook.split('=', 1)
self.received_cookies[k] = v
except ValueError:
pass
def handleContentChunk(self, data):
"""
Write a chunk of data.
This method is not intended for users.
"""
self.content.write(data)
def requestReceived(self, command, path, version):
"""
Called by channel when all data has been received.
This method is not intended for users.
@type command: C{str}
@param command: The HTTP verb of this request. This has the case
supplied by the client (eg, it maybe "get" rather than "GET").
@type path: C{str}
@param path: The URI of this request.
@type version: C{str}
@param version: The HTTP version of this request.
"""
self.content.seek(0,0)
self.args = {}
self.stack = []
self.method, self.uri = command, path
self.clientproto = version
x = self.uri.split('?', 1)
if len(x) == 1:
self.path = self.uri
else:
self.path, argstring = x
self.args = parse_qs(argstring, 1)
# cache the client and server information, we'll need this later to be
# serialized and sent with the request so CGIs will work remotely
self.client = self.channel.transport.getPeer()
self.host = self.channel.transport.getHost()
# Argument processing
args = self.args
ctype = self.requestHeaders.getRawHeaders('content-type')
if ctype is not None:
ctype = ctype[0]
if self.method == "POST" and ctype:
mfd = 'multipart/form-data'
key, pdict = cgi.parse_header(ctype)
if key == 'application/x-www-form-urlencoded':
args.update(parse_qs(self.content.read(), 1))
elif key == mfd:
try:
args.update(cgi.parse_multipart(self.content, pdict))
except KeyError, e:
if e.args[0] == 'content-disposition':
# Parse_multipart can't cope with missing
# content-dispostion headers in multipart/form-data
# parts, so we catch the exception and tell the client
# it was a bad request.
self.channel.transport.write(
"HTTP/1.1 400 Bad Request\r\n\r\n")
self.channel.transport.loseConnection()
return
raise
self.content.seek(0, 0)
self.process()
def __repr__(self):
return '<%s %s %s>'% (self.method, self.uri, self.clientproto)
def process(self):
"""
Override in subclasses.
This method is not intended for users.
"""
pass
# consumer interface
def registerProducer(self, producer, streaming):
"""
Register a producer.
"""
if self.producer:
raise ValueError, "registering producer %s before previous one (%s) was unregistered" % (producer, self.producer)
self.streamingProducer = streaming
self.producer = producer
if self.queued:
if streaming:
producer.pauseProducing()
else:
self.transport.registerProducer(producer, streaming)
def unregisterProducer(self):
"""
Unregister the producer.
"""
if not self.queued:
self.transport.unregisterProducer()
self.producer = None
# private http response methods
def _sendError(self, code, resp=''):
self.transport.write('%s %s %s\r\n\r\n' % (self.clientproto, code, resp))
# The following is the public interface that people should be
# writing to.
def getHeader(self, key):
"""
Get an HTTP request header.
@type key: C{str}
@param key: The name of the header to get the value of.
@rtype: C{str} or C{NoneType}
@return: The value of the specified header, or C{None} if that header
was not present in the request.
"""
value = self.requestHeaders.getRawHeaders(key)
if value is not None:
return value[-1]
def getCookie(self, key):
"""
Get a cookie that was sent from the network.
"""
return self.received_cookies.get(key)
def notifyFinish(self):
"""
Notify when the response to this request has finished.
@rtype: L{Deferred}
@return: A L{Deferred} which will be triggered when the request is
finished -- with a C{None} value if the request finishes
successfully or with an error if the request is interrupted by an
error (for example, the client closing the connection prematurely).
"""
self.notifications.append(Deferred())
return self.notifications[-1]
def finish(self):
"""
Indicate that all response data has been written to this L{Request}.
"""
if self._disconnected:
raise RuntimeError(
"Request.finish called on a request after its connection was lost; "
"use Request.notifyFinish to keep track of this.")
if self.finished:
warnings.warn("Warning! request.finish called twice.", stacklevel=2)
return
if not self.startedWriting:
# write headers
self.write('')
if self.chunked:
# write last chunk and closing CRLF
self.transport.write("0\r\n\r\n")
# log request
if hasattr(self.channel, "factory"):
self.channel.factory.log(self)
self.finished = 1
if not self.queued:
self._cleanup()
def write(self, data):
"""
Write some data as a result of an HTTP request. The first
time this is called, it writes out response data.
@type data: C{str}
@param data: Some bytes to be sent as part of the response body.
"""
if not self.startedWriting:
self.startedWriting = 1
version = self.clientproto
l = []
l.append('%s %s %s\r\n' % (version, self.code,
self.code_message))
# if we don't have a content length, we send data in
# chunked mode, so that we can support pipelining in
# persistent connections.
if ((version == "HTTP/1.1") and
(self.responseHeaders.getRawHeaders('content-length') is None) and
self.method != "HEAD" and self.code not in NO_BODY_CODES):
l.append("%s: %s\r\n" % ('Transfer-Encoding', 'chunked'))
self.chunked = 1
if self.lastModified is not None:
if self.responseHeaders.hasHeader('last-modified'):
log.msg("Warning: last-modified specified both in"
" header list and lastModified attribute.")
else:
self.responseHeaders.setRawHeaders(
'last-modified',
[datetimeToString(self.lastModified)])
if self.etag is not None:
self.responseHeaders.setRawHeaders('ETag', [self.etag])
for name, values in self.responseHeaders.getAllRawHeaders():
for value in values:
l.append("%s: %s\r\n" % (name, value))
for cookie in self.cookies:
l.append('%s: %s\r\n' % ("Set-Cookie", cookie))
l.append("\r\n")
self.transport.writeSequence(l)
# if this is a "HEAD" request, we shouldn't return any data
if self.method == "HEAD":
self.write = lambda data: None
return
# for certain result codes, we should never return any data
if self.code in NO_BODY_CODES:
self.write = lambda data: None
return
self.sentLength = self.sentLength + len(data)
if data:
if self.chunked:
self.transport.writeSequence(toChunk(data))
else:
self.transport.write(data)
def addCookie(self, k, v, expires=None, domain=None, path=None, max_age=None, comment=None, secure=None):
"""
Set an outgoing HTTP cookie.
In general, you should consider using sessions instead of cookies, see
L{twisted.web.server.Request.getSession} and the
L{twisted.web.server.Session} class for details.
"""
cookie = '%s=%s' % (k, v)
if expires is not None:
cookie = cookie +"; Expires=%s" % expires
if domain is not None:
cookie = cookie +"; Domain=%s" % domain
if path is not None:
cookie = cookie +"; Path=%s" % path
if max_age is not None:
cookie = cookie +"; Max-Age=%s" % max_age
if comment is not None:
cookie = cookie +"; Comment=%s" % comment
if secure:
cookie = cookie +"; Secure"
self.cookies.append(cookie)
def setResponseCode(self, code, message=None):
"""
Set the HTTP response code.
"""
if not isinstance(code, (int, long)):
raise TypeError("HTTP response code must be int or long")
self.code = code
if message:
self.code_message = message
else:
self.code_message = RESPONSES.get(code, "Unknown Status")
def setHeader(self, name, value):
"""
Set an HTTP response header. Overrides any previously set values for
this header.
@type name: C{str}
@param name: The name of the header for which to set the value.
@type value: C{str}
@param value: The value to set for the named header.
"""
self.responseHeaders.setRawHeaders(name, [value])
def redirect(self, url):
"""
Utility function that does a redirect.
The request should have finish() called after this.
"""
self.setResponseCode(FOUND)
self.setHeader("location", url)
def setLastModified(self, when):
"""
Set the C{Last-Modified} time for the response to this request.
If I am called more than once, I ignore attempts to set
Last-Modified earlier, only replacing the Last-Modified time
if it is to a later value.
If I am a conditional request, I may modify my response code
to L{NOT_MODIFIED} if appropriate for the time given.
@param when: The last time the resource being returned was
modified, in seconds since the epoch.
@type when: number
@return: If I am a C{If-Modified-Since} conditional request and
the time given is not newer than the condition, I return
L{http.CACHED<CACHED>} to indicate that you should write no
body. Otherwise, I return a false value.
"""
# time.time() may be a float, but the HTTP-date strings are
# only good for whole seconds.
when = long(math.ceil(when))
if (not self.lastModified) or (self.lastModified < when):
self.lastModified = when
modifiedSince = self.getHeader('if-modified-since')
if modifiedSince:
firstPart = modifiedSince.split(';', 1)[0]
try:
modifiedSince = stringToDatetime(firstPart)
except ValueError:
return None
if modifiedSince >= when:
self.setResponseCode(NOT_MODIFIED)
return CACHED
return None
def setETag(self, etag):
"""
Set an C{entity tag} for the outgoing response.
That's \"entity tag\" as in the HTTP/1.1 C{ETag} header, \"used
for comparing two or more entities from the same requested
resource.\"
If I am a conditional request, I may modify my response code
to L{NOT_MODIFIED} or L{PRECONDITION_FAILED}, if appropriate
for the tag given.
@param etag: The entity tag for the resource being returned.
@type etag: string
@return: If I am a C{If-None-Match} conditional request and
the tag matches one in the request, I return
L{http.CACHED<CACHED>} to indicate that you should write
no body. Otherwise, I return a false value.
"""
if etag:
self.etag = etag
tags = self.getHeader("if-none-match")
if tags:
tags = tags.split()
if (etag in tags) or ('*' in tags):
self.setResponseCode(((self.method in ("HEAD", "GET"))
and NOT_MODIFIED)
or PRECONDITION_FAILED)
return CACHED
return None
def getAllHeaders(self):
"""
Return dictionary mapping the names of all received headers to the last
value received for each.
Since this method does not return all header information,
C{self.requestHeaders.getAllRawHeaders()} may be preferred.
"""
headers = {}
for k, v in self.requestHeaders.getAllRawHeaders():
headers[k.lower()] = v[-1]
return headers
def getRequestHostname(self):
"""
Get the hostname that the user passed in to the request.
This will either use the Host: header (if it is available) or the
host we are listening on if the header is unavailable.
@returns: the requested hostname
@rtype: C{str}
"""
# XXX This method probably has no unit tests. I changed it a ton and
# nothing failed.
host = self.getHeader('host')
if host:
return host.split(':', 1)[0]
return self.getHost().host
def getHost(self):
"""
Get my originally requesting transport's host.
Don't rely on the 'transport' attribute, since Request objects may be
copied remotely. For information on this method's return value, see
twisted.internet.tcp.Port.
"""
return self.host
def setHost(self, host, port, ssl=0):
"""
Change the host and port the request thinks it's using.
This method is useful for working with reverse HTTP proxies (e.g.
both Squid and Apache's mod_proxy can do this), when the address
the HTTP client is using is different than the one we're listening on.
For example, Apache may be listening on https://www.example.com, and then
forwarding requests to http://localhost:8080, but we don't want HTML produced
by Twisted to say 'http://localhost:8080', they should say 'https://www.example.com',
so we do::
request.setHost('www.example.com', 443, ssl=1)
@type host: C{str}
@param host: The value to which to change the host header.
@type ssl: C{bool}
@param ssl: A flag which, if C{True}, indicates that the request is
considered secure (if C{True}, L{isSecure} will return C{True}).
"""
self._forceSSL = ssl
self.requestHeaders.setRawHeaders("host", [host])
self.host = address.IPv4Address("TCP", host, port)
def getClientIP(self):
"""
Return the IP address of the client who submitted this request.
@returns: the client IP address
@rtype: C{str}
"""
if isinstance(self.client, address.IPv4Address):
return self.client.host
else:
return None
def isSecure(self):
"""
Return True if this request is using a secure transport.
Normally this method returns True if this request's HTTPChannel
instance is using a transport that implements ISSLTransport.
This will also return True if setHost() has been called
with ssl=True.
@returns: True if this request is secure
@rtype: C{bool}
"""
if self._forceSSL:
return True
transport = getattr(getattr(self, 'channel', None), 'transport', None)
if interfaces.ISSLTransport(transport, None) is not None:
return True
return False
def _authorize(self):
# Authorization, (mostly) per the RFC
try:
authh = self.getHeader("Authorization")
if not authh:
self.user = self.password = ''
return
bas, upw = authh.split()
if bas.lower() != "basic":
raise ValueError
upw = base64.decodestring(upw)
self.user, self.password = upw.split(':', 1)
except (binascii.Error, ValueError):
self.user = self.password = ""
except:
log.err()
self.user = self.password = ""
def getUser(self):
"""
Return the HTTP user sent with this request, if any.
If no user was supplied, return the empty string.
@returns: the HTTP user, if any
@rtype: C{str}
"""
try:
return self.user
except:
pass
self._authorize()
return self.user
def getPassword(self):
"""
Return the HTTP password sent with this request, if any.
If no password was supplied, return the empty string.
@returns: the HTTP password, if any
@rtype: C{str}
"""
try:
return self.password
except:
pass
self._authorize()
return self.password
def getClient(self):
if self.client.type != 'TCP':
return None
host = self.client.host
try:
name, names, addresses = socket.gethostbyaddr(host)
except socket.error:
return host
names.insert(0, name)
for name in names:
if '.' in name:
return name
return names[0]
def connectionLost(self, reason):
"""
There is no longer a connection for this request to respond over.
Clean up anything which can't be useful anymore.
"""
self._disconnected = True
self.channel = None
if self.content is not None:
self.content.close()
for d in self.notifications:
d.errback(reason)
self.notifications = []
class _DataLoss(Exception):
"""
L{_DataLoss} indicates that not all of a message body was received. This
is only one of several possible exceptions which may indicate that data
was lost. Because of this, it should not be checked for by
specifically; any unexpected exception should be treated as having
caused data loss.
"""
class PotentialDataLoss(Exception):
"""
L{PotentialDataLoss} may be raised by a transfer encoding decoder's
C{noMoreData} method to indicate that it cannot be determined if the
entire response body has been delivered. This only occurs when making
requests to HTTP servers which do not set I{Content-Length} or a
I{Transfer-Encoding} in the response because in this case the end of the
response is indicated by the connection being closed, an event which may
also be due to a transient network problem or other error.
"""
class _IdentityTransferDecoder(object):
"""
Protocol for accumulating bytes up to a specified length. This handles the
case where no I{Transfer-Encoding} is specified.
@ivar contentLength: Counter keeping track of how many more bytes there are
to receive.
@ivar dataCallback: A one-argument callable which will be invoked each
time application data is received.
@ivar finishCallback: A one-argument callable which will be invoked when
the terminal chunk is received. It will be invoked with all bytes
which were delivered to this protocol which came after the terminal
chunk.
"""
def __init__(self, contentLength, dataCallback, finishCallback):
self.contentLength = contentLength
self.dataCallback = dataCallback
self.finishCallback = finishCallback
def dataReceived(self, data):
"""
Interpret the next chunk of bytes received. Either deliver them to the
data callback or invoke the finish callback if enough bytes have been
received.
@raise RuntimeError: If the finish callback has already been invoked
during a previous call to this methood.
"""
if self.dataCallback is None:
raise RuntimeError(
"_IdentityTransferDecoder cannot decode data after finishing")
if self.contentLength is None:
self.dataCallback(data)
elif len(data) < self.contentLength:
self.contentLength -= len(data)
self.dataCallback(data)
else:
# Make the state consistent before invoking any code belonging to
# anyone else in case noMoreData ends up being called beneath this
# stack frame.
contentLength = self.contentLength
dataCallback = self.dataCallback
finishCallback = self.finishCallback
self.dataCallback = self.finishCallback = None
self.contentLength = 0
dataCallback(data[:contentLength])
finishCallback(data[contentLength:])
def noMoreData(self):
"""
All data which will be delivered to this decoder has been. Check to
make sure as much data as was expected has been received.
@raise PotentialDataLoss: If the content length is unknown.
@raise _DataLoss: If the content length is known and fewer than that
many bytes have been delivered.
@return: C{None}
"""
finishCallback = self.finishCallback
self.dataCallback = self.finishCallback = None
if self.contentLength is None:
finishCallback('')
raise PotentialDataLoss()
elif self.contentLength != 0:
raise _DataLoss()
class _ChunkedTransferDecoder(object):
"""
Protocol for decoding I{chunked} Transfer-Encoding, as defined by RFC 2616,
section 3.6.1. This protocol can interpret the contents of a request or
response body which uses the I{chunked} Transfer-Encoding. It cannot
interpret any of the rest of the HTTP protocol.
It may make sense for _ChunkedTransferDecoder to be an actual IProtocol
implementation. Currently, the only user of this class will only ever
call dataReceived on it. However, it might be an improvement if the
user could connect this to a transport and deliver connection lost
notification. This way, `dataCallback` becomes `self.transport.write`
and perhaps `finishCallback` becomes `self.transport.loseConnection()`
(although I'm not sure where the extra data goes in that case). This
could also allow this object to indicate to the receiver of data that
the stream was not completely received, an error case which should be
noticed. -exarkun
@ivar dataCallback: A one-argument callable which will be invoked each
time application data is received.
@ivar finishCallback: A one-argument callable which will be invoked when
the terminal chunk is received. It will be invoked with all bytes
which were delivered to this protocol which came after the terminal
chunk.
@ivar length: Counter keeping track of how many more bytes in a chunk there
are to receive.
@ivar state: One of C{'chunk-length'}, C{'trailer'}, C{'body'}, or
C{'finished'}. For C{'chunk-length'}, data for the chunk length line
is currently being read. For C{'trailer'}, the CR LF pair which
follows each chunk is being read. For C{'body'}, the contents of a
chunk are being read. For C{'finished'}, the last chunk has been
completely read and no more input is valid.
@ivar finish: A flag indicating that the last chunk has been started. When
it finishes, the state will change to C{'finished'} and no more data
will be accepted.
"""
state = 'chunk-length'
finish = False
def __init__(self, dataCallback, finishCallback):
self.dataCallback = dataCallback
self.finishCallback = finishCallback
self._buffer = ''
def dataReceived(self, data):
"""
Interpret data from a request or response body which uses the
I{chunked} Transfer-Encoding.
"""
data = self._buffer + data
self._buffer = ''
while data:
if self.state == 'chunk-length':
if '\r\n' in data:
line, rest = data.split('\r\n', 1)
parts = line.split(';')
self.length = int(parts[0], 16)
if self.length == 0:
self.state = 'trailer'
self.finish = True
else:
self.state = 'body'
data = rest
else:
self._buffer = data
data = ''
elif self.state == 'trailer':
if data.startswith('\r\n'):
data = data[2:]
if self.finish:
self.state = 'finished'
self.finishCallback(data)
data = ''
else:
self.state = 'chunk-length'
else:
self._buffer = data
data = ''
elif self.state == 'body':
if len(data) >= self.length:
chunk, data = data[:self.length], data[self.length:]
self.dataCallback(chunk)
self.state = 'trailer'
elif len(data) < self.length:
self.length -= len(data)
self.dataCallback(data)
data = ''
elif self.state == 'finished':
raise RuntimeError(
"_ChunkedTransferDecoder.dataReceived called after last "
"chunk was processed")
def noMoreData(self):
"""
Verify that all data has been received. If it has not been, raise
L{_DataLoss}.
"""
if self.state != 'finished':
raise _DataLoss(
"Chunked decoder in %r state, still expecting more data to "
"get to finished state." % (self.state,))
class HTTPChannel(basic.LineReceiver, policies.TimeoutMixin):
"""
A receiver for HTTP requests.
@ivar _transferDecoder: C{None} or an instance of
L{_ChunkedTransferDecoder} if the request body uses the I{chunked}
Transfer-Encoding.
"""
maxHeaders = 500 # max number of headers allowed per request
length = 0
persistent = 1
__header = ''
__first_line = 1
__content = None
# set in instances or subclasses
requestFactory = Request
_savedTimeOut = None
_receivedHeaderCount = 0
def __init__(self):
# the request queue
self.requests = []
self._transferDecoder = None
def connectionMade(self):
self.setTimeout(self.timeOut)
def lineReceived(self, line):
self.resetTimeout()
if self.__first_line:
# if this connection is not persistent, drop any data which
# the client (illegally) sent after the last request.
if not self.persistent:
self.dataReceived = self.lineReceived = lambda *args: None
return
# IE sends an extraneous empty line (\r\n) after a POST request;
# eat up such a line, but only ONCE
if not line and self.__first_line == 1:
self.__first_line = 2
return
# create a new Request object
request = self.requestFactory(self, len(self.requests))
self.requests.append(request)
self.__first_line = 0
parts = line.split()
if len(parts) != 3:
self.transport.write("HTTP/1.1 400 Bad Request\r\n\r\n")
self.transport.loseConnection()
return
command, request, version = parts
self._command = command
self._path = request
self._version = version
elif line == '':
if self.__header:
self.headerReceived(self.__header)
self.__header = ''
self.allHeadersReceived()
if self.length == 0:
self.allContentReceived()
else:
self.setRawMode()
elif line[0] in ' \t':
self.__header = self.__header+'\n'+line
else:
if self.__header:
self.headerReceived(self.__header)
self.__header = line
def _finishRequestBody(self, data):
self.allContentReceived()
self.setLineMode(data)
def headerReceived(self, line):
"""
Do pre-processing (for content-length) and store this header away.
Enforce the per-request header limit.
@type line: C{str}
@param line: A line from the header section of a request, excluding the
line delimiter.
"""
header, data = line.split(':', 1)
header = header.lower()
data = data.strip()
if header == 'content-length':
self.length = int(data)
self._transferDecoder = _IdentityTransferDecoder(
self.length, self.requests[-1].handleContentChunk, self._finishRequestBody)
elif header == 'transfer-encoding' and data.lower() == 'chunked':
self.length = None
self._transferDecoder = _ChunkedTransferDecoder(
self.requests[-1].handleContentChunk, self._finishRequestBody)
reqHeaders = self.requests[-1].requestHeaders
values = reqHeaders.getRawHeaders(header)
if values is not None:
values.append(data)
else:
reqHeaders.setRawHeaders(header, [data])
self._receivedHeaderCount += 1
if self._receivedHeaderCount > self.maxHeaders:
self.transport.write("HTTP/1.1 400 Bad Request\r\n\r\n")
self.transport.loseConnection()
def allContentReceived(self):
command = self._command
path = self._path
version = self._version
# reset ALL state variables, so we don't interfere with next request
self.length = 0
self._receivedHeaderCount = 0
self.__first_line = 1
self._transferDecoder = None
del self._command, self._path, self._version
# Disable the idle timeout, in case this request takes a long
# time to finish generating output.
if self.timeOut:
self._savedTimeOut = self.setTimeout(None)
req = self.requests[-1]
req.requestReceived(command, path, version)
def rawDataReceived(self, data):
self.resetTimeout()
self._transferDecoder.dataReceived(data)
def allHeadersReceived(self):
req = self.requests[-1]
req.parseCookies()
self.persistent = self.checkPersistence(req, self._version)
req.gotLength(self.length)
def checkPersistence(self, request, version):
"""
Check if the channel should close or not.
@param request: The request most recently received over this channel
against which checks will be made to determine if this connection
can remain open after a matching response is returned.
@type version: C{str}
@param version: The version of the request.
@rtype: C{bool}
@return: A flag which, if C{True}, indicates that this connection may
remain open to receive another request; if C{False}, the connection
must be closed in order to indicate the completion of the response
to C{request}.
"""
connection = request.requestHeaders.getRawHeaders('connection')
if connection:
tokens = map(str.lower, connection[0].split(' '))
else:
tokens = []
# HTTP 1.0 persistent connection support is currently disabled,
# since we need a way to disable pipelining. HTTP 1.0 can't do
# pipelining since we can't know in advance if we'll have a
# content-length header, if we don't have the header we need to close the
# connection. In HTTP 1.1 this is not an issue since we use chunked
# encoding if content-length is not available.
#if version == "HTTP/1.0":
# if 'keep-alive' in tokens:
# request.setHeader('connection', 'Keep-Alive')
# return 1
# else:
# return 0
if version == "HTTP/1.1":
if 'close' in tokens:
request.responseHeaders.setRawHeaders('connection', ['close'])
return False
else:
return True
else:
return False
def requestDone(self, request):
"""
Called by first request in queue when it is done.
"""
if request != self.requests[0]: raise TypeError
del self.requests[0]
if self.persistent:
# notify next request it can start writing
if self.requests:
self.requests[0].noLongerQueued()
else:
if self._savedTimeOut:
self.setTimeout(self._savedTimeOut)
else:
self.transport.loseConnection()
def timeoutConnection(self):
log.msg("Timing out client: %s" % str(self.transport.getPeer()))
policies.TimeoutMixin.timeoutConnection(self)
def connectionLost(self, reason):
self.setTimeout(None)
for request in self.requests:
request.connectionLost(reason)
class HTTPFactory(protocol.ServerFactory):
"""
Factory for HTTP server.
"""
protocol = HTTPChannel
logPath = None
timeOut = 60 * 60 * 12
def __init__(self, logPath=None, timeout=60*60*12):
if logPath is not None:
logPath = os.path.abspath(logPath)
self.logPath = logPath
self.timeOut = timeout
def buildProtocol(self, addr):
p = protocol.ServerFactory.buildProtocol(self, addr)
# timeOut needs to be on the Protocol instance cause
# TimeoutMixin expects it there
p.timeOut = self.timeOut
return p
def startFactory(self):
_logDateTimeStart()
if self.logPath:
self.logFile = self._openLogFile(self.logPath)
else:
self.logFile = log.logfile
def stopFactory(self):
if hasattr(self, "logFile"):
if self.logFile != log.logfile:
self.logFile.close()
del self.logFile
_logDateTimeStop()
def _openLogFile(self, path):
"""
Override in subclasses, e.g. to use twisted.python.logfile.
"""
f = open(path, "a", 1)
return f
def _escape(self, s):
# pain in the ass. Return a string like python repr, but always
# escaped as if surrounding quotes were "".
r = repr(s)
if r[0] == "'":
return r[1:-1].replace('"', '\\"').replace("\\'", "'")
return r[1:-1]
def log(self, request):
"""
Log a request's result to the logfile, by default in combined log format.
"""
if hasattr(self, "logFile"):
line = '%s - - %s "%s" %d %s "%s" "%s"\n' % (
request.getClientIP(),
# request.getUser() or "-", # the remote user is almost never important
_logDateTime,
'%s %s %s' % (self._escape(request.method),
self._escape(request.uri),
self._escape(request.clientproto)),
request.code,
request.sentLength or "-",
self._escape(request.getHeader("referer") or "-"),
self._escape(request.getHeader("user-agent") or "-"))
self.logFile.write(line)
|