moparisthebest
/
wget
mirror of https://github.com/moparisthebest/wget
1
0
Fork 0
Code Issues Releases Wiki Activity

Documentation and code cleanup in test suite 

Add (lots) of documentation for various parts of the test suite in the
form of Python docstrings. Also, clean up some of the redundant code and
fix indentation issues.
This commit is contained in:
Darshit Shah 2014-08-08 11:24:08 +05:30
parent 5753ed4a72
commit f8e9a64ec7
20 changed files with 273 additions and 110 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
testenv/ChangeLog
View File

@ -1,3 +1,40 @@
2014-08-08 Darshit Shah <darnir@gmail.com>
* conf/__init__.py: Add extra newline according to PEP8
* conf/{authentication,expect_header,expected_files,expected_ret_code,
files_crawled,hook_sample,local_files,reject_header,response,send_header,
server_files,urls,wget_commands}.py: Add docstrings explaining the conf file
and how it should be used
* server/http/http_server (InvalidRangeHeader): Clear TODO and eliminate
this exception. Use ServerError for all such purposes.
(_Handler): Remove reference to InvalidRangeHeader
(_handler.parse_range_header): User ServerError instead of InvalidRangeHeader
(_Handler.do_GET): Add docstring
(_Handler.do_POST): Add docstring. Also create an empty dict for rules if
no rules are supplied. Send the Location header as suggested in RFC 7231
(_Handler.do_PUT): Don't pop the server file already. Push it to later in ..
(_Handler.send_put): .. Here. If the file exists respond with a 204 No
Content message and pop the file for replacement. Do not send the
Content-Length, Content-Type headers since PUT requests should not respond
with data.
(_Handler.parse_auth_header): Fit line within 80 chars
(_Handler.check_response): Better visual indent
(_Handler.authorize_digest): Better visual indent.
(_Handler.expect_headers): Remove unused function
(_Handler.guess_type): Fix indentation
(HTTPd): Add newline according to PEP8 guidelines
(HTTPSd): Fix indentation
(StoppableHTTPServer): Add docstring
(HTTPSServer): Fix indentation
(WgetHTTPRequestHandler): Merge class into _handler.
(_Handler): Add docstring
(_Handler.parse_range_header): Fix indentation
(ServerError): Split exception into separate file ...
* exc/server_error.py: ... Here
* misc/colour_terminal.py: Add docstring, fix indentation
* test/base_test.py: Fix visual indent
* test/http_test.py: Fit within 80 char lines
2014-08-04 Darshit Shah <darnir@gmail.com>
* conf/server_conf.py: Delete file. Server configuration is now done via the

1
testenv/conf/__init__.py
View File

@ -3,6 +3,7 @@ import os
# this file implements the mechanism of conf class auto-registration,
# don't modify this file if you have no idea what you're doing
def gen_hook():
hook_table = {}

13
testenv/conf/authentication.py
View File

@ -1,5 +1,18 @@
from conf import rule
""" Rule: Authentication
This file defines an authentication rule which when applied to any file will
cause the server to prompt the client for the required authentication details
before serving it.
auth_type must be either of: Basic, Digest, Both or Both-inline
When auth_type is Basic or Digest, the server asks for the respective
authentication in its response. When auth_type is Both, the server sends two
Authenticate headers, one requesting Basic and the other requesting Digest
authentication. If auth_type is Both-inline, the server sends only one
Authenticate header, but lists both Basic and Digest as supported mechanisms in
that.
"""
@rule()
class Authentication:

5
testenv/conf/expect_header.py
View File

@ -1,5 +1,10 @@
from conf import rule
""" Rule: ExpectHeader
This rule defines a dictionary of headers and their value which the server
should expect in each request for the file to which the rule was applied.
"""
@rule()
class ExpectHeader:

11
testenv/conf/expected_files.py
View File

@ -4,6 +4,15 @@ import sys
from conf import hook
from exc.test_failed import TestFailed
""" Post-Test Hook: ExpectedFiles
This is a Post-Test hook that checks the test directory for the files it
contains. A dictionary object is passed to it, which contains a mapping of
filenames and contents of all the files that the directory is expected to
contain.
Raises a TestFailed exception if the expected files are not found or if extra
files are found, else returns gracefully.
"""
@hook()
class ExpectedFiles:
@ -34,7 +43,7 @@ class ExpectedFiles:
fromfile='Actual',
tofile='Expected'):
print(line, file=sys.stderr)
raise TestFailed('Contents of %s do not match.' % file.name)
raise TestFailed('Contents of %s do not match' % file.name)
else:
raise TestFailed('Expected file %s not found.' % file.name)
if local_fs:

8
testenv/conf/expected_ret_code.py
View File

@ -1,6 +1,14 @@
from exc.test_failed import TestFailed
from conf import hook
""" Post-Test Hook: ExpectedRetCode
This is a post-test hook which checks if the exit code of the Wget instance
under test is the same as that expected. As a result, this is a very important
post test hook which is checked in all the tests.
Returns a TestFailed exception if the return code does not match the expected
value. Else returns gracefully.
"""
@hook(alias='ExpectedRetcode')
class ExpectedRetCode:

9
testenv/conf/files_crawled.py
View File

@ -2,6 +2,15 @@ from misc.colour_terminal import print_red
from conf import hook
from exc.test_failed import TestFailed
""" Post-Test Hook: FilesCrawled
This is a post test hook that is invoked in tests that check wget's behaviour
in recursive mode. It expects an ordered list of the request lines that Wget
must send to the server. If the requests received by the server do not match
the provided list, IN THE GIVEN ORDER, then it raises a TestFailed exception.
Such a test can be used to check the implementation of the recursion algorithm
in Wget too.
"""
@hook()
class FilesCrawled:

9
testenv/conf/hook_sample.py
View File

@ -1,7 +1,12 @@
from exc.test_failed import TestFailed
from conf import hook
# this file is a hook example
""" Hook: SampleHook
This a sample file for how a new hook should be defined.
Any errors should always be reported by raising a TestFailed exception instead
of returning a true or false value.
"""
@hook(alias='SampleHookAlias')
class SampleHook:
@ -12,4 +17,6 @@ class SampleHook:
def __call__(self, test_obj):
# implement hook here
# if you need the test case instance, refer to test_obj
if False:
raise TestFailed ("Reason")
pass

6
testenv/conf/local_files.py
View File

@ -1,5 +1,11 @@
from conf import hook
""" Pre-Test Hook: LocalFiles
This is a pre-test hook used to generate the specific environment before a test
is run. The LocalFiles hook creates the files which should exist on disk before
invoking Wget.
"""
@hook()
class LocalFiles:

6
testenv/conf/reject_header.py
View File

@ -1,5 +1,11 @@
from conf import rule
""" Rule: RejectHeader
This is a server side rule which expects a dictionary object of Headers and
their values which should be blacklisted by the server for a particular file's
requests.
"""
@rule()
class RejectHeader:

4
testenv/conf/response.py
View File

@ -1,5 +1,9 @@
from conf import rule
""" Rule: Response
When this rule is set against a certain file, the server will unconditionally
respond to any request for the said file with the provided response code. """
@rule()
class Response:

5
testenv/conf/send_header.py
View File

@ -1,5 +1,10 @@
from conf import rule
""" Rule: SendHeader
Have the server send custom headers when responding to a request for the file
this rule is applied to. The header_obj object is expected to be dictionary
mapping headers to their contents. """
@rule()
class SendHeader:

11
testenv/conf/server_files.py
View File

@ -1,5 +1,16 @@
from conf import hook
""" Pre-Test Hook: ServerFiles
This hook is used to define a set of files on the server's virtual filesystem.
server_files is expected to be dictionary that maps filenames to their
contents. In the future, this can be used to add additional metadat to the
files using the WgetFile class too.
This hook also does some additional processing on the contents of the file. Any
text between {{and}} is replaced by the contents of a class variable of the
same name. This is useful in creating files that contain an absolute link to
another file on the same server. """
@hook()
class ServerFiles:

4
testenv/conf/urls.py
View File

@ -1,5 +1,9 @@
from conf import hook
""" Pre-Test Hook: URLS
This hook is used to define the paths of the files on the test server that wget
will send a request for. """
@hook(alias='Urls')
class URLs:

5
testenv/conf/wget_commands.py
View File

@ -1,5 +1,10 @@
from conf import hook
""" Pre-Test Hook: WgetCommands
This hook is used to specify the test specific switches that must be passed to
wget on invokation. Default switches are hard coded in the test suite itself.
"""
@hook()
class WgetCommands:

7
testenv/exc/server_error.py Normal file
View File

@ -0,0 +1,7 @@
class ServerError (Exception):
""" A custom exception which is raised by the test servers. Often used to
handle control flow. """
def __init__ (self, err_message):
self.err_message = err_message

45
testenv/misc/colour_terminal.py
View File

@ -2,24 +2,39 @@ from functools import partial
import platform
from os import getenv
""" This module allows printing coloured output to the terminal when running a
Wget Test under certain conditions.
The output is coloured only on Linux systems. This is because coloured output
in the terminal on Windows requires too much effort for what is simply a
convenience. This might work on OSX terminals, but without a confirmation, it
remains unsupported.
Another important aspect is that the coloured output is printed only if the
environment variable MAKE_CHECK is not set. This variable is set when running
the test suite through, `make check`. In that case, the output is not only
printed to the terminal but also copied to a log file where the ANSI escape
codes on;y add clutter. """
T_COLORS = {
'PURPLE' : '\033[95m',
'BLUE' : '\033[94m',
'GREEN' : '\033[92m',
'YELLOW' : '\033[93m',
'RED' : '\033[91m',
'ENDC' : '\033[0m'
'PURPLE' : '\033[95m',
'BLUE' : '\033[94m',
'GREEN' : '\033[92m',
'YELLOW' : '\033[93m',
'RED' : '\033[91m',
'ENDC' : '\033[0m'
}
def printer (color, string):
if platform.system () == 'Linux':
if getenv ("MAKE_CHECK", "False") == "True":
print (string)
else:
print (T_COLORS.get (color) + string + T_COLORS.get ('ENDC'))
else:
print (string)
def printer (color, string):
if platform.system () == 'Linux':
if getenv ("MAKE_CHECK", "False") == "True":
print (string)
else:
print (T_COLORS.get (color) + string + T_COLORS.get ('ENDC'))
else:
print (string)
print_blue = partial(printer, 'BLUE')
@ -28,4 +43,4 @@ print_green = partial(printer, 'GREEN')
print_purple = partial(printer, 'PURPLE')
print_yellow = partial(printer, 'YELLOW')
# vim: set ts=8 sw=3 tw=0 et :
# vim: set ts=8 sw=3 tw=80 et :

184
testenv/server/http/http_server.py
View File

@ -1,4 +1,5 @@
from http.server import HTTPServer, BaseHTTPRequestHandler
from exc.server_error import ServerError
from socketserver import BaseServer
from posixpath import basename, splitext
from base64 import b64encode
@ -11,20 +12,12 @@ import ssl
import os
class InvalidRangeHeader (Exception):
""" Create an Exception for handling of invalid Range Headers. """
# TODO: Eliminate this exception and use only ServerError
def __init__ (self, err_message):
self.err_message = err_message
class ServerError (Exception):
def __init__ (self, err_message):
self.err_message = err_message
class StoppableHTTPServer (HTTPServer):
""" This class extends the HTTPServer class from default http.server library
in Python 3. The StoppableHTTPServer class is capable of starting an HTTP
server that serves a virtual set of files made by the WgetFile class and
has most of its properties configurable through the server_conf()
method. """
request_headers = list ()
@ -38,38 +31,42 @@ class StoppableHTTPServer (HTTPServer):
def get_req_headers (self):
return self.request_headers
class HTTPSServer (StoppableHTTPServer):
""" The HTTPSServer class extends the StoppableHTTPServer class with
additional support for secure connections through SSL. """
def __init__ (self, address, handler):
BaseServer.__init__ (self, address, handler)
print (os.getcwd())
CERTFILE = os.path.abspath (os.path.join ('..', 'certs', 'wget-cert.pem'))
print (CERTFILE)
fop = open (CERTFILE)
print (fop.readline())
self.socket = ssl.wrap_socket (
sock = socket.socket (self.address_family, self.socket_type),
ssl_version = ssl.PROTOCOL_TLSv1,
certfile = CERTFILE,
server_side = True
)
self.server_bind ()
self.server_activate ()
def __init__ (self, address, handler):
BaseServer.__init__ (self, address, handler)
print (os.getcwd())
CERTFILE = os.path.abspath(os.path.join('..', 'certs', 'wget-cert.pem'))
print (CERTFILE)
fop = open (CERTFILE)
print (fop.readline())
self.socket = ssl.wrap_socket (
sock = socket.socket (self.address_family, self.socket_type),
ssl_version = ssl.PROTOCOL_TLSv1,
certfile = CERTFILE,
server_side = True
)
self.server_bind()
self.server_activate()
class WgetHTTPRequestHandler (BaseHTTPRequestHandler):
""" Define methods for handling Test Checks. """
class _Handler (BaseHTTPRequestHandler):
""" This is a private class which tells the server *HOW* to handle each
request. For each HTTP Request Command that the server should be capable of
responding to, there must exist a do_REQUESTNAME() method which details the
steps in which such requests should be processed. The rest of the methods
in this class are auxilliary methods created to help in processing certain
requests. """
def get_rule_list (self, name):
r_list = self.rules.get (name) if name in self.rules else None
return r_list
class _Handler (WgetHTTPRequestHandler):
""" Define Handler Methods for different Requests. """
InvalidRangeHeader = InvalidRangeHeader
# The defailt protocol version of the server we run is HTTP/1.1 not
# HTTP/1.0 which is the default with the http.server module.
protocol_version = 'HTTP/1.1'
""" Define functions for various HTTP Requests. """
@ -78,6 +75,11 @@ class _Handler (WgetHTTPRequestHandler):
self.send_head ("HEAD")
def do_GET (self):
""" Process HTTP GET requests. This is the same as processing HEAD
requests and then actually transmitting the data to the client. If
send_head() does not specify any "start" offset, we send the complete
data, else transmit only partial data. """
content, start = self.send_head ("GET")
if content:
if start is None:
@ -86,11 +88,26 @@ class _Handler (WgetHTTPRequestHandler):
self.wfile.write (content.encode ('utf-8')[start:])
def do_POST (self):
""" According to RFC 7231 sec 4.3.3, if the resource requested in a POST
request does not exist on the server, the first POST request should
create that resource. PUT requests are otherwise used to create a
resource. Hence, we call the handle for processing PUT requests if the
resource requested does not already exist.
Currently, when the server recieves a POST request for a resource, we
simply append the body data to the existing file and return the new
file to the client. If the file does not exist, a new file is created
using the contents of the request body. """
path = self.path[1:]
self.rules = self.server.server_configs.get (path)
if not self.custom_response ():
return (None, None)
if path in self.server.fileSys:
self.rules = self.server.server_configs.get (path)
if not self.rules:
self.rules = dict ()
if not self.custom_response ():
return (None, None)
body_data = self.get_body_data ()
self.send_response (200)
self.send_header ("Content-type", "text/plain")
@ -98,6 +115,7 @@ class _Handler (WgetHTTPRequestHandler):
total_length = len (content)
self.server.fileSys[path] = content
self.send_header ("Content-Length", total_length)
self.send_header ("Location", self.path)
self.finish_headers ()
try:
self.wfile.write (content.encode ('utf-8'))
@ -111,7 +129,6 @@ class _Handler (WgetHTTPRequestHandler):
self.rules = self.server.server_configs.get (path)
if not self.custom_response ():
return (None, None)
self.server.fileSys.pop (path, None)
self.send_put (path)
""" End of HTTP Request Method Handlers. """
@ -122,12 +139,12 @@ class _Handler (WgetHTTPRequestHandler):
if header_line is None:
return None
if not header_line.startswith ("bytes="):
raise InvalidRangeHeader ("Cannot parse header Range: %s" %
(header_line))
raise ServerError ("Cannot parse header Range: %s" %
(header_line))
regex = re.match (r"^bytes=(\d*)\-$", header_line)
range_start = int (regex.group (1))
if range_start >= length:
raise InvalidRangeHeader ("Range Overflow")
raise ServerError ("Range Overflow")
return range_start
def get_body_data (self):
@ -137,23 +154,27 @@ class _Handler (WgetHTTPRequestHandler):
return body_data
def send_put (self, path):
if path in self.server.fileSys:
self.server.fileSys.pop (path, None)
self.send_response (204)
else:
self.rules = dict ()
self.send_response (201)
body_data = self.get_body_data ()
self.send_response (201)
self.server.fileSys[path] = body_data
self.send_header ("Content-type", "text/plain")
self.send_header ("Content-Length", len (body_data))
self.send_header ("Location", self.path)
self.finish_headers ()
try:
self.wfile.write (body_data.encode ('utf-8'))
except Exception:
pass
""" This empty method is called automatically when all the rules are
processed for a given request. However, send_header() should only be called
AFTER a response has been sent. But, at the moment of processing the rules,
the appropriate response has not yet been identified. As a result, we defer
the processing of this rule till later. Each do_* request handler MUST call
finish_headers() instead of end_headers(). The finish_headers() method
takes care of sending the appropriate headers before completing the
response. """
def SendHeader (self, header_obj):
pass
# headers_list = header_obj.headers
# for header_line in headers_list:
# print (header_line + " : " + headers_list[header_line])
# self.send_header (header_line, headers_list[header_line])
def send_cust_headers (self):
header_obj = self.get_rule_list ('SendHeader')
@ -191,11 +212,11 @@ class _Handler (WgetHTTPRequestHandler):
if auth_type == "Basic":
challenge_str = 'Basic realm="Wget-Test"'
elif auth_type == "Digest" or auth_type == "Both_inline":
self.nonce = md5 (str (random ()).encode ('utf-8')).hexdigest ()
self.opaque = md5 (str (random ()).encode ('utf-8')).hexdigest ()
challenge_str = 'Digest realm="Test", nonce="%s", opaque="%s"' %(
self.nonce,
self.opaque)
self.nonce = md5 (str (random ()).encode ('utf-8')).hexdigest()
self.opaque = md5 (str (random ()).encode ('utf-8')).hexdigest()
challenge_str = 'Digest realm="Test", nonce="%s", opaque="%s"' % (
self.nonce,
self.opaque)
challenge_str += ', qop="auth"'
if auth_type == "Both_inline":
challenge_str = 'Basic realm="Wget-Test", ' + challenge_str
@ -214,9 +235,9 @@ class _Handler (WgetHTTPRequestHandler):
n = len("Digest ")
auth_header = auth_header[n:].strip()
items = auth_header.split(", ")
key_values = [i.split("=", 1) for i in items]
key_values = [(k.strip(), v.strip().replace('"', '')) for k, v in key_values]
return dict(key_values)
keyvals = [i.split("=", 1) for i in items]
keyvals = [(k.strip(), v.strip().replace('"', '')) for k, v in keyvals]
return dict(keyvals)
def KD (self, secret, data):
return self.H (secret + ":" + data)
@ -233,10 +254,10 @@ class _Handler (WgetHTTPRequestHandler):
def check_response (self, params):
if "qop" in params:
data_str = params['nonce'] \
+ ":" + params['nc'] \
+ ":" + params['cnonce'] \
+ ":" + params['qop'] \
+ ":" + self.H (self.A2 (params))
+ ":" + params['nc'] \
+ ":" + params['cnonce'] \
+ ":" + params['qop'] \
+ ":" + self.H (self.A2 (params))
else:
data_str = params['nonce'] + ":" + self.H (self.A2 (params))
resp = self.KD (self.H (self.A1 ()), data_str)
@ -252,11 +273,12 @@ class _Handler (WgetHTTPRequestHandler):
params = self.parse_auth_header (auth_header)
pass_auth = True
if self.user != params['username'] or \
self.nonce != params['nonce'] or self.opaque != params['opaque']:
self.nonce != params['nonce'] or \
self.opaque != params['opaque']:
pass_auth = False
req_attribs = ['username', 'realm', 'nonce', 'uri', 'response']
for attrib in req_attribs:
if not attrib in params:
if attrib not in params:
pass_auth = False
if not self.check_response (params):
pass_auth = False
@ -322,19 +344,6 @@ class _Handler (WgetHTTPRequestHandler):
self.finish_headers ()
raise ServerError ("Header " + header_line + " not found")
def expect_headers (self):
""" This is modified code to handle a few changes. Should be removed ASAP """
exp_headers_obj = self.get_rule_list ('ExpectHeader')
if exp_headers_obj:
exp_headers = exp_headers_obj.headers
for header_line in exp_headers:
header_re = self.headers.get (header_line)
if header_re is None or header_re != exp_headers[header_line]:
self.send_error (400, 'Expected Header not Found')
self.end_headers ()
return False
return True
def RejectHeader (self, header_obj):
rej_headers = header_obj.headers
for header_line in rej_headers:
@ -396,7 +405,7 @@ class _Handler (WgetHTTPRequestHandler):
try:
self.range_begin = self.parse_range_header (
self.headers.get ("Range"), content_length)
except InvalidRangeHeader as ae:
except ServerError as ae:
# self.log_error("%s", ae.err_message)
if ae.err_message == "Range Overflow":
self.send_response (416)
@ -427,9 +436,9 @@ class _Handler (WgetHTTPRequestHandler):
base_name = basename ("/" + path)
name, ext = splitext (base_name)
extension_map = {
".txt" : "text/plain",
".css" : "text/css",
".html" : "text/html"
".txt" : "text/plain",
".css" : "text/css",
".html" : "text/html"
}
if ext in extension_map:
return extension_map[ext]
@ -440,6 +449,7 @@ class _Handler (WgetHTTPRequestHandler):
class HTTPd (threading.Thread):
server_class = StoppableHTTPServer
handler = _Handler
def __init__ (self, addr=None):
threading.Thread.__init__ (self)
if addr is None:
@ -448,7 +458,7 @@ class HTTPd (threading.Thread):
self.server_address = self.server_inst.socket.getsockname()[:2]
def run (self):
self.server_inst.serve_forever ()
self.server_inst.serve_forever ()
def server_conf (self, file_list, server_rules):
self.server_inst.server_conf (file_list, server_rules)
@ -456,6 +466,6 @@ class HTTPd (threading.Thread):
class HTTPSd (HTTPd):
server_class = HTTPSServer
server_class = HTTPSServer
# vim: set ts=4 sts=4 sw=4 tw=80 et :

6
testenv/test/base_test.py
View File

@ -28,9 +28,9 @@ class BaseTest:
Attributes should not be defined outside __init__.
"""
self.name = name
self.pre_configs = pre_hook or {} # if pre_hook == None, then
# {} (an empty dict object) is
# passed to self.pre_configs
self.pre_configs = pre_hook or {} # if pre_hook == None, then
# {} (an empty dict object) is
# passed to self.pre_configs
self.test_params = test_params or {}
self.post_configs = post_hook or {}
self.protocols = protocols

7
testenv/test/http_test.py
View File

@ -7,9 +7,10 @@ class HTTPTest(BaseTest):
""" Class for HTTP Tests. """
# Temp Notes: It is expected that when pre-hook functions are executed, only an empty test-dir exists.
# pre-hook functions are executed just prior to the call to Wget is made.
# post-hook functions will be executed immediately after the call to Wget returns.
# Temp Notes: It is expected that when pre-hook functions are executed,
# only an empty test-dir exists. pre-hook functions are executed just prior
# to the call to Wget is made. post-hook functions will be executed
# immediately after the call to Wget returns.
def __init__(self,
name="Unnamed Test",