From b89cda0a771e391ea92cc335553efae26202f810 Mon Sep 17 00:00:00 2001 From: Darshit Shah Date: Thu, 24 Jul 2014 16:41:25 +0530 Subject: [PATCH] More features to Python based test suite Squashed Commit from parallel-wget of: b31e6e4 Add support for HTTPS Servers b828a6e Sleep for n seconds before calling Wget Executable 7effa90 Support programatically setting Handler class variables 7e1f4c1 Correct the call to stop_HTTP_Server f616192 Improve error handling when wget executable isn't available 31868fe Split large function to improve readability and extensibility --- testenv/ChangeLog | 61 ++++++++++++++++++++++ testenv/HTTPServer.py | 43 ++++++++++++++-- testenv/Makefile.am | 94 +++++++++++++++++----------------- testenv/Test--https.py | 51 ++++++++++++++++++ testenv/Test-Parallel-Proto.py | 6 +-- testenv/Test-Proto.py | 7 ++- testenv/WgetTest.py | 57 +++++++++++++++++---- testenv/certs/wget-cert.pem | 30 +++++++++++ 8 files changed, 284 insertions(+), 65 deletions(-) create mode 100755 testenv/Test--https.py create mode 100644 testenv/certs/wget-cert.pem diff --git a/testenv/ChangeLog b/testenv/ChangeLog index 37be49e0..a3f71c0d 100644 --- a/testenv/ChangeLog +++ b/testenv/ChangeLog @@ -1,3 +1,64 @@ +2014-01-02 Darshit Shah + * Makefile.am: Add new Test--https.py to list of tests and EXTRA_DIST. + Also replace all tabs with spaces in file for conformity. + * Test--https.py: New test to check if Wget works correctly with HTTPS + servers + * HTTPServer.py: Import new modules for use in HTTPS Servers + (HTTPSServer): New class that generates a SSL-wrapped socket for use in a + HTTPS Server. + (HTTPSd): HTTPS daemon class. Analogous to the HTTPd class + * WgetTest.py: Define global variables HTTP and HTTPS to reflect Server + types + (CommonMethods.exec_wget): Add the protocol information to the URL before + passing it to wget + (HTTPTest.__init__): Edit syntax. The servers variable now accepts a list of + servers defined by their type. E.g. HTTP, HTTPS. + (HTTPTest.Server_setup): Reflect change in type of variable servers. + However, we maintin the value of self.servers to allow most of the code to + remain unchanged. + (HTTPTest.init_HTTPS_Server): Initialize a HTTPS Server + * Test-Parallel-Proto.py: Edit to reflect slight change in Test Fiel Syntax. + * Test-Proto.py: Same + +2014-01-02 Darshit Shah + + * WgetTest.py (CommonMentods.exec_wget): Wait for n seconds before calling + the Wget executable. + +2013-12-27 Darshit Shah + + * WgetTest.py: Add modeline + (CommonMethods.ServerConf): New pre-test hook that sets + BaseHTTPRequestHandler class variables in all available servers + * HTTPServer.py (HTTPd.ServerConf): Call the respective method in the Server + to set the class variables + (StoppableHTTPServer.server_sett): Set the handler class variables + +2013-12-26 Darshit Shah + + * WgetTest.py (HTTPTest.call_test): Correct the call to stop_HTTP_Server. + +2013-12-25 Darshit Shah + + * WgetTest.py (CommonMehtods.exec_wget): Catch and handle exception if the + Wget executable is not found at src/wget + (HTTPTest.call_test): In case of error during execution, remove all existing + servers before quitting + +2013-12-15 Darshit Shah + + * WgetTest.py (HTTPTest.HTTP_setup): Rename to Server_setup so it can be + easily reused for other non-HTTP servers. + (HTTPTest.__init__): Call Server_setup instead of HTTP_setup + (HTTPTest.Server_setup): Split into three more functions, that handle + pre-hooks, test execution and post-hooks respectively. + (HTTPTest.pre_hook_call): Set up and execute the pre-test hooks. Code split + from HTTPTest.Server_setup + (HTTPTest.call_test): Execute wget and log exit code. Code split from + HTTPTest.Server_setup + (HTTPTest.post_hook_call): Set up and execute post-test hooks. Code split + from HTTPTest.Server_setup + 2013-10-14 Giuseppe Scrivano * Makefile.am (XFAIL_TESTS): Remove Test--spider-r.py. diff --git a/testenv/HTTPServer.py b/testenv/HTTPServer.py index aab8d4f5..e554a105 100644 --- a/testenv/HTTPServer.py +++ b/testenv/HTTPServer.py @@ -1,10 +1,14 @@ from http.server import HTTPServer, BaseHTTPRequestHandler +from socketserver import BaseServer from posixpath import basename, splitext from base64 import b64encode from random import random from hashlib import md5 import threading +import socket import re +import ssl +import os class InvalidRangeHeader (Exception): @@ -31,9 +35,30 @@ class StoppableHTTPServer (HTTPServer): self.server_configs = conf_dict self.fileSys = filelist + def server_sett (self, settings): + for settings_key in settings: + setattr (self.RequestHandlerClass, settings_key, settings[settings_key]) + def get_req_headers (self): return self.request_headers +class HTTPSServer (StoppableHTTPServer): + + def __init__ (self, address, handler): + BaseServer.__init__ (self, address, handler) + print (os.getcwd()) + CERTFILE = os.path.abspath (os.path.join ('..', 'certs', 'wget-cert.pem')) + print (CERTFILE) + fop = open (CERTFILE) + print (fop.readline()) + self.socket = ssl.wrap_socket ( + sock = socket.socket (self.address_family, self.socket_type), + ssl_version = ssl.PROTOCOL_TLSv1, + certfile = CERTFILE, + server_side = True + ) + self.server_bind () + self.server_activate () class WgetHTTPRequestHandler (BaseHTTPRequestHandler): @@ -264,8 +289,13 @@ class _Handler (WgetHTTPRequestHandler): auth_type = auth_header.split(' ')[0] if auth_header else required_auth else: auth_type = required_auth - assert hasattr (self, "authorize_" + auth_type) - is_auth = getattr (self, "authorize_" + auth_type) (auth_header, auth_rule) + try: + assert hasattr (self, "authorize_" + auth_type) + is_auth = getattr (self, "authorize_" + auth_type) (auth_header, auth_rule) + except AssertionError: + raise ServerError ("Authentication Mechanism " + auth_rule + " not supported") + except AttributeError as ae: + raise ServerError (ae.__str__()) if is_auth is False: raise ServerError ("Unable to Authenticate") @@ -427,4 +457,11 @@ class HTTPd (threading.Thread): def server_conf (self, file_list, server_rules): self.server_inst.server_conf (file_list, server_rules) -# vim: set ts=8 sts=4 sw=3 tw=0 et : + def server_sett (self, settings): + self.server_inst.server_sett (settings) + +class HTTPSd (HTTPd): + + server_class = HTTPSServer + +# vim: set ts=4 sts=4 sw=4 tw=80 et : diff --git a/testenv/Makefile.am b/testenv/Makefile.am index 15eed935..ced19a77 100644 --- a/testenv/Makefile.am +++ b/testenv/Makefile.am @@ -28,53 +28,55 @@ AUTOMAKE_OPTIONS = parallel-tests AM_TESTS_ENVIRONMENT = MAKE_CHECK=True; export MAKE_CHECK; -TESTS = Test-auth-basic-fail.py \ - Test-auth-basic.py \ - Test-auth-both.py \ - Test-auth-digest.py \ - Test-auth-no-challenge.py \ - Test-auth-no-challenge-url.py \ - Test-auth-retcode.py \ - Test-auth-with-content-disposition.py \ - Test-c-full.py \ - Test-Content-disposition-2.py \ - Test-Content-disposition.py \ - Test-cookie-401.py \ - Test-cookie-domain-mismatch.py \ - Test-cookie-expires.py \ - Test-cookie.py \ - Test-Head.py \ - Test-O.py \ - Test-Post.py \ - Test--spider-r.py +TESTS = Test-auth-basic-fail.py \ + Test-auth-basic.py \ + Test-auth-both.py \ + Test-auth-digest.py \ + Test-auth-no-challenge.py \ + Test-auth-no-challenge-url.py \ + Test-auth-retcode.py \ + Test-auth-with-content-disposition.py \ + Test-c-full.py \ + Test-Content-disposition-2.py \ + Test-Content-disposition.py \ + Test-cookie-401.py \ + Test-cookie-domain-mismatch.py \ + Test-cookie-expires.py \ + Test-cookie.py \ + Test-Head.py \ + Test--https.py \ + Test-O.py \ + Test-Post.py \ + Test--spider-r.py -XFAIL_TESTS = Test-auth-both.py +XFAIL_TESTS = Test-auth-both.py LOG_COMPILER = python3 -EXTRA_DIST = ColourTerm.py \ - FTPServer.py \ - HTTPServer.py \ - README \ - Test--spider-r.py \ - Test-Content-disposition-2.py \ - Test-Content-disposition.py \ - Test-Head.py \ - Test-O.py \ - Test-Parallel-Proto.py \ - Test-Post.py \ - Test-Proto.py \ - Test-auth-basic-fail.py \ - Test-auth-basic.py \ - Test-auth-both.py \ - Test-auth-digest.py \ - Test-auth-no-challenge-url.py \ - Test-auth-no-challenge.py \ - Test-auth-retcode.py \ - Test-auth-with-content-disposition.py \ - Test-c-full.py \ - Test-cookie-401.py \ - Test-cookie-domain-mismatch.py \ - Test-cookie-expires.py \ - Test-cookie.py \ - WgetTest.py +EXTRA_DIST = ColourTerm.py \ + FTPServer.py \ + HTTPServer.py \ + README \ + Test--spider-r.py \ + Test--https.py \ + Test-Content-disposition-2.py \ + Test-Content-disposition.py \ + Test-Head.py \ + Test-O.py \ + Test-Parallel-Proto.py \ + Test-Post.py \ + Test-Proto.py \ + Test-auth-basic-fail.py \ + Test-auth-basic.py \ + Test-auth-both.py \ + Test-auth-digest.py \ + Test-auth-no-challenge-url.py \ + Test-auth-no-challenge.py \ + Test-auth-retcode.py \ + Test-auth-with-content-disposition.py \ + Test-c-full.py \ + Test-cookie-401.py \ + Test-cookie-domain-mismatch.py \ + Test-cookie-expires.py \ + Test-cookie.py \ + WgetTest.py diff --git a/testenv/Test--https.py b/testenv/Test--https.py new file mode 100755 index 00000000..17252b6b --- /dev/null +++ b/testenv/Test--https.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +from sys import exit +from WgetTest import HTTPTest, WgetFile, HTTPS, HTTP + +""" + This test ensures that Wget can download files from HTTPS Servers +""" +TEST_NAME = "HTTPS Downloads" +############# File Definitions ############################################### +File1 = "Would you like some Tea?" +File2 = "With lemon or cream?" +File3 = "Sure you're joking Mr. Feynman" + +A_File = WgetFile ("File1", File1) +B_File = WgetFile ("File2", File2) +C_File = WgetFile ("File3", File3) + +WGET_OPTIONS = "-d --no-check-certificate" +WGET_URLS = [["File1", "File2"]] + +Files = [[A_File, B_File]] +Existing_Files = [C_File] + +Servers = [HTTPS] + +ExpectedReturnCode = 0 +ExpectedDownloadedFiles = [A_File, B_File, C_File] + +################ Pre and Post Test Hooks ##################################### +pre_test = { + "ServerFiles" : Files, + "LocalFiles" : Existing_Files +} +test_options = { + "WgetCommands" : WGET_OPTIONS, + "Urls" : WGET_URLS +} +post_test = { + "ExpectedFiles" : ExpectedDownloadedFiles, + "ExpectedRetcode" : ExpectedReturnCode +} + +err = HTTPTest ( + name=TEST_NAME, + pre_hook=pre_test, + test_params=test_options, + post_hook=post_test, + servers=Servers +).begin () + +exit (err) diff --git a/testenv/Test-Parallel-Proto.py b/testenv/Test-Parallel-Proto.py index 56efd930..e7aae2ef 100755 --- a/testenv/Test-Parallel-Proto.py +++ b/testenv/Test-Parallel-Proto.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 from sys import exit -from WgetTest import HTTPTest, WgetFile +from WgetTest import HTTPTest, WgetFile, HTTP, HTTPS """ This is a Prototype Test File for multiple servers. @@ -22,7 +22,7 @@ WGET_URLS = [["File1"], ["File2"]] Files = [[A_File], [B_File]] Existing_Files = [C_File] -no_of_servers = 2 +Servers = [HTTP, HTTP] ExpectedReturnCode = 0 ExpectedDownloadedFiles = [A_File, B_File, C_File] @@ -46,7 +46,7 @@ err = HTTPTest ( pre_hook=pre_test, test_params=test_options, post_hook=post_test, - servers=no_of_servers + servers=Servers ).begin () exit (err) diff --git a/testenv/Test-Proto.py b/testenv/Test-Proto.py index 03523e5a..eaafdc13 100755 --- a/testenv/Test-Proto.py +++ b/testenv/Test-Proto.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 from sys import exit -from WgetTest import HTTPTest, WgetFile +from WgetTest import HTTPTest, WgetFile, HTTP, HTTPS """ This is a Prototype Test File. @@ -40,6 +40,8 @@ C_File = WgetFile ("File3", File3) WGET_OPTIONS = "-d --content-disposition --user=Sauron --password=TheEye" WGET_URLS = [["File1", "File2"]] +Servers = [HTTP] + Files = [[A_File, B_File]] Existing_Files = [C_File] @@ -64,7 +66,8 @@ err = HTTPTest ( name=TEST_NAME, pre_hook=pre_test, test_params=test_options, - post_hook=post_test + post_hook=post_test, + server=Servers ).begin () exit (err) diff --git a/testenv/WgetTest.py b/testenv/WgetTest.py index 9e5d2fe0..6470d936 100644 --- a/testenv/WgetTest.py +++ b/testenv/WgetTest.py @@ -5,10 +5,14 @@ import sys import traceback import HTTPServer import re +import time from subprocess import call from ColourTerm import printer from difflib import unified_diff +HTTP = "HTTP" +HTTPS = "HTTPS" + """ A Custom Exception raised by the Test Environment. """ class TestFailed (Exception): @@ -40,7 +44,13 @@ class CommonMethods: cmd_line = self.get_cmd_line (options, urls, domain_list) params = shlex.split (cmd_line) print (params) - retcode = call (params) + if os.getenv ("SERVER_WAIT"): + time.sleep (float (os.getenv ("SERVER_WAIT"))) + try: + retcode = call (params) + except FileNotFoundError as filenotfound: + raise TestFailed ( + "The Wget Executable does not exist at the expected path") return retcode def get_cmd_line (self, options, urls, domain_list): @@ -50,7 +60,8 @@ class CommonMethods: cmd_line = WGET_PATH + " " + options + " " for i in range (0, self.servers): for url in urls[i]: - cmd_line += domain_list[i] + url + " " + protocol = "http://" if self.server_types[i] is "HTTP" else "https://" + cmd_line += protocol + domain_list[i] + url + " " # for url in urls: # cmd_line += domain_list[0] + url + " " print (cmd_line) @@ -174,6 +185,10 @@ class CommonMethods: file_handler.write (file_obj.content) file_handler.close () + def ServerConf (self, server_settings): + for i in range (0, self.servers): + self.server_list[i].server_sett (server_settings) + """ Test Option Function Calls """ def WgetCommands (self, command_list): @@ -216,10 +231,10 @@ class HTTPTest (CommonMethods): pre_hook=dict(), test_params=dict(), post_hook=dict(), - servers=1 + servers=[HTTP] ): try: - self.HTTP_setup (name, pre_hook, test_params, post_hook, servers) + self.Server_setup (name, pre_hook, test_params, post_hook, servers) except TestFailed as tf: printer ("RED", "Error: " + tf.error) self.tests_passed = False @@ -232,40 +247,53 @@ class HTTPTest (CommonMethods): printer ("GREEN", "Test Passed") finally: self._exit_test () - def HTTP_setup (self, name, pre_hook, test_params, post_hook, servers): + + def Server_setup (self, name, pre_hook, test_params, post_hook, servers): self.name = name - self.servers = servers + self.server_types = servers + self.servers = len (servers) printer ("BLUE", "Running Test " + self.name) self.init_test_env (name) self.server_list = list() self.domain_list = list() - for server_number in range (0, servers): - server_inst = self.init_HTTP_Server () + for server_type in servers: + server_inst = getattr (self, "init_" + server_type + "_Server") () self.server_list.append (server_inst) domain = self.get_domain_addr (server_inst.server_address) self.domain_list.append (domain) #self.server = self.init_HTTP_Server () #self.domain = self.get_domain_addr (self.server.server_address) + self.pre_hook_call (pre_hook) + self.call_test (test_params) + self.post_hook_call (post_hook) + + def pre_hook_call (self, pre_hook): for pre_hook_func in pre_hook: try: assert hasattr (self, pre_hook_func) except AssertionError as ae: - self.stop_HTTP_Server (self.server) + self.stop_HTTP_Server () raise TestFailed ("Pre Test Function " + pre_hook_func + " not defined.") getattr (self, pre_hook_func) (pre_hook[pre_hook_func]) + def call_test (self, test_params): for test_func in test_params: try: assert hasattr (self, test_func) except AssertionError as ae: - self.stop_HTTP_Server (self.server) + self.stop_HTTP_Server () raise TestFailed ("Test Option " + test_func + " unknown.") getattr (self, test_func) (test_params[test_func]) - self.act_retcode = self.exec_wget (self.options, self.urls, self.domain_list) + try: + self.act_retcode = self.exec_wget (self.options, self.urls, self.domain_list) + except TestFailed as tf: + self.stop_HTTP_Server () + raise TestFailed (tf.__str__ ()) self.stop_HTTP_Server () + def post_hook_call (self, post_hook): for post_hook_func in post_hook: try: assert hasattr (self, post_hook_func) @@ -278,6 +306,11 @@ class HTTPTest (CommonMethods): server.start () return server + def init_HTTPS_Server (self): + server = HTTPServer.HTTPSd () + server.start () + return server + def stop_HTTP_Server (self): self.Request_remaining = list () for server in self.server_list: @@ -300,3 +333,5 @@ class WgetFile: self.content = content self.timestamp = timestamp self.rules = rules + +# vim: set ts=4 sts=4 sw=4 tw=80 et : diff --git a/testenv/certs/wget-cert.pem b/testenv/certs/wget-cert.pem new file mode 100644 index 00000000..b83069e2 --- /dev/null +++ b/testenv/certs/wget-cert.pem @@ -0,0 +1,30 @@ +-----BEGIN PRIVATE KEY----- +MIICdQIBADANBgkqhkiG9w0BAQEFAASCAl8wggJbAgEAAoGBAMV8qEpuSVUdWaAY +F2N1ljGEJ/907Og5B0aZLeDskmLAOohKMWTiiSx+lseXVD/Zf/LaFfy/+q0Rk5+o +pFEPEEjadvdxogb9HPwjfj48ng74yV1c5ZGRx/aIeIJN9cacfs4J5NlT3ZPiV8/2 +mpBurBYvta5tneUl+lx4NHTEBmjTAgMBAAECgYBHlFlDMRovWYYEuvavPA2GQQpm +UzETMqhqdFbmsZiVZmtQvuOMV3e0wuVPzo/g3Kq9kUJq7AKl/DrvoaZ9IuKZgkDD +0QEBYo/lcxEA9qcfgVs5XLp9ED1mXzJSZ3bmpCDqa2NjG7yFdWzPxc1DXmT05MrF +bZbb0Wao0tvMwoeJYQJBAOql5uOyjDHvLLuS0IFKbYz4LQwAp7Gjs0ZS9qLNhQQn +m5Vr8xS9QwFID693K6aDl3tqSCIwSnyInacj8M8v18sCQQDXdReE2i4LKOVLcQsP +XabN96fFLlnoIh9MqFza4skjhXJWqjBLgJuFqyT5CTbU9TmaoIPXdo4454P1CCgR +KEIZAkAZE7nlQ8Ov4nvJYBtgde/XTP6jdb52QaR7M4qgQ46frwv1oB/Oa5upm2Xx +vq6vkQiza9xhqv+K557RqgmmWtqZAkASoXJmL4OZvXCOZHkDXCLHXqnoOAjYNNMm +Csz0tHWWF7z6V38TmExac6Ef07clFQtlHoooAH1t2D8l2g205hlJAkBfeghbZDdY +16NtVnvtzjjhKqZFqwTSANFV8NSzgb/QiNnX0hsMPt9bbc5VCo77Ly2oP5SvixfZ +kjrIQqDV8MLu +-----END PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIICODCCAaGgAwIBAgIJAOiSkPuPcAwqMA0GCSqGSIb3DQEBBQUAMDUxCzAJBgNV +BAYTAklOMRMwEQYDVQQIDApTb21lLVN0YXRlMREwDwYDVQQKDAhHTlUgV2dldDAe +Fw0xMzEyMDcwNTA3NTRaFw0xNDEyMDcwNTA3NTRaMDUxCzAJBgNVBAYTAklOMRMw +EQYDVQQIDApTb21lLVN0YXRlMREwDwYDVQQKDAhHTlUgV2dldDCBnzANBgkqhkiG +9w0BAQEFAAOBjQAwgYkCgYEAxXyoSm5JVR1ZoBgXY3WWMYQn/3Ts6DkHRpkt4OyS +YsA6iEoxZOKJLH6Wx5dUP9l/8toV/L/6rRGTn6ikUQ8QSNp293GiBv0c/CN+Pjye +DvjJXVzlkZHH9oh4gk31xpx+zgnk2VPdk+JXz/aakG6sFi+1rm2d5SX6XHg0dMQG +aNMCAwEAAaNQME4wHQYDVR0OBBYEFLhtTG9a6v3ihL5DeWKfq6doYI42MB8GA1Ud +IwQYMBaAFLhtTG9a6v3ihL5DeWKfq6doYI42MAwGA1UdEwQFMAMBAf8wDQYJKoZI +hvcNAQEFBQADgYEApTEZX3cgmgdXDJsu7wtkejtq3vuyi6NXBUlHzoYzWaS5wn8P +uDG4G9zd1cwmwrbYA8lS+ANWvkcqjM68gMs1ARMZRS0IrYMCN8bokQw+16sqImZO +THX50Sb5U+9e1IotDWyRBNO10znsoh569BxhJ5WZdIaoKHOJdXEYV+3Y/hg= +-----END CERTIFICATE-----