More features to Python based test suite

Squashed Commit from parallel-wget of:
b31e6e4 Add support for HTTPS Servers
b828a6e Sleep for n seconds before calling Wget Executable
7effa90 Support programatically setting Handler class variables
7e1f4c1 Correct the call to stop_HTTP_Server
f616192 Improve error handling when wget executable isn't available
31868fe Split large function to improve readability and extensibility

testenv/ChangeLog

@@ -1,3 +1,64 @@
+2014-01-02  Darshit Shah  <darnir@gmail.com>
+	* Makefile.am: Add new Test--https.py to list of tests and EXTRA_DIST.
+	Also replace all tabs with spaces in file for conformity.
+	* Test--https.py: New test to check if Wget works correctly with HTTPS
+	servers
+	* HTTPServer.py: Import new modules for use in HTTPS Servers
+	(HTTPSServer): New class that generates a SSL-wrapped socket for use in a
+	HTTPS Server.
+	(HTTPSd): HTTPS daemon class. Analogous to the HTTPd class
+	* WgetTest.py: Define global variables HTTP and HTTPS to reflect Server
+	types
+	(CommonMethods.exec_wget): Add the protocol information to the URL before
+	passing it to wget
+	(HTTPTest.__init__): Edit syntax. The servers variable now accepts a list of
+	servers defined by their type. E.g. HTTP, HTTPS.
+	(HTTPTest.Server_setup): Reflect change in type of variable servers.
+	However, we maintin the value of self.servers to allow most of the code to
+	remain unchanged.
+	(HTTPTest.init_HTTPS_Server): Initialize a HTTPS Server
+	* Test-Parallel-Proto.py: Edit to reflect slight change in Test Fiel Syntax.
+	* Test-Proto.py: Same
+
+2014-01-02  Darshit Shah  <darnir@gmail.com>
+
+	* WgetTest.py (CommonMentods.exec_wget): Wait for n seconds before calling
+	the Wget executable.
+
+2013-12-27  Darshit Shah  <darnir@gmail.com>
+
+	* WgetTest.py: Add modeline
+	(CommonMethods.ServerConf): New pre-test hook that sets
+	BaseHTTPRequestHandler class variables in all available servers
+	* HTTPServer.py (HTTPd.ServerConf): Call the respective method in the Server
+	to set the class variables
+	(StoppableHTTPServer.server_sett): Set the handler class variables
+
+2013-12-26  Darshit Shah  <darnir@gmail.com>
+
+	* WgetTest.py (HTTPTest.call_test): Correct the call to stop_HTTP_Server.
+
+2013-12-25  Darshit Shah  <darnir@gmail.com>
+
+	* WgetTest.py (CommonMehtods.exec_wget): Catch and handle exception if the
+	Wget executable is not found at src/wget
+	(HTTPTest.call_test): In case of error during execution, remove all existing
+	servers before quitting
+
+2013-12-15  Darshit Shah  <darnir@gmail.com>
+
+	* WgetTest.py (HTTPTest.HTTP_setup): Rename to Server_setup so it can be
+	easily reused for other non-HTTP servers.
+	(HTTPTest.__init__): Call Server_setup instead of HTTP_setup
+	(HTTPTest.Server_setup): Split into three more functions, that handle
+	pre-hooks, test execution and post-hooks respectively.
+	(HTTPTest.pre_hook_call): Set up and execute the pre-test hooks. Code split
+	from HTTPTest.Server_setup
+	(HTTPTest.call_test): Execute wget and log exit code. Code split from
+	HTTPTest.Server_setup
+	(HTTPTest.post_hook_call): Set up and execute post-test hooks. Code split
+	from HTTPTest.Server_setup
+
 2013-10-14  Giuseppe Scrivano  <gscrivan@redhat.com>
 
 	* Makefile.am (XFAIL_TESTS): Remove Test--spider-r.py.

testenv/HTTPServer.py

@@ -1,10 +1,14 @@
 from http.server import HTTPServer, BaseHTTPRequestHandler
+from socketserver import BaseServer
 from posixpath import basename, splitext
 from base64 import b64encode
 from random import random
 from hashlib import md5
 import threading
+import socket
 import re
+import ssl
+import os
 
 
 class InvalidRangeHeader (Exception):
@@ -31,9 +35,30 @@ class StoppableHTTPServer (HTTPServer):
         self.server_configs = conf_dict
         self.fileSys = filelist
 
+    def server_sett (self, settings):
+        for settings_key in settings:
+            setattr (self.RequestHandlerClass, settings_key, settings[settings_key])
+
     def get_req_headers (self):
         return self.request_headers
 
+class HTTPSServer (StoppableHTTPServer):
+
+   def __init__ (self, address, handler):
+         BaseServer.__init__ (self, address, handler)
+         print (os.getcwd())
+         CERTFILE = os.path.abspath (os.path.join ('..', 'certs', 'wget-cert.pem'))
+         print (CERTFILE)
+         fop = open (CERTFILE)
+         print (fop.readline())
+         self.socket = ssl.wrap_socket (
+               sock = socket.socket (self.address_family, self.socket_type),
+               ssl_version = ssl.PROTOCOL_TLSv1,
+               certfile = CERTFILE,
+               server_side = True
+               )
+         self.server_bind ()
+         self.server_activate ()
 
 class WgetHTTPRequestHandler (BaseHTTPRequestHandler):
 
@@ -264,8 +289,13 @@ class _Handler (WgetHTTPRequestHandler):
             auth_type = auth_header.split(' ')[0] if auth_header else required_auth
         else:
             auth_type = required_auth
+        try:
             assert hasattr (self, "authorize_" + auth_type)
             is_auth = getattr (self, "authorize_" + auth_type) (auth_header, auth_rule)
+        except AssertionError:
+            raise ServerError ("Authentication Mechanism " + auth_rule + " not supported")
+        except AttributeError as ae:
+            raise ServerError (ae.__str__())
         if is_auth is False:
             raise ServerError ("Unable to Authenticate")
 
@@ -427,4 +457,11 @@ class HTTPd (threading.Thread):
     def server_conf (self, file_list, server_rules):
         self.server_inst.server_conf (file_list, server_rules)
 
-# vim: set ts=8 sts=4 sw=3 tw=0 et :
+    def server_sett (self, settings):
+         self.server_inst.server_sett (settings)
+
+class HTTPSd (HTTPd):
+
+   server_class = HTTPSServer
+
+# vim: set ts=4 sts=4 sw=4 tw=80 et :

testenv/Makefile.am

@@ -44,6 +44,7 @@ TESTS = Test-auth-basic-fail.py			\
     Test-cookie-expires.py                  \
     Test-cookie.py                          \
     Test-Head.py                            \
+    Test--https.py							\
     Test-O.py                               \
     Test-Post.py                            \
     Test--spider-r.py
@@ -57,6 +58,7 @@ EXTRA_DIST = ColourTerm.py			\
     HTTPServer.py               \
     README                  \
     Test--spider-r.py           \
+    Test--https.py				\
     Test-Content-disposition-2.py       \
     Test-Content-disposition.py     \
     Test-Head.py                \

testenv/Test--https.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python3
+from sys import exit
+from WgetTest import HTTPTest, WgetFile, HTTPS, HTTP
+
+"""
+    This test ensures that Wget can download files from HTTPS Servers
+"""
+TEST_NAME = "HTTPS Downloads"
+############# File Definitions ###############################################
+File1 = "Would you like some Tea?"
+File2 = "With lemon or cream?"
+File3 = "Sure you're joking Mr. Feynman"
+
+A_File = WgetFile ("File1", File1)
+B_File = WgetFile ("File2", File2)
+C_File = WgetFile ("File3", File3)
+
+WGET_OPTIONS = "-d --no-check-certificate"
+WGET_URLS = [["File1", "File2"]]
+
+Files = [[A_File, B_File]]
+Existing_Files = [C_File]
+
+Servers = [HTTPS]
+
+ExpectedReturnCode = 0
+ExpectedDownloadedFiles = [A_File, B_File, C_File]
+
+################ Pre and Post Test Hooks #####################################
+pre_test = {
+    "ServerFiles"       : Files,
+    "LocalFiles"        : Existing_Files
+}
+test_options = {
+    "WgetCommands"      : WGET_OPTIONS,
+    "Urls"              : WGET_URLS
+}
+post_test = {
+    "ExpectedFiles"     : ExpectedDownloadedFiles,
+    "ExpectedRetcode"   : ExpectedReturnCode
+}
+
+err = HTTPTest (
+                name=TEST_NAME,
+                pre_hook=pre_test,
+                test_params=test_options,
+                post_hook=post_test,
+                servers=Servers
+).begin ()
+
+exit (err)

testenv/Test-Parallel-Proto.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 from sys import exit
-from WgetTest import HTTPTest, WgetFile
+from WgetTest import HTTPTest, WgetFile, HTTP, HTTPS
 
 """
     This is a Prototype Test File for multiple servers.
@@ -22,7 +22,7 @@ WGET_URLS = [["File1"], ["File2"]]
 Files = [[A_File], [B_File]]
 Existing_Files = [C_File]
 
-no_of_servers = 2
+Servers = [HTTP, HTTP]
 
 ExpectedReturnCode = 0
 ExpectedDownloadedFiles = [A_File, B_File, C_File]
@@ -46,7 +46,7 @@ err = HTTPTest (
                 pre_hook=pre_test,
                 test_params=test_options,
                 post_hook=post_test,
-                servers=no_of_servers
+                servers=Servers
 ).begin ()
 
 exit (err)

testenv/Test-Proto.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 from sys import exit
-from WgetTest import HTTPTest, WgetFile
+from WgetTest import HTTPTest, WgetFile, HTTP, HTTPS
 
 """
     This is a Prototype Test File.
@@ -40,6 +40,8 @@ C_File = WgetFile ("File3", File3)
 WGET_OPTIONS = "-d --content-disposition --user=Sauron --password=TheEye"
 WGET_URLS = [["File1", "File2"]]
 
+Servers = [HTTP]
+
 Files = [[A_File, B_File]]
 Existing_Files = [C_File]
 
@@ -64,7 +66,8 @@ err = HTTPTest (
                 name=TEST_NAME,
                 pre_hook=pre_test,
                 test_params=test_options,
-                post_hook=post_test
+                post_hook=post_test,
+                server=Servers
 ).begin ()
 
 exit (err)

testenv/WgetTest.py

@@ -5,10 +5,14 @@ import sys
 import traceback
 import HTTPServer
 import re
+import time
 from subprocess import call
 from ColourTerm import printer
 from difflib import unified_diff
 
+HTTP = "HTTP"
+HTTPS = "HTTPS"
+
 """ A Custom Exception raised by the Test Environment. """
 
 class TestFailed (Exception):
@@ -40,7 +44,13 @@ class CommonMethods:
         cmd_line = self.get_cmd_line (options, urls, domain_list)
         params = shlex.split (cmd_line)
         print (params)
+        if os.getenv ("SERVER_WAIT"):
+            time.sleep (float (os.getenv ("SERVER_WAIT")))
+        try:
             retcode = call (params)
+        except FileNotFoundError as filenotfound:
+            raise TestFailed (
+                "The Wget Executable does not exist at the expected path")
         return retcode
 
     def get_cmd_line (self, options, urls, domain_list):
@@ -50,7 +60,8 @@ class CommonMethods:
         cmd_line = WGET_PATH + " " + options + " "
         for i in range (0, self.servers):
             for url in urls[i]:
-                cmd_line += domain_list[i] + url + " "
+                protocol = "http://" if self.server_types[i] is "HTTP" else "https://"
+                cmd_line += protocol + domain_list[i] + url + " "
 #        for url in urls:
 #            cmd_line += domain_list[0] + url + " "
         print (cmd_line)
@@ -174,6 +185,10 @@ class CommonMethods:
             file_handler.write (file_obj.content)
             file_handler.close ()
 
+    def ServerConf (self, server_settings):
+        for i in range (0, self.servers):
+            self.server_list[i].server_sett (server_settings)
+
     """ Test Option Function Calls """
 
     def WgetCommands (self, command_list):
@@ -216,10 +231,10 @@ class HTTPTest (CommonMethods):
         pre_hook=dict(),
         test_params=dict(),
         post_hook=dict(),
-        servers=1
+        servers=[HTTP]
     ):
         try:
-            self.HTTP_setup (name, pre_hook, test_params, post_hook, servers)
+            self.Server_setup (name, pre_hook, test_params, post_hook, servers)
         except TestFailed as tf:
             printer ("RED", "Error: " + tf.error)
             self.tests_passed = False
@@ -232,40 +247,53 @@ class HTTPTest (CommonMethods):
             printer ("GREEN", "Test Passed")
         finally:
             self._exit_test ()
-    def HTTP_setup (self, name, pre_hook, test_params, post_hook, servers):
+
+    def Server_setup (self, name, pre_hook, test_params, post_hook, servers):
         self.name = name
-        self.servers = servers
+        self.server_types = servers
+        self.servers = len (servers)
         printer ("BLUE", "Running Test " + self.name)
         self.init_test_env (name)
         self.server_list = list()
         self.domain_list = list()
-        for server_number in range (0, servers):
+        for server_type in servers:
-            server_inst = self.init_HTTP_Server ()
+            server_inst = getattr (self, "init_" + server_type + "_Server") ()
             self.server_list.append (server_inst)
             domain = self.get_domain_addr (server_inst.server_address)
             self.domain_list.append (domain)
         #self.server = self.init_HTTP_Server ()
         #self.domain = self.get_domain_addr (self.server.server_address)
 
+        self.pre_hook_call (pre_hook)
+        self.call_test (test_params)
+        self.post_hook_call (post_hook)
+
+    def pre_hook_call (self, pre_hook):
         for pre_hook_func in pre_hook:
             try:
                 assert hasattr (self, pre_hook_func)
             except AssertionError as ae:
-                self.stop_HTTP_Server (self.server)
+                self.stop_HTTP_Server ()
                 raise TestFailed ("Pre Test Function " + pre_hook_func + " not defined.")
             getattr (self, pre_hook_func) (pre_hook[pre_hook_func])
 
+    def call_test (self, test_params):
         for test_func in test_params:
             try:
                 assert hasattr (self, test_func)
             except AssertionError as ae:
-                self.stop_HTTP_Server (self.server)
+                self.stop_HTTP_Server ()
                 raise TestFailed ("Test Option " + test_func + " unknown.")
             getattr (self, test_func) (test_params[test_func])
 
+        try:
             self.act_retcode = self.exec_wget (self.options, self.urls, self.domain_list)
+        except TestFailed as tf:
+            self.stop_HTTP_Server ()
+            raise TestFailed (tf.__str__ ())
         self.stop_HTTP_Server ()
 
+    def post_hook_call (self, post_hook):
         for post_hook_func in post_hook:
             try:
                 assert hasattr (self, post_hook_func)
@@ -278,6 +306,11 @@ class HTTPTest (CommonMethods):
         server.start ()
         return server
 
+    def init_HTTPS_Server (self):
+        server = HTTPServer.HTTPSd ()
+        server.start ()
+        return server
+
     def stop_HTTP_Server (self):
         self.Request_remaining = list ()
         for server in self.server_list:
@@ -300,3 +333,5 @@ class WgetFile:
         self.content = content
         self.timestamp = timestamp
         self.rules = rules
+
+# vim: set ts=4 sts=4 sw=4 tw=80 et :

testenv/certs/wget-cert.pem

@@ -0,0 +1,30 @@
+-----BEGIN PRIVATE KEY-----
+MIICdQIBADANBgkqhkiG9w0BAQEFAASCAl8wggJbAgEAAoGBAMV8qEpuSVUdWaAY
+F2N1ljGEJ/907Og5B0aZLeDskmLAOohKMWTiiSx+lseXVD/Zf/LaFfy/+q0Rk5+o
+pFEPEEjadvdxogb9HPwjfj48ng74yV1c5ZGRx/aIeIJN9cacfs4J5NlT3ZPiV8/2
+mpBurBYvta5tneUl+lx4NHTEBmjTAgMBAAECgYBHlFlDMRovWYYEuvavPA2GQQpm
+UzETMqhqdFbmsZiVZmtQvuOMV3e0wuVPzo/g3Kq9kUJq7AKl/DrvoaZ9IuKZgkDD
+0QEBYo/lcxEA9qcfgVs5XLp9ED1mXzJSZ3bmpCDqa2NjG7yFdWzPxc1DXmT05MrF
+bZbb0Wao0tvMwoeJYQJBAOql5uOyjDHvLLuS0IFKbYz4LQwAp7Gjs0ZS9qLNhQQn
+m5Vr8xS9QwFID693K6aDl3tqSCIwSnyInacj8M8v18sCQQDXdReE2i4LKOVLcQsP
+XabN96fFLlnoIh9MqFza4skjhXJWqjBLgJuFqyT5CTbU9TmaoIPXdo4454P1CCgR
+KEIZAkAZE7nlQ8Ov4nvJYBtgde/XTP6jdb52QaR7M4qgQ46frwv1oB/Oa5upm2Xx
+vq6vkQiza9xhqv+K557RqgmmWtqZAkASoXJmL4OZvXCOZHkDXCLHXqnoOAjYNNMm
+Csz0tHWWF7z6V38TmExac6Ef07clFQtlHoooAH1t2D8l2g205hlJAkBfeghbZDdY
+16NtVnvtzjjhKqZFqwTSANFV8NSzgb/QiNnX0hsMPt9bbc5VCo77Ly2oP5SvixfZ
+kjrIQqDV8MLu
+-----END PRIVATE KEY-----
+-----BEGIN CERTIFICATE-----
+MIICODCCAaGgAwIBAgIJAOiSkPuPcAwqMA0GCSqGSIb3DQEBBQUAMDUxCzAJBgNV
+BAYTAklOMRMwEQYDVQQIDApTb21lLVN0YXRlMREwDwYDVQQKDAhHTlUgV2dldDAe
+Fw0xMzEyMDcwNTA3NTRaFw0xNDEyMDcwNTA3NTRaMDUxCzAJBgNVBAYTAklOMRMw
+EQYDVQQIDApTb21lLVN0YXRlMREwDwYDVQQKDAhHTlUgV2dldDCBnzANBgkqhkiG
+9w0BAQEFAAOBjQAwgYkCgYEAxXyoSm5JVR1ZoBgXY3WWMYQn/3Ts6DkHRpkt4OyS
+YsA6iEoxZOKJLH6Wx5dUP9l/8toV/L/6rRGTn6ikUQ8QSNp293GiBv0c/CN+Pjye
+DvjJXVzlkZHH9oh4gk31xpx+zgnk2VPdk+JXz/aakG6sFi+1rm2d5SX6XHg0dMQG
+aNMCAwEAAaNQME4wHQYDVR0OBBYEFLhtTG9a6v3ihL5DeWKfq6doYI42MB8GA1Ud
+IwQYMBaAFLhtTG9a6v3ihL5DeWKfq6doYI42MAwGA1UdEwQFMAMBAf8wDQYJKoZI
+hvcNAQEFBQADgYEApTEZX3cgmgdXDJsu7wtkejtq3vuyi6NXBUlHzoYzWaS5wn8P
+uDG4G9zd1cwmwrbYA8lS+ANWvkcqjM68gMs1ARMZRS0IrYMCN8bokQw+16sqImZO
+THX50Sb5U+9e1IotDWyRBNO10znsoh569BxhJ5WZdIaoKHOJdXEYV+3Y/hg=
+-----END CERTIFICATE-----