tests: make Python-based servers compatible with Python 2 and 3

Update smbserver.py and negtelnetserver.py to be compatible with
Python 3 while staying backwards-compatible to support Python 2.

Fix string encoding and handling of echoed and transferred data.

Tested with both Python 2.7.17 and Python 3.7.7

Reported-by: Daniel Stenberg
Assisted-by: Kamil Dudka
Reviewed-by: Marcel Raad

Fixes #5104
Closes #5110

.cirrus.yml

@@ -17,7 +17,11 @@ task:
 
   pkginstall_script:
     - pkg update -f
-    - pkg install -y autoconf automake libtool pkgconf brotli openldap-client heimdal libpsl libmetalink libssh2 openssh-portable libidn2 librtmp libnghttp2 nghttp2 stunnel py27-impacket
+    - pkg install -y autoconf automake libtool pkgconf brotli openldap-client heimdal libpsl libmetalink libssh2 openssh-portable libidn2 librtmp libnghttp2 nghttp2 stunnel
+    - case `python -V` in
+        Python?3.7*) pkg install -y py37-impacket ;;
+        Python?2.7*) pkg install -y py27-impacket ;;
+      esac
     - pkg delete -y curl
   configure_script:
     - ./buildconf

tests/README

@@ -39,7 +39,7 @@ The curl Test Suite
  1.1 Requires to run
 
   perl (and a unix-style shell)
-  python (and a unix-style shell)
+  python (and a unix-style shell, for SMB and TELNET tests)
   python-impacket (for SMB tests)
   diff (when a test fails, a diff is shown)
   stunnel (for HTTPS and FTPS tests)
@@ -47,13 +47,28 @@ The curl Test Suite
   nghttpx (for HTTP/2 tests)
   nroff (for --manual tests)
 
-  Please install python-impacket in the correct Python environment.
+ 1.1.1 Installation of python-impacket
-  At the moment the Python-based test servers still require Python 2.
-  Therefore you will need to use pip2 or your OS' package manager to
-  install the Python 2 version of impacket (if it is still available).
 
-  On stable Debian/Ubuntu the package name is "python-impacket".
+  The Python-based test servers support both recent Python 2 and 3.
-  On FreeBSD the package name is "py27-impacket".
+  You can figure out your default Python interpreter with python -V
+
+  Please install python-impacket in the correct Python environment.
+  You can use pip or your OS' package manager to install 'impacket'.
+
+  On Debian/Ubuntu the package names are:
+    Python 2: 'python-impacket'
+    Python 3: 'python3-impacket'
+
+  On FreeBSD the package names are:
+    Python 2: 'py27-impacket'
+    Python 3: 'py37-impacket'
+
+  On any system where pip is available:
+    Python 2: 'pip2 install impacket'
+    Python 3: 'pip3 install impacket'
+
+  You may also need to manually install the Python package 'six'
+  as that may be a missing requirement for impacket on Python 3.
 
  1.2 Port numbers used by test servers
 

tests/curl_test_data.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
-#                                  _   _ ____  _
+#
 #  Project                     ___| | | |  _ \| |
 #                             / __| | | | |_) | |
 #                            | (__| |_| |  _ <| |___

tests/data/test1451

@@ -29,7 +29,7 @@ Basic SMB request
 -u 'curltest:curltest' smb://%HOSTIP:%SMBPORT/TESTS/1451
 </command>
 <precheck>
-python2 -c "__import__('pkgutil').find_loader('impacket') or (__import__('sys').stdout.write('Test only works if Python package impacket is installed\n'), __import__('sys').exit(1))"
+python -c "__import__('pkgutil').find_loader('impacket') or (__import__('sys').stdout.write('Test only works if Python package impacket is installed\n'), __import__('sys').exit(1))"
 </precheck>
 </client>
 

tests/data/test1452

@@ -3,7 +3,6 @@
 <keywords>
 TELNET
 UPLOAD
-flaky
 </keywords>
 </info>
 

tests/negtelnetserver.py

@@ -1,6 +1,24 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 #
+#  Project                     ___| | | |  _ \| |
+#                             / __| | | | |_) | |
+#                            | (__| |_| |  _ <| |___
+#                             \___|\___/|_| \_\_____|
+#
+# Copyright (C) 2017 - 2020, Daniel Stenberg, <daniel@haxx.se>, et al.
+#
+# This software is licensed as described in the file COPYING, which
+# you should have received as part of this distribution. The terms
+# are also available at https://curl.haxx.se/docs/copyright.html.
+#
+# You may opt to use, copy, modify, merge, publish, distribute and/or sell
+# copies of the Software, and permit persons to whom the Software is
+# furnished to do so, under the terms of the COPYING file.
+#
+# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
+# KIND, either express or implied.
+#
 """ A telnet server which negotiates"""
 
 from __future__ import (absolute_import, division, print_function,
@@ -9,11 +27,10 @@ import argparse
 import os
 import sys
 import logging
-try:  # Python 2
+if sys.version_info.major >= 3:
-    import SocketServer as socketserver
-except ImportError:  # Python 3
     import socketserver
-
+else:
+    import SocketServer as socketserver
 
 log = logging.getLogger(__name__)
 HOST = "localhost"
@@ -67,13 +84,13 @@ class NegotiatingTelnetHandler(socketserver.BaseRequestHandler):
             data = neg.recv(1024)
             log.debug("Incoming data: %r", data)
 
-            if VERIFIED_REQ.encode('ascii') in data:
+            if VERIFIED_REQ.encode('utf-8') in data:
                 log.debug("Received verification request from test framework")
                 response = VERIFIED_RSP.format(pid=os.getpid())
-                response_data = response.encode('ascii')
+                response_data = response.encode('utf-8')
             else:
                 log.debug("Received normal request - echoing back")
-                response_data = data.strip()
+                response_data = data.decode('utf-8').strip().encode('utf-8')
 
             if response_data:
                 log.debug("Sending %r", response_data)

tests/smbserver.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python
 # -*- coding: utf-8 -*-
 #
 #  Project                     ___| | | |  _ \| |
@@ -6,7 +6,7 @@
 #                            | (__| |_| |  _ <| |___
 #                             \___|\___/|_| \_\_____|
 #
-# Copyright (C) 2017, Daniel Stenberg, <daniel@haxx.se>, et al.
+# Copyright (C) 2017 - 2020, Daniel Stenberg, <daniel@haxx.se>, et al.
 #
 # This software is licensed as described in the file COPYING, which
 # you should have received as part of this distribution. The terms
@@ -22,7 +22,7 @@
 """Server for testing SMB"""
 
 from __future__ import (absolute_import, division, print_function)
-# unicode_literals)
+# NOTE: the impacket configuration is not unicode_literals compatible!
 import argparse
 import os
 import sys
@@ -52,7 +52,7 @@ log = logging.getLogger(__name__)
 SERVER_MAGIC = "SERVER_MAGIC"
 TESTS_MAGIC = "TESTS_MAGIC"
 VERIFIED_REQ = "verifiedserver"
-VERIFIED_RSP = b"WE ROOLZ: {pid}\n"
+VERIFIED_RSP = "WE ROOLZ: {pid}\n"
 
 
 def smbserver(options):
@@ -267,7 +267,7 @@ class TestSmbServer(imp_smbserver.SMBSERVER):
 
         if requested_filename == VERIFIED_REQ:
             log.debug("[SMB] Verifying server is alive")
-            contents = VERIFIED_RSP.format(pid=os.getpid())
+            contents = VERIFIED_RSP.format(pid=os.getpid()).encode('utf-8')
 
         self.write_to_fid(fid, contents)
         return fid, filename
@@ -288,7 +288,7 @@ class TestSmbServer(imp_smbserver.SMBSERVER):
                   filename, fid, requested_filename)
 
         try:
-            contents = self.ctd.get_test_data(requested_filename)
+            contents = self.ctd.get_test_data(requested_filename).encode('utf-8')
             self.write_to_fid(fid, contents)
             return fid, filename