test1452: add telnet negotiation

Add a basic telnet server for negotiating some telnet options before
echoing back any data that's sent to it.

Closes #1645

tests/README

@@ -75,6 +75,9 @@ The curl Test Suite
   - TCP/9014 for HTTP pipelining server
   - TCP/9015 for HTTP/2 server
   - TCP/9016 for DICT server
+  - TCP/9017 for SMB server
+  - TCP/9018 for SMBS server (reserved)
+  - TCP/9019 for TELNET server with negotiation support
 
  1.3 Test servers
 

tests/data/Makefile.inc

@@ -155,7 +155,7 @@ test1424 test1425 test1426 \
 test1428 test1429 test1430 test1431 test1432 test1433 test1434 test1435 \
 test1436 test1437 test1438 test1439 test1440 test1441 test1442 test1443 \
 test1444 test1445 test1446                            test1450 test1451 \
-\
+test1452 \
 test1500 test1501 test1502 test1503 test1504 test1505 test1506 test1507 \
 test1508 test1509 test1510 test1511 test1512 test1513 test1514 test1515 \
 test1516 test1517 \

tests/data/test1452

@@ -0,0 +1,41 @@
+<testcase>
+<info>
+<keywords>
+TELNET
+UPLOAD
+</keywords>
+</info>
+
+#
+# Server-side
+<reply>
+<data>
+</data>
+</reply>
+
+#
+# Client-side
+<client>
+<server>
+telnet
+</server>
+<features>
+telnet
+</features>
+ <name>
+Basic TELNET negotiation
+ </name>
+<stdin>
+test1452
+</stdin>
+ <command>
+telnet://%HOSTIP:%NEGTELNETPORT --upload-file -
+</command>
+</client>
+
+#
+# Verify data after the test has been "shot"
+<verify>
+<stdout>test1452</stdout>
+</verify>
+</testcase>

tests/negtelnetserver.py

@@ -0,0 +1,349 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+""" A telnet server which negotiates"""
+
+from __future__ import (absolute_import, division, print_function,
+                        unicode_literals)
+import argparse
+import os
+import sys
+import logging
+import struct
+try:  # Python 2
+    import SocketServer as socketserver
+except ImportError:  # Python 3
+    import socketserver
+
+
+log = logging.getLogger(__name__)
+HOST = "localhost"
+IDENT = "NTEL"
+
+
+# The strings that indicate the test framework is checking our aliveness
+VERIFIED_REQ = b"verifiedserver"
+VERIFIED_RSP = b"WE ROOLZ: {pid}"
+
+
+def telnetserver(options):
+    """
+    Starts up a TCP server with a telnet handler and serves DICT requests
+    forever.
+    """
+    if options.pidfile:
+        pid = os.getpid()
+        with open(options.pidfile, "w") as f:
+            f.write(b"{0}".format(pid))
+
+    local_bind = (HOST, options.port)
+    log.info("Listening on %s", local_bind)
+
+    # Need to set the allow_reuse on the class, not on the instance.
+    socketserver.TCPServer.allow_reuse_address = True
+    server = socketserver.TCPServer(local_bind, NegotiatingTelnetHandler)
+    server.serve_forever()
+
+    return ScriptRC.SUCCESS
+
+
+class NegotiatingTelnetHandler(socketserver.BaseRequestHandler):
+    """Handler class for Telnet connections.
+
+    """
+    def handle(self):
+        """
+        Negotiates options before reading data.
+        """
+        neg = Negotiator(self.request)
+
+        try:
+            # Send some initial negotiations.
+            neg.send_do("NEW_ENVIRON")
+            neg.send_will("NEW_ENVIRON")
+            neg.send_dont("NAWS")
+            neg.send_wont("NAWS")
+
+            # Get the data passed through the negotiator
+            data = neg.recv(1024)
+            log.debug("Incoming data: %r", data)
+
+            if VERIFIED_REQ in data:
+                log.debug("Received verification request from test framework")
+                response_data = VERIFIED_RSP.format(pid=os.getpid())
+            else:
+                log.debug("Received normal request - echoing back")
+                response_data = data.strip()
+
+            if response_data:
+                log.debug("Sending %r", response_data)
+                self.request.sendall(response_data)
+
+        except IOError:
+            log.exception("IOError hit during request")
+
+
+class Negotiator(object):
+    NO_NEG = 0
+    START_NEG = 1
+    WILL = 2
+    WONT = 3
+    DO = 4
+    DONT = 5
+
+    def __init__(self, tcp):
+        self.tcp = tcp
+        self.state = self.NO_NEG
+
+    def recv(self, bytes):
+        """
+        Read bytes from TCP, handling negotiation sequences
+
+        :param bytes: Number of bytes to read
+        :return: a buffer of bytes
+        """
+        buffer = bytearray()
+
+        # If we keep receiving negotiation sequences, we won't fill the buffer.
+        # Keep looping while we can, and until we have something to give back
+        # to the caller.
+        while len(buffer) == 0:
+            data = self.tcp.recv(bytes)
+            if not data:
+                # TCP failed to give us any data. Break out.
+                break
+
+            for byte in data:
+                byte_int = self.byte_to_int(byte)
+
+                if self.state == self.NO_NEG:
+                    self.no_neg(byte, byte_int, buffer)
+                elif self.state == self.START_NEG:
+                    self.start_neg(byte_int)
+                elif self.state in [self.WILL, self.WONT, self.DO, self.DONT]:
+                    self.handle_option(byte_int)
+                else:
+                    # Received an unexpected byte. Stop negotiations
+                    log.error("Unexpected byte %s in state %s",
+                              byte_int,
+                              self.state)
+                    self.state = self.NO_NEG
+
+        return buffer
+
+    def byte_to_int(self, byte):
+        return struct.unpack(b'B', byte)[0]
+
+    def no_neg(self, byte, byte_int, buffer):
+        # Not negotiating anything thus far. Check to see if we
+        # should.
+        if byte_int == NegTokens.IAC:
+            # Start negotiation
+            log.debug("Starting negotiation (IAC)")
+            self.state = self.START_NEG
+        else:
+            # Just append the incoming byte to the buffer
+            buffer.append(byte)
+
+    def start_neg(self, byte_int):
+        # In a negotiation.
+        log.debug("In negotiation (%s)",
+                  NegTokens.from_val(byte_int))
+
+        if byte_int == NegTokens.WILL:
+            # Client is confirming they are willing to do an option
+            log.debug("Client is willing")
+            self.state = self.WILL
+        elif byte_int == NegTokens.WONT:
+            # Client is confirming they are unwilling to do an
+            # option
+            log.debug("Client is unwilling")
+            self.state = self.WONT
+        elif byte_int == NegTokens.DO:
+            # Client is indicating they can do an option
+            log.debug("Client can do")
+            self.state = self.DO
+        elif byte_int == NegTokens.DONT:
+            # Client is indicating they can't do an option
+            log.debug("Client can't do")
+            self.state = self.DONT
+        else:
+            # Received an unexpected byte. Stop negotiations
+            log.error("Unexpected byte %s in state %s",
+                      byte_int,
+                      self.state)
+            self.state = self.NO_NEG
+
+    def handle_option(self, byte_int):
+        if byte_int in [NegOptions.BINARY,
+                        NegOptions.CHARSET,
+                        NegOptions.SUPPRESS_GO_AHEAD,
+                        NegOptions.NAWS,
+                        NegOptions.NEW_ENVIRON]:
+            log.debug("Option: %s", NegOptions.from_val(byte_int))
+
+            # No further negotiation of this option needed. Reset the state.
+            self.state = self.NO_NEG
+
+        else:
+            # Received an unexpected byte. Stop negotiations
+            log.error("Unexpected byte %s in state %s",
+                      byte_int,
+                      self.state)
+            self.state = self.NO_NEG
+
+    def send_message(self, message):
+        packed_message = self.pack(message)
+        self.tcp.sendall(packed_message)
+
+    def pack(self, arr):
+        return struct.pack(b'{0}B'.format(len(arr)), *arr)
+
+    def send_iac(self, arr):
+        message = [NegTokens.IAC]
+        message.extend(arr)
+        self.send_message(message)
+
+    def send_do(self, option_str):
+        log.debug("Sending DO %s", option_str)
+        self.send_iac([NegTokens.DO, NegOptions.to_val(option_str)])
+
+    def send_dont(self, option_str):
+        log.debug("Sending DONT %s", option_str)
+        self.send_iac([NegTokens.DONT, NegOptions.to_val(option_str)])
+
+    def send_will(self, option_str):
+        log.debug("Sending WILL %s", option_str)
+        self.send_iac([NegTokens.WILL, NegOptions.to_val(option_str)])
+
+    def send_wont(self, option_str):
+        log.debug("Sending WONT %s", option_str)
+        self.send_iac([NegTokens.WONT, NegOptions.to_val(option_str)])
+
+
+class NegBase(object):
+    @classmethod
+    def to_val(cls, name):
+        return getattr(cls, name)
+
+    @classmethod
+    def from_val(cls, val):
+        for k in cls.__dict__.keys():
+            if getattr(cls, k) == val:
+                return k
+
+        return "<unknown>"
+
+
+class NegTokens(NegBase):
+    # The start of a negotiation sequence
+    IAC = 255
+    # Confirm willingness to negotiate
+    WILL = 251
+    # Confirm unwillingness to negotiate
+    WONT = 252
+    # Indicate willingness to negotiate
+    DO = 253
+    # Indicate unwillingness to negotiate
+    DONT = 254
+
+    # The start of sub-negotiation options.
+    SB = 250
+    # The end of sub-negotiation options.
+    SE = 240
+
+
+class NegOptions(NegBase):
+    # Binary Transmission
+    BINARY = 0
+    # Suppress Go Ahead
+    SUPPRESS_GO_AHEAD = 3
+    # NAWS - width and height of client
+    NAWS = 31
+    # NEW-ENVIRON - environment variables on client
+    NEW_ENVIRON = 39
+    # Charset option
+    CHARSET = 42
+
+
+def get_options():
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument("--port", action="store", default=9019,
+                        type=int, help="port to listen on")
+    parser.add_argument("--verbose", action="store", type=int, default=0,
+                        help="verbose output")
+    parser.add_argument("--pidfile", action="store",
+                        help="file name for the PID")
+    parser.add_argument("--logfile", action="store",
+                        help="file name for the log")
+    parser.add_argument("--srcdir", action="store", help="test directory")
+    parser.add_argument("--id", action="store", help="server ID")
+    parser.add_argument("--ipv4", action="store_true", default=0,
+                        help="IPv4 flag")
+
+    return parser.parse_args()
+
+
+def setup_logging(options):
+    """
+    Set up logging from the command line options
+    """
+    root_logger = logging.getLogger()
+    add_stdout = False
+
+    formatter = logging.Formatter("%(asctime)s %(levelname)-5.5s "
+                                  "[{ident}] %(message)s"
+                                  .format(ident=IDENT))
+
+    # Write out to a logfile
+    if options.logfile:
+        handler = logging.FileHandler(options.logfile, mode="w")
+        handler.setFormatter(formatter)
+        handler.setLevel(logging.DEBUG)
+        root_logger.addHandler(handler)
+    else:
+        # The logfile wasn't specified. Add a stdout logger.
+        add_stdout = True
+
+    if options.verbose:
+        # Add a stdout logger as well in verbose mode
+        root_logger.setLevel(logging.DEBUG)
+        add_stdout = True
+    else:
+        root_logger.setLevel(logging.INFO)
+
+    if add_stdout:
+        stdout_handler = logging.StreamHandler(sys.stdout)
+        stdout_handler.setFormatter(formatter)
+        stdout_handler.setLevel(logging.DEBUG)
+        root_logger.addHandler(stdout_handler)
+
+
+class ScriptRC(object):
+    """Enum for script return codes"""
+    SUCCESS = 0
+    FAILURE = 1
+    EXCEPTION = 2
+
+
+class ScriptException(Exception):
+    pass
+
+
+if __name__ == '__main__':
+    # Get the options from the user.
+    options = get_options()
+
+    # Setup logging using the user options
+    setup_logging(options)
+
+    # Run main script.
+    try:
+        rc = telnetserver(options)
+    except Exception as e:
+        log.exception(e)
+        rc = ScriptRC.EXCEPTION
+
+    log.info("Returning %d", rc)
+    sys.exit(rc)

tests/runtests.pl

@@ -148,6 +148,7 @@ my $HTTP2PORT;           # HTTP/2 server port
 my $DICTPORT;            # DICT server port
 my $SMBPORT;             # SMB server port
 my $SMBSPORT;            # SMBS server port
+my $NEGTELNETPORT;       # TELNET server port with negotiation
 
 my $srcdir = $ENV{'srcdir'} || '.';
 my $CURL="../src/curl".exe_ext(); # what curl executable to run on the tests
@@ -382,7 +383,7 @@ sub init_serverpidfile_hash {
     }
   }
   for my $proto (('tftp', 'sftp', 'socks', 'ssh', 'rtsp', 'gopher', 'httptls',
-                  'dict', 'smb', 'smbs')) {
+                  'dict', 'smb', 'smbs', 'telnet')) {
     for my $ipvnum ((4, 6)) {
       for my $idnum ((1, 2)) {
         my $serv = servername_id($proto, $ipvnum, $idnum);
@@ -1183,6 +1184,67 @@ sub verifysmb {
     return $pid;
 }
 
+#######################################################################
+# Verify that the server that runs on $ip, $port is our server.  This also
+# implies that we can speak with it, as there might be occasions when the
+# server runs fine but we cannot talk to it ("Failed to connect to ::1: Can't
+# assign requested address")
+#
+sub verifytelnet {
+    my ($proto, $ipvnum, $idnum, $ip, $port) = @_;
+    my $server = servername_id($proto, $ipvnum, $idnum);
+    my $pid = 0;
+    my $time=time();
+    my $extra="";
+
+    my $verifylog = "$LOGDIR/".
+        servername_canon($proto, $ipvnum, $idnum) .'_verify.log';
+    unlink($verifylog) if(-f $verifylog);
+
+    my $flags = "--max-time $server_response_maxtime ";
+    $flags .= "--silent ";
+    $flags .= "--verbose ";
+    $flags .= "--globoff ";
+    $flags .= "--upload-file - ";
+    $flags .= $extra;
+    $flags .= "\"$proto://$ip:$port\"";
+
+    my $cmd = "echo 'verifiedserver' | $VCURL $flags 2>$verifylog";
+
+    # check if this is our server running on this port:
+    logmsg "RUN: $cmd\n" if($verbose);
+    my @data = runclientoutput($cmd);
+
+    my $res = $? >> 8; # rotate the result
+    if($res & 128) {
+        logmsg "RUN: curl command died with a coredump\n";
+        return -1;
+    }
+
+    foreach my $line (@data) {
+        if($line =~ /WE ROOLZ: (\d+)/) {
+            # this is our test server with a known pid!
+            $pid = 0+$1;
+            last;
+        }
+    }
+    if($pid <= 0 && @data && $data[0]) {
+        # this is not a known server
+        logmsg "RUN: Unknown server on our $server port: $port\n";
+        return 0;
+    }
+    # we can/should use the time it took to verify the server as a measure
+    # on how fast/slow this host is.
+    my $took = int(0.5+time()-$time);
+
+    if($verbose) {
+        logmsg "RUN: Verifying our test $server server took $took seconds\n";
+    }
+
+    return $pid;
+}
+
+
 #######################################################################
 # Verify that the server that runs on $ip, $port is our server.
 # Retry over several seconds before giving up.  The ssh server in
@@ -1210,7 +1272,8 @@ my %protofunc = ('http' => \&verifyhttp,
                  'gopher' => \&verifyhttp,
                  'httptls' => \&verifyhttptls,
                  'dict' => \&verifyftp,
-                 'smb' => \&verifysmb);
+                 'smb' => \&verifysmb,
+                 'telnet' => \&verifytelnet);
 
 sub verifyserver {
     my ($proto, $ipvnum, $idnum, $ip, $port) = @_;
@@ -2383,6 +2446,82 @@ sub runsmbserver {
     return ($smbpid, $pid2);
 }
 
+#######################################################################
+# start the telnet server
+#
+sub runnegtelnetserver {
+    my ($verbose, $alt, $port) = @_;
+    my $proto = "telnet";
+    my $ip = $HOSTIP;
+    my $ipvnum = 4;
+    my $idnum = 1;
+    my $server;
+    my $srvrname;
+    my $pidfile;
+    my $logfile;
+    my $flags = "";
+
+    if($alt eq "ipv6") {
+        # No IPv6
+    }
+
+    $server = servername_id($proto, $ipvnum, $idnum);
+
+    $pidfile = $serverpidfile{$server};
+
+    # don't retry if the server doesn't work
+    if ($doesntrun{$pidfile}) {
+        return (0,0);
+    }
+
+    my $pid = processexists($pidfile);
+    if($pid > 0) {
+        stopserver($server, "$pid");
+    }
+    unlink($pidfile) if(-f $pidfile);
+
+    $srvrname = servername_str($proto, $ipvnum, $idnum);
+
+    $logfile = server_logfilename($LOGDIR, $proto, $ipvnum, $idnum);
+
+    $flags .= "--verbose 1 " if($debugprotocol);
+    $flags .= "--pidfile \"$pidfile\" --logfile \"$logfile\" ";
+    $flags .= "--id $idnum " if($idnum > 1);
+    $flags .= "--port $port --srcdir \"$srcdir\"";
+
+    my $cmd = "$srcdir/negtelnetserver.py $flags";
+    my ($ntelpid, $pid2) = startnew($cmd, $pidfile, 15, 0);
+
+    if($ntelpid <= 0 || !pidexists($ntelpid)) {
+        # it is NOT alive
+        logmsg "RUN: failed to start the $srvrname server\n";
+        stopserver($server, "$pid2");
+        displaylogs($testnumcheck);
+        $doesntrun{$pidfile} = 1;
+        return (0,0);
+    }
+
+    # Server is up. Verify that we can speak to it.
+    my $pid3 = verifyserver($proto, $ipvnum, $idnum, $ip, $port);
+    if(!$pid3) {
+        logmsg "RUN: $srvrname server failed verification\n";
+        # failed to talk to it properly. Kill the server and return failure
+        stopserver($server, "$ntelpid $pid2");
+        displaylogs($testnumcheck);
+        $doesntrun{$pidfile} = 1;
+        return (0,0);
+    }
+    $pid2 = $pid3;
+
+    if($verbose) {
+        logmsg "RUN: $srvrname server is now running PID $ntelpid\n";
+    }
+
+    sleep(1);
+
+    return ($ntelpid, $pid2);
+}
+
 
 #######################################################################
 # Single shot http and gopher server responsiveness test. This should only
@@ -2987,6 +3126,8 @@ sub subVariables {
   $$thing =~ s/%SMBPORT/$SMBPORT/g;
   $$thing =~ s/%SMBSPORT/$SMBSPORT/g;
 
+  $$thing =~ s/%NEGTELNETPORT/$NEGTELNETPORT/g;
+
   # server Unix domain socket paths
 
   $$thing =~ s/%HTTPUNIXPATH/$HTTPUNIXPATH/g;
@@ -4860,6 +5001,19 @@ sub startservers {
                 $run{'dict'}="$pid $pid2";
             }
         }
+        elsif($what eq "telnet") {
+            if(!$run{'telnet'}) {
+                ($pid, $pid2) = runnegtelnetserver($verbose,
+                                                   "",
+                                                   $NEGTELNETPORT);
+                if($pid <= 0) {
+                    return "failed starting neg TELNET server";
+                }
+                logmsg sprintf ("* pid neg TELNET => %d %d\n", $pid, $pid2)
+                    if($verbose);
+                $run{'dict'}="$pid $pid2";
+            }
+        }
         elsif($what eq "none") {
             logmsg "* starts no server\n" if ($verbose);
         }
@@ -5322,6 +5476,7 @@ $HTTP2PORT       = $base++; # HTTP/2 port
 $DICTPORT        = $base++; # DICT port
 $SMBPORT         = $base++; # SMB port
 $SMBSPORT        = $base++; # SMBS port
+$NEGTELNETPORT   = $base++; # TELNET port with negotiation
 $HTTPUNIXPATH    = 'http.sock'; # HTTP server Unix domain socket path
 
 #######################################################################

tests/serverhelp.pm

@@ -105,7 +105,7 @@ sub servername_str {
 
     $proto = uc($proto) if($proto);
     die "unsupported protocol: '$proto'" unless($proto &&
-        ($proto =~ /^(((FTP|HTTP|HTTP\/2|IMAP|POP3|SMTP|HTTP-PIPE)S?)|(TFTP|SFTP|SOCKS|SSH|RTSP|GOPHER|HTTPTLS|DICT|SMB|SMBS))$/));
+        ($proto =~ /^(((FTP|HTTP|HTTP\/2|IMAP|POP3|SMTP|HTTP-PIPE)S?)|(TFTP|SFTP|SOCKS|SSH|RTSP|GOPHER|HTTPTLS|DICT|SMB|SMBS|TELNET))$/));
 
     $ipver = (not $ipver) ? 'ipv4' : lc($ipver);
     die "unsupported IP version: '$ipver'" unless($ipver &&