urlapi: add CURLU_GUESS_SCHEME and fix hostname acceptance

In order for this API to fully work for libcurl itself, it now offers a
CURLU_GUESS_SCHEME flag that makes it "guess" scheme based on the host
name prefix just like libcurl always did. If there's no known prefix, it
will guess "http://".

Separately, it relaxes the check of the host name so that IDN host names
can be passed in as well.

Both these changes are necessary for libcurl itself to use this API.

Assisted-by: Daniel Gustafsson
Closes #3018

docs/libcurl/curl_url_set.3

@@ -96,6 +96,16 @@ The query part gets space-to-plus conversion before the URL conversion.
 
 This URL encoding is charset unaware and will convert the input on a
 byte-by-byte manner.
+.IP CURLU_DEFAULT_SCHEME
+If set, will make libcurl allow the URL to be set without a scheme and then
+sets that to the default scheme: HTTPS. Overrides the \fICURLU_GUESS_SCHEME\fP
+option if both are set.
+.IP CURLU_GUESS_SCHEME
+If set, will make libcurl allow the URL to be set without a scheme and it
+instead "guesses" which scheme that was intended based on the host name.  If
+the outermost sub-domain name matches DICT, FTP, IMAP, LDAP, POP3 or SMTP then
+that scheme will be used, otherwise it picks HTTP. Conflicts with the
+\fICURLU_DEFAULT_SCHEME\fP option which takes precendence if both are set.
 .SH RETURN VALUE
 Returns a CURLUcode error value, which is CURLUE_OK (0) if everything went
 fine.

include/curl/urlapi.h

@@ -75,6 +75,7 @@ typedef enum {
 #define CURLU_URLDECODE (1<<6)          /* URL decode on get */
 #define CURLU_URLENCODE (1<<7)          /* URL encode on set */
 #define CURLU_APPENDQUERY (1<<8)        /* append a form style part */
+#define CURLU_GUESS_SCHEME (1<<9)       /* legacy curl-style guessing */
 
 typedef struct Curl_URL CURLU;
 

lib/urlapi.c

@@ -554,7 +554,7 @@ static CURLUcode junkscan(char *part)
 
 static CURLUcode hostname_check(char *hostname, unsigned int flags)
 {
-  const char *l; /* accepted characters */
+  const char *l = NULL; /* accepted characters */
   size_t len;
   size_t hlen = strlen(hostname);
   (void)flags;
@@ -564,14 +564,21 @@ static CURLUcode hostname_check(char *hostname, unsigned int flags)
     l = "0123456789abcdefABCDEF::.";
     hlen -= 2;
   }
-  else /* % for URL escaped letters */
-    l = "0123456789abcdefghijklimnopqrstuvwxyz-_.ABCDEFGHIJKLIMNOPQRSTUVWXYZ%";
 
+  if(l) {
+    /* only valid letters are ok */
     len = strspn(hostname, l);
     if(hlen != len)
       /* hostname with bad content */
       return CURLUE_MALFORMED_INPUT;
-
+  }
+  else {
+    /* letters from the second string is not ok */
+    len = strcspn(hostname, " ");
+    if(hlen != len)
+      /* hostname with bad content */
+      return CURLUE_MALFORMED_INPUT;
+  }
   return CURLUE_OK;
 }
 
@@ -587,7 +594,7 @@ static CURLUcode seturl(const char *url, CURLU *u, unsigned int flags)
   CURLUcode result;
   bool url_has_scheme = FALSE;
   char schemebuf[MAX_SCHEME_LEN];
-  char *schemep;
+  char *schemep = NULL;
   size_t schemelen = 0;
   size_t urllen;
   const struct Curl_handler *h = NULL;
@@ -723,8 +730,9 @@ static CURLUcode seturl(const char *url, CURLU *u, unsigned int flags)
     else {
       /* no scheme! */
 
-      if(!(flags & CURLU_DEFAULT_SCHEME))
+      if(!(flags & (CURLU_DEFAULT_SCHEME|CURLU_GUESS_SCHEME)))
         return CURLUE_MALFORMED_INPUT;
+      if(flags & CURLU_DEFAULT_SCHEME)
         schemep = (char *) DEFAULT_SCHEME;
 
       /*
@@ -744,6 +752,24 @@ static CURLUcode seturl(const char *url, CURLU *u, unsigned int flags)
     memcpy(hostname, hostp, len);
     hostname[len] = 0;
 
+    if((flags & CURLU_GUESS_SCHEME) && !schemep) {
+      /* legacy curl-style guess based on host name */
+      if(checkprefix("ftp.", hostname))
+        schemep = (char *)"ftp";
+      else if(checkprefix("dict.", hostname))
+        schemep = (char *)"dict";
+      else if(checkprefix("ldap.", hostname))
+        schemep = (char *)"ldap";
+      else if(checkprefix("imap.", hostname))
+        schemep = (char *)"imap";
+      else if(checkprefix("smtp.", hostname))
+        schemep = (char *)"smtp";
+      else if(checkprefix("pop3.", hostname))
+        schemep = (char *)"pop3";
+      else
+        schemep = (char *)"http";
+    }
+
     len = strlen(p);
     memcpy(path, p, len);
     path[len] = 0;

tests/data/test1560

@@ -16,6 +16,12 @@ none
 file
 https
 http
+pop3
+smtp
+imap
+ldap
+dict
+ftp
 </features>
  <name>
 URL API

tests/libtest/lib1560.c

@@ -246,8 +246,32 @@ static struct testcase get_parts_list[] ={
 };
 
 static struct urltestcase get_url_list[] = {
+  {"smtp.example.com/path/html",
+   "smtp://smtp.example.com/path/html",
+   CURLU_GUESS_SCHEME, 0, CURLUE_OK},
+  {"https.example.com/path/html",
+   "http://https.example.com/path/html",
+   CURLU_GUESS_SCHEME, 0, CURLUE_OK},
+  {"dict.example.com/path/html",
+   "dict://dict.example.com/path/html",
+   CURLU_GUESS_SCHEME, 0, CURLUE_OK},
+  {"pop3.example.com/path/html",
+   "pop3://pop3.example.com/path/html",
+   CURLU_GUESS_SCHEME, 0, CURLUE_OK},
+  {"ldap.example.com/path/html",
+   "ldap://ldap.example.com/path/html",
+   CURLU_GUESS_SCHEME, 0, CURLUE_OK},
+  {"imap.example.com/path/html",
+   "imap://imap.example.com/path/html",
+   CURLU_GUESS_SCHEME, 0, CURLUE_OK},
+  {"ftp.example.com/path/html",
+   "ftp://ftp.example.com/path/html",
+   CURLU_GUESS_SCHEME, 0, CURLUE_OK},
+  {"example.com/path/html",
+   "http://example.com/path/html",
+   CURLU_GUESS_SCHEME, 0, CURLUE_OK},
   {"HTTP://test/", "http://test/", 0, 0, CURLUE_OK},
-  {"http://HO0_-st..~./", "", 0, 0, CURLUE_MALFORMED_INPUT},
+  {"http://HO0_-st..~./", "http://HO0_-st..~./", 0, 0, CURLUE_OK},
   {"http:/@example.com: 123/", "", 0, 0, CURLUE_BAD_PORT_NUMBER},
   {"http:/@example.com:123 /", "", 0, 0, CURLUE_BAD_PORT_NUMBER},
   {"http:/@example.com:123a/", "", 0, 0, CURLUE_BAD_PORT_NUMBER},