Georg Lippitsch brought CURLOPT_SEEKFUNCTION and CURLOPT_SEEKDATA to allow

libcurl to seek in a given input stream. This is particularly important when
doing upload resumes when there's already a huge part of the file present
remotely. Before, and still if this callback isn't used, libcurl will read
and through away the entire file up to the point to where the resuming
begins (which of course can be a slow opereration depending on file size,
I/O bandwidth and more). This new function will also be preferred to get
used instead of the CURLOPT_IOCTLFUNCTION for seeking back in a stream when
doing multi-stage HTTP auth with POST/PUT.

CHANGES

@@ -7,6 +7,16 @@
                                   Changelog
 
 Daniel S (10 Jan 2008)
+- Georg Lippitsch brought CURLOPT_SEEKFUNCTION and CURLOPT_SEEKDATA to allow
+  libcurl to seek in a given input stream. This is particularly important when
+  doing upload resumes when there's already a huge part of the file present
+  remotely. Before, and still if this callback isn't used, libcurl will read
+  and through away the entire file up to the point to where the resuming
+  begins (which of course can be a slow opereration depending on file size,
+  I/O bandwidth and more). This new function will also be preferred to get
+  used instead of the CURLOPT_IOCTLFUNCTION for seeking back in a stream when
+  doing multi-stage HTTP auth with POST/PUT.
+
 - Nikitinskit Dmitriy filed bug report #1868255
   (http://curl.haxx.se/bug/view.cgi?id=1868255) with a patch. It identifies
   and fixes a problem with parsing WWW-Authenticate: headers with additional

RELEASE-NOTES

@@ -2,7 +2,7 @@ Curl and libcurl 7.18.0
 
  Public curl releases:         103
  Command line options:         125
- curl_easy_setopt() options:   148
+ curl_easy_setopt() options:   150
  Public functions in libcurl:  56
  Public web site mirrors:      42
  Known libcurl bindings:       36
@@ -10,13 +10,14 @@ Curl and libcurl 7.18.0
 
 This release includes the following changes:
  
- o --data-urlencode was added
+ o --data-urlencode
- o CURLOPT_PROXY_TRANSFER_MODE was added
+ o CURLOPT_PROXY_TRANSFER_MODE
- o --no-keep-alive was added, since starting now curl is doing connections with
+ o --no-keep-alive - now curl does connections with keep-alive enabled by
-   keep-alive enabled by default
+   default
  o --socks4a added (proxy type CURLPROXY_SOCKS4A for libcurl)
  o --socks5-hostname added (CURLPROXY_SOCKS5_HOSTNAME for libcurl)
- o curl_easy_pause() added
+ o curl_easy_pause()
+ o CURLOPT_SEEKFUNCTION and CURLOPT_SEEKDATA
 
 This release includes the following bugfixes:
 
@@ -69,6 +70,6 @@ advice from friends like these:
  Emil Romanus, Alessandro Vesely, Ray Pekowski, Spacen Jasset, Andrew Moise,
  Gilles Blanc, David Wright, Vikram Saxena, Mateusz Loskot, Gary Maxwell,
  Dmitry Kurochkin, Mohun Biswas, Richard Atterer, Maxim Perenesenko,
- Daniel Egger, Jeff Johnson
+ Daniel Egger, Jeff Johnson, Nikitinskit Dmitriy, Georg Lippitsch
  
         Thanks! (and sorry if I forgot to mention someone)

docs/libcurl/curl_easy_setopt.3

@@ -171,11 +171,32 @@ something special I/O-related needs to be done that the library can't do by
 itself. For now, rewinding the read data stream is the only action it can
 request. The rewinding of the read data stream may be necessary when doing a
 HTTP PUT or POST with a multi-pass authentication method.  (Option added in
-7.12.3)
+7.12.3).
+
+Use \fICURLOPT_SEEKFUNCTION\fP instead to provide seeking!
 .IP CURLOPT_IOCTLDATA
 Pass a pointer that will be untouched by libcurl and passed as the 3rd
 argument in the ioctl callback set with \fICURLOPT_IOCTLFUNCTION\fP.  (Option
 added in 7.12.3)
+.IP CURLOPT_SEEKFUNCTION
+Function pointer that should match the following prototype: \fIint
+function(void *instream, curl_off_t offset, int origin);\fP This function gets
+called by libcurl to seek to a certain position in the input stream and can be
+used to fast forward a file in a resumed upload (instead of reading all
+uploaded bytes with the normal read function/callback). It is also called to
+rewind a stream when doing a HTTP PUT or POST with a multi-pass authentication
+method. The function shall work like "fseek" or "lseek" and accepted SEEK_SET,
+SEEK_CUR and SEEK_END as argument for origin, although (in 7.18.0) libcurl
+only passes SEEK_SET. The callback must return 0 on success as returning
+non-zero will cause the upload operation to fail.
+
+If you forward the input arguments directly to "fseek" or "lseek", note that
+the data type for \fIoffset\fP is not the same as defined for curl_off_t on
+many systems! (Option added in 7.18.0)
+.IP CURLOPT_SEEKDATA
+Data pointer to pass to the file read function. If you use the
+\fICURLOPT_SEEKFUNCTION\fP option, this is the pointer you'll get as input. If
+you don't specify a seek callback, NULL is passed. (Option added in 7.18.0)
 .IP CURLOPT_SOCKOPTFUNCTION
 Function pointer that should match the \fIcurl_sockopt_callback\fP prototype
 found in \fI<curl/curl.h>\fP. This function gets called by libcurl after the

include/curl/curl.h

@@ -244,6 +244,10 @@ typedef size_t (*curl_write_callback)(char *buffer,
 /* This is a return code for the read callback that, when returned, will
    signal libcurl to pause sending data on the current transfer. */
 #define CURL_READFUNC_PAUSE 0x10000001
+typedef int (*curl_seek_callback)(void *instream,
+                                  curl_off_t offset,
+				  int origin); /* 'whence' */
+
 typedef size_t (*curl_read_callback)(char *buffer,
                                       size_t size,
                                       size_t nitems,
@@ -1180,6 +1184,10 @@ typedef enum {
   /* set transfer mode (;type=<a|i>) when doing FTP via an HTTP proxy */
   CINIT(PROXY_TRANSFER_MODE, LONG, 166),
 
+  /* Callback function for seeking in the input stream */
+  CINIT(SEEKFUNCTION, FUNCTIONPOINT, 167),
+  CINIT(SEEKDATA, OBJECTPOINT, 168),
+
   CURLOPT_LASTENTRY /* the last unused */
 } CURLoption;
 

lib/ftp.c

@@ -1525,7 +1525,6 @@ static CURLcode ftp_state_ul_setup(struct connectdata *conn,
   struct FTP *ftp = conn->data->state.proto.ftp;
   struct SessionHandle *data = conn->data;
   struct ftp_conn *ftpc = &conn->proto.ftpc;
-  curl_off_t passed=0;
 
   if((data->state.resume_from && !sizechecked) ||
      ((data->state.resume_from > 0) && sizechecked)) {
@@ -1552,12 +1551,19 @@ static CURLcode ftp_state_ul_setup(struct connectdata *conn,
     /* enable append */
     data->set.ftp_append = TRUE;
 
-    /* Let's read off the proper amount of bytes from the input. If we knew it
+    /* Let's read off the proper amount of bytes from the input. */
-       was a proper file we could've just fseek()ed but we only have a stream
+    if(conn->seek_func) {
-       here */
+      curl_off_t readthisamountnow = data->state.resume_from;
 
-    /* TODO: allow the ioctlfunction to provide a fast forward function that
+      if(conn->seek_func(conn->seek_client,
-       can be used here and use this method only as a fallback! */
+			 readthisamountnow, SEEK_SET) != 0) {
+        failf(data, "Could not seek stream");
+        return CURLE_FTP_COULDNT_USE_REST;
+      }
+    }
+
+    else {
+      curl_off_t passed=0;
       do {
         curl_off_t readthisamountnow = (data->state.resume_from - passed);
         curl_off_t actuallyread;
@@ -1577,6 +1583,7 @@ static CURLcode ftp_state_ul_setup(struct connectdata *conn,
           return CURLE_FTP_COULDNT_USE_REST;
         }
       } while(passed < data->state.resume_from);
+    }
 
     /* now, decrease the size of the read */
     if(data->set.infilesize>0) {

lib/http.c

@@ -5,7 +5,7 @@
  *                            | (__| |_| |  _ <| |___
  *                             \___|\___/|_| \_\_____|
  *
- * Copyright (C) 1998 - 2007, Daniel Stenberg, <daniel@haxx.se>, et al.
+ * Copyright (C) 1998 - 2008, Daniel Stenberg, <daniel@haxx.se>, et al.
  *
  * This software is licensed as described in the file COPYING, which
  * you should have received as part of this distribution. The terms
@@ -1793,6 +1793,8 @@ CURLcode Curl_http_done(struct connectdata *conn,
   /* set the proper values (possibly modified on POST) */
   conn->fread_func = data->set.fread_func; /* restore */
   conn->fread_in = data->set.in; /* restore */
+  conn->seek_func = data->set.seek_func; /* restore */
+  conn->seek_client = data->set.seek_client; /* restore */
 
   if(http == NULL)
     return CURLE_OK;
@@ -2186,11 +2188,21 @@ CURLcode Curl_http(struct connectdata *conn, bool *done)
 
     if(data->state.resume_from && !data->state.this_is_a_follow) {
       /* do we still game? */
-      curl_off_t passed=0;
 
       /* Now, let's read off the proper amount of bytes from the
-         input. If we knew it was a proper file we could've just
+         input. */
-         fseek()ed but we only have a stream here */
+      if(conn->seek_func) {
+        curl_off_t readthisamountnow = data->state.resume_from;
+
+        if(conn->seek_func(conn->seek_client,
+			   readthisamountnow, SEEK_SET) != 0) {
+          failf(data, "Could not seek stream");
+          return CURLE_READ_ERROR;
+        }
+      }
+      else {
+	curl_off_t passed=0;
+
         do {
 	  size_t readthisamountnow = (size_t)(data->state.resume_from - passed);
           size_t actuallyread;
@@ -2198,8 +2210,8 @@ CURLcode Curl_http(struct connectdata *conn, bool *done)
           if(readthisamountnow > BUFSIZE)
             readthisamountnow = BUFSIZE;
 
-        actuallyread =
+          actuallyread = data->set.fread_func(data->state.buffer, 1,
-          data->set.fread_func(data->state.buffer, 1, (size_t)readthisamountnow,
+                                              (size_t)readthisamountnow,
                                               data->set.in);
 
           passed += actuallyread;
@@ -2210,6 +2222,7 @@ CURLcode Curl_http(struct connectdata *conn, bool *done)
             return CURLE_READ_ERROR;
           }
         } while(passed != data->state.resume_from); /* loop until done */
+      }
 
       /* now, decrease the size of the read */
       if(data->set.infilesize>0) {

lib/transfer.c

@@ -245,10 +245,19 @@ CURLcode Curl_readrewind(struct connectdata *conn)
      (data->set.httpreq == HTTPREQ_POST_FORM))
     ; /* do nothing */
   else {
-    if(data->set.ioctl_func) {
+    if(data->set.seek_func) {
+      int err;
+
+      err = (data->set.seek_func)(data->set.seek_client, 0, SEEK_SET);
+      if(err) {
+	failf(data, "seek callback returned error %d\n", (int)err);
+        return CURLE_SEND_FAIL_REWIND;
+      }
+    }
+    else if(data->set.ioctl_func) {
       curlioerr err;
 
-      err = (data->set.ioctl_func) (data, CURLIOCMD_RESTARTREAD,
+      err = (data->set.ioctl_func)(data, CURLIOCMD_RESTARTREAD,
                                    data->set.ioctl_client);
       infof(data, "the ioctl callback returned %d\n", (int)err);
 

lib/url.c

@@ -685,6 +685,10 @@ CURLcode Curl_open(struct SessionHandle **curl)
     /* use fread as default function to read input */
     data->set.fread_func = (curl_read_callback)fread;
 
+    /* don't use a seek function by default */
+    data->set.seek_func = ZERO_NULL;
+    data->set.seek_client = ZERO_NULL;
+
     /* conversion callbacks for non-ASCII hosts */
     data->set.convfromnetwork = ZERO_NULL;
     data->set.convtonetwork   = ZERO_NULL;
@@ -1627,6 +1631,18 @@ CURLcode Curl_setopt(struct SessionHandle *data, CURLoption option,
       /* When set to NULL, reset to our internal default function */
       data->set.fread_func = (curl_read_callback)fread;
     break;
+  case CURLOPT_SEEKFUNCTION:
+    /*
+     * Seek callback. Might be NULL.
+     */
+    data->set.seek_func = va_arg(param, curl_seek_callback);
+    break;
+  case CURLOPT_SEEKDATA:
+    /*
+     * Seek control callback. Might be NULL.
+     */
+    data->set.seek_client = va_arg(param, void *);
+    break;
   case CURLOPT_CONV_FROM_NETWORK_FUNCTION:
     /*
      * "Convert from network encoding" callback
@@ -4038,6 +4054,8 @@ static CURLcode CreateConnection(struct SessionHandle *data,
    * the persistent connection stuff */
   conn->fread_func = data->set.fread_func;
   conn->fread_in = data->set.in;
+  conn->seek_func = data->set.seek_func;
+  conn->seek_client = data->set.seek_client;
 
   if((conn->protocol&PROT_HTTP) &&
       data->set.upload &&

lib/urldata.h

@@ -961,6 +961,9 @@ struct connectdata {
   size_t buf_len; /* Length of the buffer?? */
 
 
+  curl_seek_callback seek_func; /* function that seeks the input */
+  void *seek_client;            /* pointer to pass to the seek() above */
+
   /*************** Request - specific items ************/
 
   /* previously this was in the urldata struct */
@@ -1324,6 +1327,7 @@ struct UserDefined {
   bool free_referer; /* set TRUE if 'referer' points to a string we
                         allocated */
   void *postfields;  /* if POST, set the fields' values here */
+  curl_seek_callback seek_func;      /* function that seeks the input */
   curl_off_t postfieldsize; /* if POST, this might have a size to use instead
                                of strlen(), and then the data *may* be binary
                                (contain zero bytes) */
@@ -1342,6 +1346,7 @@ struct UserDefined {
                                            the address and opening the socket */
   void* opensocket_client;
 
+  void *seek_client;    /* pointer to pass to the seek callback */
   /* the 3 curl_conv_callback functions below are used on non-ASCII hosts */
   /* function to convert from the network encoding: */
   curl_conv_callback convfromnetwork;

src/main.c

@@ -3020,6 +3020,7 @@ struct InStruct {
   struct Configurable *config;
 };
 
+#if 1
 static curlioerr my_ioctl(CURL *handle, curliocmd cmd, void *userp)
 {
   struct InStruct *in=(struct InStruct *)userp;
@@ -3040,6 +3041,24 @@ static curlioerr my_ioctl(CURL *handle, curliocmd cmd, void *userp)
   }
   return CURLIOE_OK;
 }
+#else
+static int my_seek(void *stream, curl_off_t offset, int whence)
+{
+  struct InStruct *in=(struct InStruct *)stream;
+
+  /* We can't use fseek() here since it can't do 64bit seeks on Windows and
+     possibly elsewhere. We need to switch to the lseek family of tricks. For
+     that to work, we need to switch from fread() to plain read() etc */
+
+  if(-1 == fseek(in->stream, (off_t)offset, whence))
+    /* couldn't rewind, the reason is in errno but errno is just not
+       portable enough and we don't actually care that much why we failed. */
+    return 1;
+
+  return 0;
+}
+
+#endif
 
 static size_t my_fread(void *buffer, size_t sz, size_t nmemb, void *userp)
 {
@@ -4270,10 +4289,17 @@ operate(struct Configurable *config, int argc, argv_item_t argv[])
         /* what call to read */
         my_setopt(curl, CURLOPT_READFUNCTION, my_fread);
 
+#if 1
         /* the ioctl function is at this point only used to rewind files
            that are posted when using NTLM etc */
         my_setopt(curl, CURLOPT_IOCTLDATA, &input);
         my_setopt(curl, CURLOPT_IOCTLFUNCTION, my_ioctl);
+#else
+        /* in 7.18.0, the SEEKFUNCTION/DATA pair is taking over what IOCTL*
+           previously provided for seeking */
+        my_setopt(curl, CURLOPT_SEEKDATA, &input);
+        my_setopt(curl, CURLOPT_SEEKFUNCTION, my_seek);
+#endif
 
         if(config->recvpersecond)
           /* tell libcurl to use a smaller sized buffer as it allows us to