From 08adf679691006a8fc45fca07c7a10e6a458283e Mon Sep 17 00:00:00 2001 From: Daniel Stenberg Date: Fri, 11 Jan 2008 14:20:41 +0000 Subject: [PATCH] Daniel Egger made CURLOPT_RANGE work on file:// URLs the very same way it already worked for FTP:// URLs --- CHANGES | 3 ++ RELEASE-NOTES | 1 + docs/curl.1 | 6 +-- docs/libcurl/curl_easy_setopt.3 | 2 + lib/file.c | 89 +++++++++++++++++++++++++++++++-- tests/data/Makefile.am | 40 +++++++-------- tests/data/test1016 | 35 +++++++++++++ tests/data/test1017 | 35 +++++++++++++ tests/data/test1018 | 35 +++++++++++++ tests/data/test1019 | 37 ++++++++++++++ tests/data/test1020 | 37 ++++++++++++++ 11 files changed, 292 insertions(+), 28 deletions(-) create mode 100644 tests/data/test1016 create mode 100644 tests/data/test1017 create mode 100644 tests/data/test1018 create mode 100644 tests/data/test1019 create mode 100644 tests/data/test1020 diff --git a/CHANGES b/CHANGES index 032aac0a0..203d94634 100644 --- a/CHANGES +++ b/CHANGES @@ -7,6 +7,9 @@ Changelog Daniel S (11 Jan 2008) +- Daniel Egger made CURLOPT_RANGE work on file:// URLs the very same way it + already worked for FTP:// URLs. + - I made the curl tool switch from using CURLOPT_IOCTLFUNCTION to now use the spanking new CURLOPT_SEEKFUNCTION simply to take advantage of the improved performance for the upload resume cases where you want to upload the last diff --git a/RELEASE-NOTES b/RELEASE-NOTES index 8013284a4..49f540fe9 100644 --- a/RELEASE-NOTES +++ b/RELEASE-NOTES @@ -49,6 +49,7 @@ This release includes the following bugfixes: o time zone offsets from -1400 to +1400 are now accepted by the date parser o allows more spaces in WWW/Proxy-Authenticate: headers o curl-config --libs skips /usr/lib64 + o range support for file:// transfers This release includes the following known bugs: diff --git a/docs/curl.1 b/docs/curl.1 index 2f6d3999c..0babaaaf4 100644 --- a/docs/curl.1 +++ b/docs/curl.1 @@ -974,9 +974,9 @@ This option can be used multiple times. random data. The data is used to seed the random engine for SSL connections. See also the \fI--egd-file\fP option. .IP "-r/--range " -(HTTP/FTP) -Retrieve a byte range (i.e a partial document) from a HTTP/1.1 or FTP -server. Ranges can be specified in a number of ways. +(HTTP/FTP/FILE) Retrieve a byte range (i.e a partial document) from a +HTTP/1.1, FTP server or a local FILE. Ranges can be specified in a number of +ways. .RS .TP 10 .B 0-499 diff --git a/docs/libcurl/curl_easy_setopt.3 b/docs/libcurl/curl_easy_setopt.3 index 56b763fe7..dec4371e1 100644 --- a/docs/libcurl/curl_easy_setopt.3 +++ b/docs/libcurl/curl_easy_setopt.3 @@ -1119,6 +1119,8 @@ transfers also support several intervals, separated with commas as in \fI"X-Y,N-M"\fP. Using this kind of multiple intervals will cause the HTTP server to send the response document in pieces (using standard MIME separation techniques). Pass a NULL to this option to disable the use of ranges. + +Ranges work on HTTP, FTP and FILE (since 7.18.0) transfers only. .IP CURLOPT_RESUME_FROM Pass a long as parameter. It contains the offset in number of bytes that you want the transfer to start from. Set this option to 0 to make the transfer diff --git a/lib/file.c b/lib/file.c index 858226416..cbc9f4ce8 100644 --- a/lib/file.c +++ b/lib/file.c @@ -5,7 +5,7 @@ * | (__| |_| | _ <| |___ * \___|\___/|_| \_\_____| * - * Copyright (C) 1998 - 2007, Daniel Stenberg, , et al. + * Copyright (C) 1998 - 2008, Daniel Stenberg, , et al. * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms @@ -70,6 +70,7 @@ #endif +#include "strtoofft.h" #include "urldata.h" #include #include "progress.h" @@ -119,6 +120,61 @@ const struct Curl_handler Curl_handler_file = { PROT_FILE /* protocol */ }; + + /* + Check if this is a range download, and if so, set the internal variables + properly. This code is copied from the FTP implementation and might as + well be factored out. + */ +static CURLcode file_range(struct connectdata *conn) +{ + curl_off_t from, to; + curl_off_t totalsize=-1; + char *ptr; + char *ptr2; + struct SessionHandle *data = conn->data; + + if(data->state.use_range && data->state.range) { + from=curlx_strtoofft(data->state.range, &ptr, 0); + while(ptr && *ptr && (isspace((int)*ptr) || (*ptr=='-'))) + ptr++; + to=curlx_strtoofft(ptr, &ptr2, 0); + if(ptr == ptr2) { + /* we didn't get any digit */ + to=-1; + } + if((-1 == to) && (from>=0)) { + /* X - */ + data->state.resume_from = from; + DEBUGF(infof(data, "RANGE %" FORMAT_OFF_T " to end of file\n", + from)); + } + else if(from < 0) { + /* -Y */ + totalsize = -from; + data->req.maxdownload = -from; + data->state.resume_from = from; + DEBUGF(infof(data, "RANGE the last %" FORMAT_OFF_T " bytes\n", + totalsize)); + } + else { + /* X-Y */ + totalsize = to-from; + data->req.maxdownload = totalsize+1; /* include last byte */ + data->state.resume_from = from; + DEBUGF(infof(data, "RANGE from %" FORMAT_OFF_T + " getting %" FORMAT_OFF_T " bytes\n", + from, data->req.maxdownload)); + } + DEBUGF(infof(data, "range-download from %" FORMAT_OFF_T + " to %" FORMAT_OFF_T ", totally %" FORMAT_OFF_T " bytes\n", + from, to, data->req.maxdownload)); + } + else + data->req.maxdownload = -1; + return CURLE_OK; +} + /* * file_connect() gets called from Curl_protocol_connect() to allow us to * do protocol-specific actions at connect-time. We emulate a @@ -287,8 +343,8 @@ static CURLcode file_upload(struct connectdata *conn) Curl_pgrsSetUploadSize(data, data->set.infilesize); /* treat the negative resume offset value as the case of "-" */ - if(data->state.resume_from < 0){ - if(stat(file->path, &file_stat)){ + if(data->state.resume_from < 0) { + if(stat(file->path, &file_stat)) { fclose(fp); failf(data, "Can't get the size of %s", file->path); return CURLE_WRITE_ERROR; @@ -434,6 +490,20 @@ static CURLcode file_do(struct connectdata *conn, bool *done) return result; } + /* Check whether file range has been specified */ + file_range(conn); + + /* Adjust the start offset in case we want to get the N last bytes + * of the stream iff the filesize could be determined */ + if(data->state.resume_from < 0) { + if(!fstated) { + failf(data, "Can't get the size of file."); + return CURLE_READ_ERROR; + } + else + data->state.resume_from += (curl_off_t)statbuf.st_size; + } + if(data->state.resume_from <= expected_size) expected_size -= data->state.resume_from; else { @@ -441,6 +511,10 @@ static CURLcode file_do(struct connectdata *conn, bool *done) return CURLE_BAD_DOWNLOAD_RESUME; } + /* A high water mark has been specified so we obey... */ + if (data->req.maxdownload > 0) + expected_size = data->req.maxdownload; + if(fstated && (expected_size == 0)) return CURLE_OK; @@ -460,15 +534,20 @@ static CURLcode file_do(struct connectdata *conn, bool *done) Curl_pgrsTime(data, TIMER_STARTTRANSFER); while(res == CURLE_OK) { - nread = read(fd, buf, BUFSIZE-1); + /* Don't fill a whole buffer if we want less than all data */ + if (expected_size < BUFSIZE-1) + nread = read(fd, buf, expected_size); + else + nread = read(fd, buf, BUFSIZE-1); if( nread > 0) buf[nread] = 0; - if(nread <= 0) + if (nread <= 0 || expected_size == 0) break; bytecount += nread; + expected_size -= nread; res = Curl_client_write(conn, CLIENTWRITE_BODY, buf, nread); if(res) diff --git a/tests/data/Makefile.am b/tests/data/Makefile.am index 4c214bc4f..87a41e438 100644 --- a/tests/data/Makefile.am +++ b/tests/data/Makefile.am @@ -28,26 +28,26 @@ EXTRA_DIST = test1 test108 test117 test127 test20 test27 test34 test46 \ test194 test195 test196 test197 test198 test515 test516 test517 test518 \ test210 test211 test212 test220 test221 test222 test223 test224 test206 \ test207 test208 test209 test213 test240 test241 test242 test519 test214 \ - test215 test216 test217 test218 test199 test225 test226 test227 \ - test228 test229 test233 test234 test235 test236 test520 \ - test237 test238 test239 test243 test245 test246 test247 test248 test249 \ - test250 test251 test252 test253 test254 test255 test521 test522 test523 \ - test256 test257 test258 test259 test260 test261 test262 test263 test264 \ - test265 test266 test267 test268 test269 test270 test271 test272 test273 \ - test274 test275 test524 test525 test276 test277 test526 test527 test528 \ - test530 DISABLED test278 test279 test531 test280 test529 test532 test533 \ - test534 test535 test281 test537 test282 test283 test284 test538 test285 \ - test286 test307 test308 test287 test400 test288 test600 test601 test602 \ - test603 test401 test402 test290 test291 test292 test293 test403 test404 \ - test405 test604 test605 test606 test607 test608 test609 test294 test295 \ - test296 test297 test298 test610 test611 test612 test406 test407 test408 \ - test409 test613 test614 test700 test701 test702 test704 test705 test703 \ - test706 test707 test350 test351 test352 test353 test289 test540 test354 \ - test231 test1000 test1001 test1002 test1003 test1004 test1005 test1006 \ - test615 test1007 test541 test1010 test1011 test1012 test542 test543 \ - test536 test1008 test1009 test2000 test2001 test2002 test2003 test35 \ - test544 test545 test2004 test546 test1013 test1014 test1015 \ - test547 test548 test549 test550 test551 test552 + test215 test216 test217 test218 test199 test225 test226 test227 test228 \ + test229 test233 test234 test235 test236 test520 test237 test238 test239 \ + test243 test245 test246 test247 test248 test249 test250 test251 test252 \ + test253 test254 test255 test521 test522 test523 test256 test257 test258 \ + test259 test260 test261 test262 test263 test264 test265 test266 test267 \ + test268 test269 test270 test271 test272 test273 test274 test275 test524 \ + test525 test276 test277 test526 test527 test528 test530 DISABLED test278 \ + test279 test531 test280 test529 test532 test533 test534 test535 test281 \ + test537 test282 test283 test284 test538 test285 test286 test307 test308 \ + test287 test400 test288 test600 test601 test602 test603 test401 test402 \ + test290 test291 test292 test293 test403 test404 test405 test604 test605 \ + test606 test607 test608 test609 test294 test295 test296 test297 test298 \ + test610 test611 test612 test406 test407 test408 test409 test613 test614 \ + test700 test701 test702 test704 test705 test703 test706 test707 test350 \ + test351 test352 test353 test289 test540 test354 test231 test1000 test1001 \ + test1002 test1003 test1004 test1005 test1006 test615 test1007 test541 \ + test1010 test1011 test1012 test542 test543 test536 test1008 test1009 \ + test2000 test2001 test2002 test2003 test35 test544 test545 test2004 \ + test546 test1013 test1014 test1015 test547 test548 test549 test550 \ + test551 test552 test1016 test1017 test1018 test1019 test1020 filecheck: @mkdir test-place; \ diff --git a/tests/data/test1016 b/tests/data/test1016 new file mode 100644 index 000000000..4b8fc6270 --- /dev/null +++ b/tests/data/test1016 @@ -0,0 +1,35 @@ + + + +FILE + + + + + + + + +# Client-side + + +none + + +X-Y range on a file:// URL to stdout + + +-r 1-4 file://localhost/%PWD/log/test1016.txt + + +1234567890 + + + +# Verify data after the test has been "shot" + + +2345 + + + diff --git a/tests/data/test1017 b/tests/data/test1017 new file mode 100644 index 000000000..8c4c0a811 --- /dev/null +++ b/tests/data/test1017 @@ -0,0 +1,35 @@ + + + +FILE + + +# Server-side + + + + + +# Client-side + + +none + + +0-Y range on a file:// URL to stdout + + +-r 0-3 file://localhost/%PWD/log/test1017.txt + + +1234567890 + + + +# Verify data after the test has been "shot" + + +1234 + + + diff --git a/tests/data/test1018 b/tests/data/test1018 new file mode 100644 index 000000000..c66dc663a --- /dev/null +++ b/tests/data/test1018 @@ -0,0 +1,35 @@ + + + +FILE + + + + + + + + +# Client-side + + +none + + +X-X range on a file:// URL to stdout + + +-r 4-4 file://localhost/%PWD/log/test1018.txt + + +1234567890 + + + +# Verify data after the test has been "shot" + + +5 + + + diff --git a/tests/data/test1019 b/tests/data/test1019 new file mode 100644 index 000000000..f5ab151f5 --- /dev/null +++ b/tests/data/test1019 @@ -0,0 +1,37 @@ + + + +FILE + + +# Server-side + + + + + +# Client-side + + +none + + +X- range on a file:// URL to stdout + + +-r 7- file://localhost/%PWD/log/test1019.txt + + +1234567890 +1234567890 + + + +# Verify data after the test has been "shot" + + +890 +1234567890 + + + diff --git a/tests/data/test1020 b/tests/data/test1020 new file mode 100644 index 000000000..fe834d27c --- /dev/null +++ b/tests/data/test1020 @@ -0,0 +1,37 @@ + + + +FILE + + +# Server-side + + + + + +# Client-side + + +none + + +-Y range on a file:// URL to stdout + + +-r -9 file://localhost/%PWD/log/test1020.txt + + +1234567890 +1234567890 + + + +# Verify data after the test has been "shot" + + +34567890 + + + +