diff --git a/ChangeLog b/ChangeLog index 83daff9c..77cc367a 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,8 @@ +2005-02-20 Hrvoje Niksic + + * configure.in: Check for LFS. Determine SIZEOF_OFF_T. + Check for ftello. + 2005-02-18 Marco Colombo * po/it.po: Updated Italian translation. diff --git a/NEWS b/NEWS index 1cb56b28..05757e79 100644 --- a/NEWS +++ b/NEWS @@ -1,11 +1,15 @@ GNU Wget NEWS -- history of user-visible changes. -Copyright (C) 1997, 1998, 2000, 2001, 2002, 2003 Free Software Foundation, Inc. +Copyright (C) 2005 Free Software Foundation, Inc. See the end for copying conditions. Please send GNU Wget bug reports to . -* Changes in Wget 1.9+. +* Changes in Wget 1.10. + +** Downloading files greater than 2GB, also known as "large files", +now works on systems that support them. This includes most modern +Unix variants, as well as Windows. ** IPv6 is now supported by Wget. Unlike the experimental code in 1.9, this version has no problems with dual-family systems. The new @@ -495,7 +499,7 @@ geturl -vo log http://fly.cc.fer.hr/ ---------------------------------------------------------------------- Copyright information: -Copyright (C) 1997, 1998, 2000, 2001, 2003 Free Software Foundation, Inc. +Copyright (C) 2005 Free Software Foundation, Inc. Permission is granted to anyone to make or distribute verbatim copies of this document as received, in any medium, provided that diff --git a/configure.in b/configure.in index beb3ae4c..f2872f78 100644 --- a/configure.in +++ b/configure.in @@ -182,6 +182,14 @@ AC_CHECK_SIZEOF(int) AC_CHECK_SIZEOF(long) AC_CHECK_SIZEOF(long long) +dnl +dnl Check for large file support. This check needs to come fairly +dnl early because it could (in principle) affect whether functions and +dnl headers are available, whether they work, etc. +dnl +AC_SYS_LARGEFILE +AC_CHECK_SIZEOF(off_t) + dnl dnl Checks for non-universal or system-specific types. dnl @@ -205,9 +213,11 @@ dnl Checks for library functions. dnl AC_FUNC_ALLOCA AC_FUNC_MMAP +AC_FUNC_FSEEKO AC_CHECK_FUNCS(strdup strstr strcasecmp strncasecmp strpbrk memmove) AC_CHECK_FUNCS(gettimeofday mktime strptime strerror snprintf vsnprintf) -AC_CHECK_FUNCS(usleep select sigblock sigsetjmp signal symlink access isatty) +AC_CHECK_FUNCS(usleep select ftello sigblock sigsetjmp signal) +AC_CHECK_FUNCS(symlink access isatty) dnl dnl Call Wget's local macros defined in aclocal. diff --git a/src/ChangeLog b/src/ChangeLog index 57146d70..cf9818fb 100644 --- a/src/ChangeLog +++ b/src/ChangeLog @@ -1,3 +1,35 @@ +2005-02-20 Hrvoje Niksic + + * mswindows.c (wget_ftello): Wget's replacement for ftello. + + * utils.c (file_size): Use ftello where available. + + * ftp-ls.c (ftp_parse_unix_ls): Use str_to_wgint to parse the file + size. + (ftp_parse_winnt_ls): Ditto. + + * ftp-basic.c (ftp_size): Use str_to_wgint to convert number to + wgint; pass 10 instead of 0 as the BASE argument. + + * ftp.c (ftp_expected_bytes): Use str_to_wgint to parse the file + size. + + * sysdep.h (LARGE_INT_FMT): Use __int64 as LARGE_INT on + MSVC/Windows; print it with "%I64". + + * wget.h: Define a `wgint' type, normally aliased to (the possibly + 64-bit variant of) off_t. + + * all: Use `wgint' instead of `long' for numeric variables that + can hold file sizes. + + * utils.c (number_to_string): Support printing of `wgint' + argument. + (number_to_static_string): New function. + + * all: Replace printf("%ld", long_value) with printf("%s", + number_to_static_string(wgint_value)). + 2005-02-18 Mauro Tortonesi * main.c: Added the --ftp-passwd command line option. diff --git a/src/ftp-basic.c b/src/ftp-basic.c index 409bacf1..a246e37c 100644 --- a/src/ftp-basic.c +++ b/src/ftp-basic.c @@ -897,15 +897,13 @@ ftp_cwd (int csock, const char *dir) /* Sends REST command to the FTP server. */ uerr_t -ftp_rest (int csock, long offset) +ftp_rest (int csock, wgint offset) { char *request, *respline; int nwritten; uerr_t err; - static char numbuf[24]; /* Buffer for the number */ - number_to_string (numbuf, offset); - request = ftp_request ("REST", numbuf); + request = ftp_request ("REST", number_to_static_string (offset)); nwritten = fd_write (csock, request, strlen (request), -1); if (nwritten < 0) { @@ -1114,7 +1112,7 @@ ftp_pwd (int csock, char **pwd) /* Sends the SIZE command to the server, and returns the value in 'size'. * If an error occurs, size is set to zero. */ uerr_t -ftp_size (int csock, const char *file, long int *size) +ftp_size (int csock, const char *file, wgint *size) { char *request, *respline; int nwritten; @@ -1150,8 +1148,8 @@ ftp_size (int csock, const char *file, long int *size) } errno = 0; - *size = strtol (respline + 4, NULL, 0); - if (errno) + *size = str_to_wgint (respline + 4, NULL, 10); + if (errno) { /* * Couldn't parse the response for some reason. On the (few) diff --git a/src/ftp-ls.c b/src/ftp-ls.c index d42a9ae7..148c79bf 100644 --- a/src/ftp-ls.c +++ b/src/ftp-ls.c @@ -212,17 +212,28 @@ ftp_parse_unix_ls (const char *file, int ignore_perms) size, and the filename is three tokens away. */ if (i != 12) { - char *t = tok - 2; - long mul = 1; + wgint size; - for (cur.size = 0; t > line && ISDIGIT (*t); mul *= 10, t--) - cur.size += mul * (*t - '0'); + /* Back up to the beginning of the previous token + and parse it with str_to_wgint. */ + char *t = tok - 2; + while (t > line && ISDIGIT (*t)) + --t; if (t == line) { - /* Something is seriously wrong. */ + /* Something has gone wrong during parsing. */ error = 1; break; } + errno = 0; + size = str_to_wgint (t, NULL, 10); + if (size == WGINT_MAX && errno == ERANGE) + /* Out of range -- ignore the size. #### Should + we refuse to start the download. */ + cur.size = 0; + else + cur.size = size; + month = i; next = 5; DEBUGP (("month: %s; ", months[month])); @@ -368,14 +379,14 @@ ftp_parse_unix_ls (const char *file, int ignore_perms) if (!dir) { - l = dir = (struct fileinfo *)xmalloc (sizeof (struct fileinfo)); + l = dir = xnew (struct fileinfo); memcpy (l, &cur, sizeof (cur)); l->prev = l->next = NULL; } else { cur.prev = l; - l->next = (struct fileinfo *)xmalloc (sizeof (struct fileinfo)); + l->next = xnew (struct fileinfo); l = l->next; memcpy (l, &cur, sizeof (cur)); l->next = NULL; @@ -516,10 +527,16 @@ ftp_parse_winnt_ls (const char *file) } else { + wgint size; cur.type = FT_PLAINFILE; - cur.size = atoi(tok); + errno = 0; + size = str_to_wgint (tok, NULL, 10); + if (size == WGINT_MAX && errno == ERANGE) + cur.size = 0; /* overflow */ + else + cur.size = size; cur.perms = 0644; - DEBUGP(("File, size %ld bytes\n", cur.size)); + DEBUGP(("File, size %s bytes\n", number_to_static_string (cur.size))); } cur.linkto = NULL; @@ -527,14 +544,14 @@ ftp_parse_winnt_ls (const char *file) /* And put everything into the linked list */ if (!dir) { - l = dir = (struct fileinfo *)xmalloc (sizeof (struct fileinfo)); + l = dir = xnew (struct fileinfo); memcpy (l, &cur, sizeof (cur)); l->prev = l->next = NULL; } else { cur.prev = l; - l->next = (struct fileinfo *)xmalloc (sizeof (struct fileinfo)); + l->next = xnew (struct fileinfo); l = l->next; memcpy (l, &cur, sizeof (cur)); l->next = NULL; diff --git a/src/ftp.c b/src/ftp.c index 7ad73fb0..3f121a1b 100644 --- a/src/ftp.c +++ b/src/ftp.c @@ -83,12 +83,12 @@ typedef struct /* Look for regexp "( *[0-9]+ *byte" (literal parenthesis) anywhere in - the string S, and return the number converted to long, if found, 0 + the string S, and return the number converted to wgint, if found, 0 otherwise. */ -static long +static wgint ftp_expected_bytes (const char *s) { - long res; + wgint res; while (1) { @@ -96,18 +96,7 @@ ftp_expected_bytes (const char *s) ++s; if (!*s) return 0; - for (++s; *s && ISSPACE (*s); s++); - if (!*s) - return 0; - if (!ISDIGIT (*s)) - continue; - res = 0; - do - { - res = (*s - '0') + 10 * res; - ++s; - } - while (*s && ISDIGIT (*s)); + res = str_to_wgint (s, (char **) &s, 10); if (!*s) return 0; while (*s && ISSPACE (*s)) @@ -234,7 +223,7 @@ ftp_do_port (int csock, int *local_sock) connection to the server. It always closes the data connection, and closes the control connection in case of error. */ static uerr_t -getftp (struct url *u, long *len, long restval, ccon *con) +getftp (struct url *u, wgint *len, wgint restval, ccon *con) { int csock, dtsock, local_sock, res; uerr_t err; @@ -243,10 +232,10 @@ getftp (struct url *u, long *len, long restval, ccon *con) char *tms, *tmrate; int cmd = con->cmd; int pasv_mode_open = 0; - long expected_bytes = 0L; + wgint expected_bytes = 0L; int rest_failed = 0; int flags; - long rd_size; + wgint rd_size; assert (con != NULL); assert (con->target != NULL); @@ -770,7 +759,7 @@ Error in server response, closing control connection.\n")); if (restval && (cmd & DO_RETR)) { if (!opt.server_response) - logprintf (LOG_VERBOSE, "==> REST %ld ... ", restval); + logprintf (LOG_VERBOSE, "==> REST %s ... ", number_to_static_string (restval)); err = ftp_rest (csock, restval); /* FTPRERR, WRITEFAILED, FTPRESTFAIL */ @@ -1123,11 +1112,11 @@ static uerr_t ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con) { int count, orig_lp; - long restval, len = 0; + wgint restval, len = 0; char *tms, *locf; char *tmrate = NULL; uerr_t err; - struct stat st; + struct_stat st; if (!con->target) con->target = url_file_name (u); @@ -1263,16 +1252,16 @@ ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con) con->csock = -1; } if (!opt.spider) - logprintf (LOG_VERBOSE, _("%s (%s) - `%s' saved [%ld]\n\n"), - tms, tmrate, locf, len); + logprintf (LOG_VERBOSE, _("%s (%s) - `%s' saved [%s]\n\n"), + tms, tmrate, locf, number_to_static_string (len)); if (!opt.verbose && !opt.quiet) { /* Need to hide the password from the URL. The `if' is here so that we don't do the needless allocation every time. */ char *hurl = url_string (u, 1); - logprintf (LOG_NONVERBOSE, "%s URL: %s [%ld] -> \"%s\" [%d]\n", - tms, hurl, len, locf, count); + logprintf (LOG_NONVERBOSE, "%s URL: %s [%s] -> \"%s\" [%d]\n", + tms, hurl, number_to_static_string (len), locf, count); xfree (hurl); } @@ -1389,7 +1378,7 @@ ftp_retrieve_list (struct url *u, struct fileinfo *f, ccon *con) static int depth = 0; uerr_t err; struct fileinfo *orig; - long local_size; + wgint local_size; time_t tml; int dlthis; @@ -1440,7 +1429,7 @@ ftp_retrieve_list (struct url *u, struct fileinfo *f, ccon *con) dlthis = 1; if (opt.timestamping && f->type == FT_PLAINFILE) { - struct stat st; + struct_stat st; /* If conversion of HTML files retrieved via FTP is ever implemented, we'll need to stat() .orig here when -K has been specified. I'm not implementing it now since files on an FTP server are much @@ -1482,7 +1471,8 @@ Remote file is newer than local file `%s' -- retrieving.\n\n"), { /* Sizes do not match */ logprintf (LOG_VERBOSE, _("\ -The sizes do not match (local %ld) -- retrieving.\n\n"), local_size); +The sizes do not match (local %s) -- retrieving.\n\n"), + number_to_static_string (local_size)); } } } /* opt.timestamping && f->type == FT_PLAINFILE */ @@ -1501,7 +1491,7 @@ The sizes do not match (local %ld) -- retrieving.\n\n"), local_size); _("Invalid name of the symlink, skipping.\n")); else { - struct stat st; + struct_stat st; /* Check whether we already have the correct symbolic link. */ int rc = lstat (con->target, &st); @@ -1842,15 +1832,15 @@ ftp_loop (struct url *u, int *dt, struct url *proxy) { if (!opt.output_document) { - struct stat st; - long sz; + struct_stat st; + wgint sz; if (stat (filename, &st) == 0) sz = st.st_size; else sz = -1; logprintf (LOG_NOTQUIET, - _("Wrote HTML-ized index to `%s' [%ld].\n"), - filename, sz); + _("Wrote HTML-ized index to `%s' [%s].\n"), + filename, number_to_static_string (sz)); } else logprintf (LOG_NOTQUIET, diff --git a/src/ftp.h b/src/ftp.h index 5de9c5fc..18a957ad 100644 --- a/src/ftp.h +++ b/src/ftp.h @@ -56,11 +56,11 @@ uerr_t ftp_epsv PARAMS ((int, ip_address *, int *)); uerr_t ftp_type PARAMS ((int, int)); uerr_t ftp_cwd PARAMS ((int, const char *)); uerr_t ftp_retr PARAMS ((int, const char *)); -uerr_t ftp_rest PARAMS ((int, long)); +uerr_t ftp_rest PARAMS ((int, wgint)); uerr_t ftp_list PARAMS ((int, const char *)); uerr_t ftp_syst PARAMS ((int, enum stype *)); uerr_t ftp_pwd PARAMS ((int, char **)); -uerr_t ftp_size PARAMS ((int, const char *, long int *)); +uerr_t ftp_size PARAMS ((int, const char *, wgint *)); #ifdef USE_OPIE const char *skey_response PARAMS ((int, const char *, const char *)); @@ -89,7 +89,7 @@ struct fileinfo { enum ftype type; /* file type */ char *name; /* file name */ - long size; /* file size */ + wgint size; /* file size */ long tstamp; /* time-stamp */ int perms; /* file permissions */ char *linkto; /* link to which file points */ diff --git a/src/html-url.c b/src/html-url.c index 59d873b3..4216c796 100644 --- a/src/html-url.c +++ b/src/html-url.c @@ -599,7 +599,7 @@ get_urls_html (const char *file, const char *url, int *meta_disallow_follow) logprintf (LOG_NOTQUIET, "%s: %s\n", file, strerror (errno)); return NULL; } - DEBUGP (("Loaded %s (size %ld).\n", file, fm->length)); + DEBUGP (("Loaded %s (size %s).\n", file, number_to_static_string (fm->length))); ctx.text = fm->content; ctx.head = ctx.tail = NULL; @@ -651,7 +651,7 @@ get_urls_file (const char *file) logprintf (LOG_NOTQUIET, "%s: %s\n", file, strerror (errno)); return NULL; } - DEBUGP (("Loaded %s (size %ld).\n", file, fm->length)); + DEBUGP (("Loaded %s (size %s).\n", file, number_to_static_string (fm->length))); head = tail = NULL; text = fm->content; diff --git a/src/http.c b/src/http.c index 07c3c417..93f072b5 100644 --- a/src/http.c +++ b/src/http.c @@ -219,7 +219,8 @@ release_header (struct request_header *hdr) request_set_header (req, "Referer", opt.referer, rel_none); // Value freshly allocated, free it when done. - request_set_header (req, "Range", aprintf ("bytes=%ld-", hs->restval), + request_set_header (req, "Range", + aprintf ("bytes=%s-", number_to_static_string (hs->restval)), rel_value); */ @@ -359,10 +360,10 @@ request_free (struct request *req) longer, read only that much; if the file is shorter, report an error. */ static int -post_file (int sock, const char *file_name, long promised_size) +post_file (int sock, const char *file_name, wgint promised_size) { static char chunk[8192]; - long written = 0; + wgint written = 0; int write_error; FILE *fp; @@ -705,10 +706,10 @@ print_server_response (const struct response *resp, const char *prefix) /* Parse the `Content-Range' header and extract the information it contains. Returns 1 if successful, -1 otherwise. */ static int -parse_content_range (const char *hdr, long *first_byte_ptr, - long *last_byte_ptr, long *entity_length_ptr) +parse_content_range (const char *hdr, wgint *first_byte_ptr, + wgint *last_byte_ptr, wgint *entity_length_ptr) { - long num; + wgint num; /* Ancient versions of Netscape proxy server, presumably predating rfc2068, sent out `Content-Range' without the "bytes" @@ -751,7 +752,7 @@ parse_content_range (const char *hdr, long *first_byte_ptr, which need to be read anyway. */ static void -skip_short_body (int fd, long contlen) +skip_short_body (int fd, wgint contlen) { /* Skipping the body doesn't make sense if the content length is unknown because, in that case, persistent connections cannot be @@ -759,7 +760,7 @@ skip_short_body (int fd, long contlen) still be used with the magic of the "chunked" transfer!) */ if (contlen == -1) return; - DEBUGP (("Skipping %ld bytes of body data... ", contlen)); + DEBUGP (("Skipping %s bytes of body data... ", number_to_static_string (contlen))); while (contlen > 0) { @@ -974,15 +975,15 @@ persistent_available_p (const char *host, int port, int ssl, struct http_stat { - long len; /* received length */ - long contlen; /* expected length */ - long restval; /* the restart value */ + wgint len; /* received length */ + wgint contlen; /* expected length */ + wgint restval; /* the restart value */ int res; /* the result of last read */ char *newloc; /* new location (redirection) */ char *remote_time; /* remote time-stamp string */ char *error; /* textual HTTP error */ int statcode; /* status code */ - long rd_size; /* amount of data read from socket */ + wgint rd_size; /* amount of data read from socket */ double dltime; /* time it took to download the data */ const char *referer; /* value of the referer header. */ char **local_file; /* local file. */ @@ -1034,7 +1035,7 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy) char *proxyauth; int statcode; int write_error; - long contlen, contrange; + wgint contlen, contrange; struct url *conn; FILE *fp; @@ -1060,7 +1061,7 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy) int inhibit_keep_alive = !opt.http_keep_alive || opt.ignore_length; /* Headers sent when using POST. */ - long post_data_size = 0; + wgint post_data_size = 0; int host_lookup_failed = 0; @@ -1134,7 +1135,9 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy) request_set_header (req, "Pragma", "no-cache", rel_none); if (hs->restval) request_set_header (req, "Range", - aprintf ("bytes=%ld-", hs->restval), rel_value); + aprintf ("bytes=%s-", + number_to_static_string (hs->restval)), + rel_value); if (opt.useragent) request_set_header (req, "User-Agent", opt.useragent, rel_none); else @@ -1259,7 +1262,8 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy) } } request_set_header (req, "Content-Length", - aprintf ("%ld", post_data_size), rel_value); + xstrdup (number_to_static_string (post_data_size)), + rel_value); } /* Add the user headers. */ @@ -1463,7 +1467,20 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy) if (!opt.ignore_length && response_header_copy (resp, "Content-Length", hdrval, sizeof (hdrval))) - contlen = strtol (hdrval, NULL, 10); + { + wgint parsed; + errno = 0; + parsed = str_to_wgint (hdrval, NULL, 10); + if (parsed == WGINT_MAX && errno == ERANGE) + /* Out of range. + #### If Content-Length is out of range, it most likely + means that the file is larger than 2G and that we're + compiled without LFS. In that case we should probably + refuse to even attempt to download the file. */ + contlen = -1; + else + contlen = parsed; + } /* Check for keep-alive related responses. */ if (!inhibit_keep_alive && contlen != -1) @@ -1562,7 +1579,7 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy) } if (response_header_copy (resp, "Content-Range", hdrval, sizeof (hdrval))) { - long first_byte_pos, last_byte_pos, entity_length; + wgint first_byte_pos, last_byte_pos, entity_length; if (parse_content_range (hdrval, &first_byte_pos, &last_byte_pos, &entity_length)) contrange = first_byte_pos; @@ -1768,10 +1785,10 @@ http_loop (struct url *u, char **newloc, char **local_file, const char *referer, char *tms, *locf, *tmrate; uerr_t err; time_t tml = -1, tmr = -1; /* local and remote time-stamps */ - long local_size = 0; /* the size of the local file */ + wgint local_size = 0; /* the size of the local file */ size_t filename_len; struct http_stat hstat; /* HTTP status */ - struct stat st; + struct_stat st; char *dummy = NULL; /* This used to be done in main(), but it's a better idea to do it @@ -1864,7 +1881,7 @@ File `%s' already there, will not retrieve.\n"), *hstat.local_file); point I profiled Wget, and found that a measurable and non-negligible amount of time was lost calling sprintf() in url.c. Replacing sprintf with inline calls to - strcpy() and long_to_string() made a difference. + strcpy() and number_to_string() made a difference. --hniksic */ memcpy (filename_plus_orig_suffix, *hstat.local_file, filename_len); memcpy (filename_plus_orig_suffix + filename_len, @@ -2097,7 +2114,8 @@ Server file no newer than local file `%s' -- not retrieving.\n\n"), } else if (tml >= tmr) logprintf (LOG_VERBOSE, _("\ -The sizes do not match (local %ld) -- retrieving.\n"), local_size); +The sizes do not match (local %s) -- retrieving.\n"), + number_to_static_string (local_size)); else logputs (LOG_VERBOSE, _("Remote file is newer, retrieving.\n")); @@ -2141,11 +2159,16 @@ The sizes do not match (local %ld) -- retrieving.\n"), local_size); if (*dt & RETROKF) { logprintf (LOG_VERBOSE, - _("%s (%s) - `%s' saved [%ld/%ld]\n\n"), - tms, tmrate, locf, hstat.len, hstat.contlen); + _("%s (%s) - `%s' saved [%s/%s]\n\n"), + tms, tmrate, locf, + number_to_static_string (hstat.len), + number_to_static_string (hstat.contlen)); logprintf (LOG_NONVERBOSE, - "%s URL:%s [%ld/%ld] -> \"%s\" [%d]\n", - tms, u->url, hstat.len, hstat.contlen, locf, count); + "%s URL:%s [%s/%s] -> \"%s\" [%d]\n", + tms, u->url, + number_to_static_string (hstat.len), + number_to_static_string (hstat.contlen), + locf, count); } ++opt.numurls; total_downloaded_bytes += hstat.len; @@ -2168,11 +2191,13 @@ The sizes do not match (local %ld) -- retrieving.\n"), local_size); if (*dt & RETROKF) { logprintf (LOG_VERBOSE, - _("%s (%s) - `%s' saved [%ld]\n\n"), - tms, tmrate, locf, hstat.len); + _("%s (%s) - `%s' saved [%s]\n\n"), + tms, tmrate, locf, + number_to_static_string (hstat.len)); logprintf (LOG_NONVERBOSE, - "%s URL:%s [%ld] -> \"%s\" [%d]\n", - tms, u->url, hstat.len, locf, count); + "%s URL:%s [%s] -> \"%s\" [%d]\n", + tms, u->url, number_to_static_string (hstat.len), + locf, count); } ++opt.numurls; total_downloaded_bytes += hstat.len; @@ -2191,8 +2216,8 @@ The sizes do not match (local %ld) -- retrieving.\n"), local_size); connection too soon */ { logprintf (LOG_VERBOSE, - _("%s (%s) - Connection closed at byte %ld. "), - tms, tmrate, hstat.len); + _("%s (%s) - Connection closed at byte %s. "), + tms, tmrate, number_to_static_string (hstat.len)); printwhat (count, opt.ntry); free_hstat (&hstat); continue; @@ -2200,11 +2225,16 @@ The sizes do not match (local %ld) -- retrieving.\n"), local_size); else if (!opt.kill_longer) /* meaning we got more than expected */ { logprintf (LOG_VERBOSE, - _("%s (%s) - `%s' saved [%ld/%ld])\n\n"), - tms, tmrate, locf, hstat.len, hstat.contlen); + _("%s (%s) - `%s' saved [%s/%s])\n\n"), + tms, tmrate, locf, + number_to_static_string (hstat.len), + number_to_static_string (hstat.contlen)); logprintf (LOG_NONVERBOSE, - "%s URL:%s [%ld/%ld] -> \"%s\" [%d]\n", - tms, u->url, hstat.len, hstat.contlen, locf, count); + "%s URL:%s [%s/%s] -> \"%s\" [%d]\n", + tms, u->url, + number_to_static_string (hstat.len), + number_to_static_string (hstat.contlen), + locf, count); ++opt.numurls; total_downloaded_bytes += hstat.len; @@ -2221,8 +2251,10 @@ The sizes do not match (local %ld) -- retrieving.\n"), local_size); else /* the same, but not accepted */ { logprintf (LOG_VERBOSE, - _("%s (%s) - Connection closed at byte %ld/%ld. "), - tms, tmrate, hstat.len, hstat.contlen); + _("%s (%s) - Connection closed at byte %s/%s. "), + tms, tmrate, + number_to_static_string (hstat.len), + number_to_static_string (hstat.contlen)); printwhat (count, opt.ntry); free_hstat (&hstat); continue; @@ -2233,8 +2265,9 @@ The sizes do not match (local %ld) -- retrieving.\n"), local_size); if (hstat.contlen == -1) { logprintf (LOG_VERBOSE, - _("%s (%s) - Read error at byte %ld (%s)."), - tms, tmrate, hstat.len, strerror (errno)); + _("%s (%s) - Read error at byte %s (%s)."), + tms, tmrate, number_to_static_string (hstat.len), + strerror (errno)); printwhat (count, opt.ntry); free_hstat (&hstat); continue; @@ -2242,8 +2275,10 @@ The sizes do not match (local %ld) -- retrieving.\n"), local_size); else /* hstat.res == -1 and contlen is given */ { logprintf (LOG_VERBOSE, - _("%s (%s) - Read error at byte %ld/%ld (%s). "), - tms, tmrate, hstat.len, hstat.contlen, + _("%s (%s) - Read error at byte %s/%s (%s). "), + tms, tmrate, + number_to_static_string (hstat.len), + number_to_static_string (hstat.contlen), strerror (errno)); printwhat (count, opt.ntry); free_hstat (&hstat); diff --git a/src/init.c b/src/init.c index 1aaa13e9..31059183 100644 --- a/src/init.c +++ b/src/init.c @@ -852,8 +852,8 @@ cmd_directory_vector (const char *com, const char *val, void *closure) static int simple_atof PARAMS ((const char *, const char *, double *)); -/* Enginge for cmd_bytes and cmd_bytes_large: converts a string such - as "100k" or "2.5G" to a floating point number. */ +/* Engine for cmd_bytes and cmd_bytes_large: converts a string such as + "100k" or "2.5G" to a floating point number. */ static int parse_bytes_helper (const char *val, double *result) @@ -910,7 +910,7 @@ parse_bytes_helper (const char *val, double *result) } /* Parse VAL as a number and set its value to CLOSURE (which should - point to a long int). + point to a wgint). By default, the value is assumed to be in bytes. If "K", "M", or "G" are appended, the value is multiplied with 1<<10, 1<<20, or @@ -933,7 +933,7 @@ cmd_bytes (const char *com, const char *val, void *closure) exec_name, com, val); return 0; } - *(long *)closure = (long)byte_value; + *(wgint *)closure = (wgint)byte_value; return 1; } diff --git a/src/main.c b/src/main.c index f9827dd8..43772abe 100644 --- a/src/main.c +++ b/src/main.c @@ -870,7 +870,7 @@ Can't timestamp and not clobber old files at the same time.\n")); output_stream = stdout; else { - struct stat st; + struct_stat st; output_stream = fopen (opt.output_document, opt.always_rest ? "ab" : "wb"); if (output_stream == NULL) diff --git a/src/mswindows.c b/src/mswindows.c index 4a9afeef..93bf0c69 100644 --- a/src/mswindows.c +++ b/src/mswindows.c @@ -86,6 +86,146 @@ xsleep (double seconds) #endif /* not HAVE_USLEEP */ } +#if defined(_MSC_VER) && _MSC_VER < 1300 + +static inline int +char_value (char c, int base) +{ + int value; + if (c < '0') + return -1; + if ('0' <= c && c <= '9') + value = c - '0'; + else if ('a' <= c && c <= 'z') + value = c - 'a' + 10; + else if ('A' <= c && c <= 'Z') + value = c - 'A' + 10; + else + return -1; + if (value >= base) + return -1; + return value; +} + +/* A fairly simple strtoll replacement for MS VC versions that don't + supply _strtoi64. */ + +__int64 +str_to_int64 (const char *nptr, char **endptr, int base) +{ +#define OVERFLOW 9223372036854775807I64 +#define UNDERFLOW (-OVERFLOW - 1) + + __int64 result = 0; + int negative; + + if (base != 0 && (base < 2 || base > 36)) + { + errno = EINVAL; + return 0; + } + + while (*nptr == ' ' || *nptr == '\t') + ++nptr; + if (*nptr == '-') + { + negative = 1; + ++nptr; + } + else if (*nptr == '+') + { + negative = 0; + ++nptr; + } + else + negative = 0; + + /* If base is 0, determine the real base based on the beginning on + the number; octal numbers begin with "0", hexadecimal with "0x", + and the others are considered octal. */ + if (*nptr == '0') + { + if ((base == 0 || base == 16) + && + (*(nptr + 1) == 'x' || *(nptr + 1) == 'X')) + { + base = 16; + nptr += 2; + } + else if (base == 0) + base = 8; + } + else if (base == 0) + base = 10; + + if (!negative) + { + /* Parse positive number, checking for overflow. */ + int val; + for (; (val = char_value (*nptr, base)) != -1; ++nptr) + { + __int64 newresult = base * result + val; + if (newresult < result) + { + result = OVERFLOW; + errno = ERANGE; + break; + } + result = newresult; + } + } + else + { + /* Parse negative number, checking for underflow. */ + int val; + for (; (val = char_value (*nptr, base)) != -1; ++nptr) + { + __int64 newresult = base * result - val; + if (newresult > result) + { + result = UNDERFLOW; + errno = ERANGE; + break; + } + result = newresult; + } + } + if (endptr) + *endptr = (char *) nptr; + return result; +} + +#else /* !defined(_MSC_VER) || _MSC_VER >= 1300 */ + +__int64 +str_to_int64 (const char *nptr, char **endptr, int base) +{ +#ifdef _MSC_VER + return _strtoi64 (nptr, endptr, base); +#else + return strtoll (nptr, endptr, base); +#endif +} + +#endif /* !defined(_MSC_VER) || _MSC_VER >= 1300 */ + +/* A simple clone of ftello. The normal ftell doesn't work for large + files, so this is needed, and used by file_size(), which is itself + used for the --post-file option. + + This function uses fgetpos incorrectly and should be considered a + hack until a better way to tell the stream position is found. */ + +__int64 +wget_ftello (FILE *fp) +{ + fpos_t pos; + if (fgetpos (fp, &pos) != 0) + return -1; + else + return pos; +} + void windows_main_junk (int *argc, char **argv, char **exec_name) { diff --git a/src/mswindows.h b/src/mswindows.h index e66fa314..652d6f0e 100644 --- a/src/mswindows.h +++ b/src/mswindows.h @@ -80,9 +80,28 @@ so, delete this exception statement from your version. */ #define snprintf _snprintf #define vsnprintf _vsnprintf -/* No stat on Windows. */ +/* Define a wgint type under Windows. */ +typedef __int64 wgint; +#define SIZEOF_WGINT 8 + +#if defined(_MSC_VER) || defined (__WATCOMC__) +# define WGINT_MAX 9223372036854775807I64 +#else +# define WGINT_MAX 9223372036854775807LL +#endif + +#define str_to_wgint str_to_int64 +__int64 str_to_int64 (const char *, char **, int); + +/* No lstat on Windows. */ #define lstat stat +/* Transparently support large files, in spirit similar to the POSIX + LFS API. */ +#define stat(fname, buf) _stati64 (fname, buf) +#define fstat(fd, buf) _fstati64(fd, buf) +#define struct_stat struct _stati64 + #define PATH_SEPARATOR '\\' /* Microsoft says stat is _stat, Borland doesn't. */ @@ -97,7 +116,12 @@ so, delete this exception statement from your version. */ #endif #endif -#define REALCLOSE(x) closesocket (x) +/* If ftello is unavailable, use an approximation. */ +#ifndef HAVE_FTELLO +__int64 wget_ftello (FILE *); +# define ftello wget_ftello +# define HAVE_FTELLO +#endif /* #### Do we need this? */ #include diff --git a/src/netrc.c b/src/netrc.c index 4d97ff19..bb4dc8c5 100644 --- a/src/netrc.c +++ b/src/netrc.c @@ -84,7 +84,7 @@ search_netrc (const char *host, const char **acc, const char **passwd, if (home) { int err; - struct stat buf; + struct_stat buf; char *path = (char *)alloca (strlen (home) + 1 + strlen (NETRC_FILE_NAME) + 1); sprintf (path, "%s/%s", home, NETRC_FILE_NAME); @@ -471,7 +471,7 @@ free_netrc(acc_t *l) int main (int argc, char **argv) { - struct stat sb; + struct_stat sb; char *program_name, *file, *target; acc_t *head, *a; diff --git a/src/progress.c b/src/progress.c index 3fae615d..937856cd 100644 --- a/src/progress.c +++ b/src/progress.c @@ -52,21 +52,21 @@ so, delete this exception statement from your version. */ struct progress_implementation { const char *name; int interactive; - void *(*create) PARAMS ((long, long)); - void (*update) PARAMS ((void *, long, double)); + void *(*create) PARAMS ((wgint, wgint)); + void (*update) PARAMS ((void *, wgint, double)); void (*finish) PARAMS ((void *, double)); void (*set_params) PARAMS ((const char *)); }; /* Necessary forward declarations. */ -static void *dot_create PARAMS ((long, long)); -static void dot_update PARAMS ((void *, long, double)); +static void *dot_create PARAMS ((wgint, wgint)); +static void dot_update PARAMS ((void *, wgint, double)); static void dot_finish PARAMS ((void *, double)); static void dot_set_params PARAMS ((const char *)); -static void *bar_create PARAMS ((long, long)); -static void bar_update PARAMS ((void *, long, double)); +static void *bar_create PARAMS ((wgint, wgint)); +static void bar_update PARAMS ((void *, wgint, double)); static void bar_finish PARAMS ((void *, double)); static void bar_set_params PARAMS ((const char *)); @@ -156,7 +156,7 @@ progress_schedule_redirect (void) advance. */ void * -progress_create (long initial, long total) +progress_create (wgint initial, wgint total) { /* Check if the log status has changed under our feet. */ if (output_redirected) @@ -184,7 +184,7 @@ progress_interactive_p (void *progress) time in milliseconds since the beginning of the download. */ void -progress_update (void *progress, long howmuch, double dltime) +progress_update (void *progress, wgint howmuch, double dltime) { current_impl->update (progress, howmuch, dltime); } @@ -201,9 +201,9 @@ progress_finish (void *progress, double dltime) /* Dot-printing. */ struct dot_progress { - long initial_length; /* how many bytes have been downloaded + wgint initial_length; /* how many bytes have been downloaded previously. */ - long total_length; /* expected total byte count when the + wgint total_length; /* expected total byte count when the download finishes */ int accumulated; @@ -216,7 +216,7 @@ struct dot_progress { /* Dot-progress backend for progress_create. */ static void * -dot_create (long initial, long total) +dot_create (wgint initial, wgint total) { struct dot_progress *dp = xnew0 (struct dot_progress); dp->initial_length = initial; @@ -225,10 +225,10 @@ dot_create (long initial, long total) if (dp->initial_length) { int dot_bytes = opt.dot_bytes; - long row_bytes = opt.dot_bytes * opt.dots_in_line; + wgint row_bytes = opt.dot_bytes * opt.dots_in_line; int remainder = (int) (dp->initial_length % row_bytes); - long skipped = dp->initial_length - remainder; + wgint skipped = dp->initial_length - remainder; if (skipped) { @@ -244,7 +244,7 @@ dot_create (long initial, long total) 2 + skipped_k_len, "", skipped_k); } - logprintf (LOG_VERBOSE, "\n%5ldK", skipped / 1024); + logprintf (LOG_VERBOSE, "\n%5ldK", (long) (skipped / 1024)); for (; remainder >= dot_bytes; remainder -= dot_bytes) { if (dp->dots % opt.dot_spacing == 0) @@ -262,14 +262,14 @@ dot_create (long initial, long total) } static void -print_percentage (long bytes, long expected) +print_percentage (wgint bytes, wgint expected) { int percentage = (int)(100.0 * bytes / expected); logprintf (LOG_VERBOSE, "%3d%%", percentage); } static void -print_download_speed (struct dot_progress *dp, long bytes, double dltime) +print_download_speed (struct dot_progress *dp, wgint bytes, double dltime) { logprintf (LOG_VERBOSE, " %s", retr_rate (bytes, dltime - dp->last_timer_value, 1)); @@ -279,11 +279,11 @@ print_download_speed (struct dot_progress *dp, long bytes, double dltime) /* Dot-progress backend for progress_update. */ static void -dot_update (void *progress, long howmuch, double dltime) +dot_update (void *progress, wgint howmuch, double dltime) { struct dot_progress *dp = progress; int dot_bytes = opt.dot_bytes; - long row_bytes = opt.dot_bytes * opt.dots_in_line; + wgint row_bytes = opt.dot_bytes * opt.dots_in_line; log_set_flush (0); @@ -291,7 +291,7 @@ dot_update (void *progress, long howmuch, double dltime) for (; dp->accumulated >= dot_bytes; dp->accumulated -= dot_bytes) { if (dp->dots == 0) - logprintf (LOG_VERBOSE, "\n%5ldK", dp->rows * row_bytes / 1024); + logprintf (LOG_VERBOSE, "\n%5ldK", (long) (dp->rows * row_bytes / 1024)); if (dp->dots % opt.dot_spacing == 0) logputs (LOG_VERBOSE, " "); @@ -300,7 +300,7 @@ dot_update (void *progress, long howmuch, double dltime) ++dp->dots; if (dp->dots >= opt.dots_in_line) { - long row_qty = row_bytes; + wgint row_qty = row_bytes; if (dp->rows == dp->initial_length / row_bytes) row_qty -= dp->initial_length % row_bytes; @@ -323,13 +323,13 @@ dot_finish (void *progress, double dltime) { struct dot_progress *dp = progress; int dot_bytes = opt.dot_bytes; - long row_bytes = opt.dot_bytes * opt.dots_in_line; + wgint row_bytes = opt.dot_bytes * opt.dots_in_line; int i; log_set_flush (0); if (dp->dots == 0) - logprintf (LOG_VERBOSE, "\n%5ldK", dp->rows * row_bytes / 1024); + logprintf (LOG_VERBOSE, "\n%5ldK", (long) (dp->rows * row_bytes / 1024)); for (i = dp->dots; i < opt.dots_in_line; i++) { if (i % opt.dot_spacing == 0) @@ -345,7 +345,7 @@ dot_finish (void *progress, double dltime) } { - long row_qty = dp->dots * dot_bytes + dp->accumulated; + wgint row_qty = dp->dots * dot_bytes + dp->accumulated; if (dp->rows == dp->initial_length / row_bytes) row_qty -= dp->initial_length % row_bytes; print_download_speed (dp, row_qty, dltime); @@ -443,11 +443,11 @@ static volatile sig_atomic_t received_sigwinch; #define STALL_START_TIME 5000 struct bar_progress { - long initial_length; /* how many bytes have been downloaded + wgint initial_length; /* how many bytes have been downloaded previously. */ - long total_length; /* expected total byte count when the + wgint total_length; /* expected total byte count when the download finishes */ - long count; /* bytes downloaded so far */ + wgint count; /* bytes downloaded so far */ double last_screen_update; /* time of the last screen update, measured since the beginning of @@ -470,18 +470,18 @@ struct bar_progress { details. */ struct bar_progress_hist { int pos; - long times[DLSPEED_HISTORY_SIZE]; - long bytes[DLSPEED_HISTORY_SIZE]; + wgint times[DLSPEED_HISTORY_SIZE]; + wgint bytes[DLSPEED_HISTORY_SIZE]; /* The sum of times and bytes respectively, maintained for efficiency. */ - long total_time; - long total_bytes; + wgint total_time; + wgint total_bytes; } hist; double recent_start; /* timestamp of beginning of current position. */ - long recent_bytes; /* bytes downloaded so far. */ + wgint recent_bytes; /* bytes downloaded so far. */ int stalled; /* set when no data arrives for longer than STALL_START_TIME, then reset @@ -492,14 +492,14 @@ struct bar_progress { double last_eta_time; /* time of the last update to download speed and ETA, measured since the beginning of download. */ - long last_eta_value; + wgint last_eta_value; }; static void create_image PARAMS ((struct bar_progress *, double)); static void display_image PARAMS ((char *)); static void * -bar_create (long initial, long total) +bar_create (wgint initial, wgint total) { struct bar_progress *bp = xnew0 (struct bar_progress); @@ -536,10 +536,10 @@ bar_create (long initial, long total) return bp; } -static void update_speed_ring PARAMS ((struct bar_progress *, long, double)); +static void update_speed_ring PARAMS ((struct bar_progress *, wgint, double)); static void -bar_update (void *progress, long howmuch, double dltime) +bar_update (void *progress, wgint howmuch, double dltime) { struct bar_progress *bp = progress; int force_screen_update = 0; @@ -622,7 +622,7 @@ bar_finish (void *progress, double dltime) 3-second average would be too erratic. */ static void -update_speed_ring (struct bar_progress *bp, long howmuch, double dltime) +update_speed_ring (struct bar_progress *bp, wgint howmuch, double dltime) { struct bar_progress_hist *hist = &bp->hist; double recent_age = dltime - bp->recent_start; @@ -719,7 +719,7 @@ static void create_image (struct bar_progress *bp, double dl_total_time) { char *p = bp->buffer; - long size = bp->initial_length + bp->count; + wgint size = bp->initial_length + bp->count; char *size_legible = legible (size); int size_legible_len = strlen (size_legible); @@ -838,7 +838,7 @@ create_image (struct bar_progress *bp, double dl_total_time) int units = 0; /* Calculate the download speed using the history ring and recent data that hasn't made it to the ring yet. */ - long dlquant = hist->total_bytes + bp->recent_bytes; + wgint dlquant = hist->total_bytes + bp->recent_bytes; double dltime = hist->total_time + (dl_total_time - bp->recent_start); double dlspeed = calc_rate (dlquant, dltime, &units); sprintf (p, " %7.2f%s", dlspeed, short_units[units]); @@ -852,7 +852,7 @@ create_image (struct bar_progress *bp, double dl_total_time) reliable. */ if (bp->total_length > 0 && bp->count > 0 && dl_total_time > 3000) { - long eta; + wgint eta; int eta_hrs, eta_min, eta_sec; /* Don't change the value of ETA more than approximately once @@ -871,8 +871,8 @@ create_image (struct bar_progress *bp, double dl_total_time) I found that doing that results in a very jerky and ultimately unreliable ETA. */ double time_sofar = (double)dl_total_time / 1000; - long bytes_remaining = bp->total_length - size; - eta = (long) (time_sofar * bytes_remaining / bp->count); + wgint bytes_remaining = bp->total_length - size; + eta = (wgint) (time_sofar * bytes_remaining / bp->count); bp->last_eta_value = eta; bp->last_eta_time = dl_total_time; } diff --git a/src/progress.h b/src/progress.h index 5437a1d5..bbe63492 100644 --- a/src/progress.h +++ b/src/progress.h @@ -34,9 +34,9 @@ int valid_progress_implementation_p PARAMS ((const char *)); void set_progress_implementation PARAMS ((const char *)); void progress_schedule_redirect PARAMS ((void)); -void *progress_create PARAMS ((long, long)); +void *progress_create PARAMS ((wgint, wgint)); int progress_interactive_p PARAMS ((void *)); -void progress_update PARAMS ((void *, long, double)); +void progress_update PARAMS ((void *, wgint, double)); void progress_finish PARAMS ((void *, double)); RETSIGTYPE progress_handle_sigwinch PARAMS ((int)); diff --git a/src/retr.c b/src/retr.c index 605db831..339a2fa0 100644 --- a/src/retr.c +++ b/src/retr.c @@ -75,7 +75,7 @@ FILE *output_stream; int output_stream_regular; static struct { - long chunk_bytes; + wgint chunk_bytes; double chunk_start; double sleep_adjust; } limit_data; @@ -92,7 +92,7 @@ limit_bandwidth_reset (void) is the timer that started at the beginning of download. */ static void -limit_bandwidth (long bytes, struct wget_timer *timer) +limit_bandwidth (wgint bytes, struct wget_timer *timer) { double delta_t = wtimer_read (timer) - limit_data.chunk_start; double expected; @@ -110,12 +110,14 @@ limit_bandwidth (long bytes, struct wget_timer *timer) double t0, t1; if (slp < 200) { - DEBUGP (("deferring a %.2f ms sleep (%ld/%.2f).\n", - slp, limit_data.chunk_bytes, delta_t)); + DEBUGP (("deferring a %.2f ms sleep (%s/%.2f).\n", + slp, number_to_static_string (limit_data.chunk_bytes), + delta_t)); return; } - DEBUGP (("\nsleeping %.2f ms for %ld bytes, adjust %.2f ms\n", - slp, limit_data.chunk_bytes, limit_data.sleep_adjust)); + DEBUGP (("\nsleeping %.2f ms for %s bytes, adjust %.2f ms\n", + slp, number_to_static_string (limit_data.chunk_bytes), + limit_data.sleep_adjust)); t0 = wtimer_read (timer); xsleep (slp / 1000); @@ -142,8 +144,8 @@ limit_bandwidth (long bytes, struct wget_timer *timer) of data written. */ static int -write_data (FILE *out, const char *buf, int bufsize, long *skip, - long *written) +write_data (FILE *out, const char *buf, int bufsize, wgint *skip, + wgint *written) { if (!out) return 1; @@ -192,8 +194,8 @@ write_data (FILE *out, const char *buf, int bufsize, long *skip, writing data, -2 is returned. */ int -fd_read_body (int fd, FILE *out, long toread, long startpos, - long *qtyread, long *qtywritten, double *elapsed, int flags) +fd_read_body (int fd, FILE *out, wgint toread, wgint startpos, + wgint *qtyread, wgint *qtywritten, double *elapsed, int flags) { int ret = 0; @@ -213,11 +215,11 @@ fd_read_body (int fd, FILE *out, long toread, long startpos, int progress_interactive = 0; int exact = flags & rb_read_exactly; - long skip = 0; + wgint skip = 0; /* How much data we've read/written. */ - long sum_read = 0; - long sum_written = 0; + wgint sum_read = 0; + wgint sum_written = 0; if (flags & rb_skip_startpos) skip = startpos; @@ -489,7 +491,7 @@ fd_read_line (int fd) appropriate for the speed. If PAD is non-zero, strings will be padded to the width of 7 characters (xxxx.xx). */ char * -retr_rate (long bytes, double msecs, int pad) +retr_rate (wgint bytes, double msecs, int pad) { static char res[20]; static const char *rate_names[] = {"B/s", "KB/s", "MB/s", "GB/s" }; @@ -509,7 +511,7 @@ retr_rate (long bytes, double msecs, int pad) UNITS is zero for B/s, one for KB/s, two for MB/s, and three for GB/s. */ double -calc_rate (long bytes, double msecs, int *units) +calc_rate (wgint bytes, double msecs, int *units) { double dlrate; @@ -912,7 +914,7 @@ rotate_backups(const char *fname) int maxlen = strlen (fname) + 1 + numdigit (opt.backups) + 1; char *from = (char *)alloca (maxlen); char *to = (char *)alloca (maxlen); - struct stat sb; + struct_stat sb; int i; if (stat (fname, &sb) == 0) diff --git a/src/retr.h b/src/retr.h index 9f9a3026..fa90ea95 100644 --- a/src/retr.h +++ b/src/retr.h @@ -36,7 +36,7 @@ enum { rb_skip_startpos = 2 }; -int fd_read_body PARAMS ((int, FILE *, long, long, long *, long *, double *, +int fd_read_body PARAMS ((int, FILE *, wgint, wgint, wgint *, wgint *, double *, int)); typedef const char *(*hunk_terminator_t) PARAMS ((const char *, int, int)); @@ -48,8 +48,8 @@ uerr_t retrieve_url PARAMS ((const char *, char **, char **, const char *, int *)); uerr_t retrieve_from_file PARAMS ((const char *, int, int *)); -char *retr_rate PARAMS ((long, double, int)); -double calc_rate PARAMS ((long, double, int *)); +char *retr_rate PARAMS ((wgint, double, int)); +double calc_rate PARAMS ((wgint, double, int *)); void printwhat PARAMS ((int, int)); void sleep_between_retrievals PARAMS ((int)); diff --git a/src/sysdep.h b/src/sysdep.h index 9b875e09..2aaa757d 100644 --- a/src/sysdep.h +++ b/src/sysdep.h @@ -111,7 +111,7 @@ so, delete this exception statement from your version. */ /* Define a large integral type useful for storing large sizes that exceed sizes of one download, such as when printing the sum of all downloads. Note that this has nothing to do with large file - support, yet. + support, which determines the wgint type. We use a 64-bit integral type where available, `double' otherwise. It's hard to print LARGE_INT's portably, but fortunately it's @@ -127,12 +127,38 @@ typedef long LARGE_INT; typedef long long LARGE_INT; # define LARGE_INT_FMT "%lld" # else +# if _MSC_VER +/* Use __int64 under Windows. */ +typedef __int64 LARGE_INT; +# define LARGE_INT_FMT "%I64" +# else /* Large integer type unavailable; use `double' instead. */ typedef double LARGE_INT; -# define LARGE_INT_FMT "%.0f" +# define LARGE_INT_FMT "%.0f" +# endif # endif #endif +/* Under Windows we #define struct_stat to struct _stati64. */ +#ifndef struct_stat +# define struct_stat struct stat +#endif + +#ifdef HAVE_LIMITS_H +# include +#endif + +#ifndef CHAR_BIT +# define CHAR_BIT 8 +#endif + +#ifndef LONG_MAX +# define LONG_MAX ((long) ~((unsigned long)1 << (CHAR_BIT * sizeof (long) - 1))) +#endif +#ifndef LLONG_MAX +# define LLONG_MAX ((long long) ~((unsigned long long)1 << (CHAR_BIT * sizeof (long long) - 1))) +#endif + /* These are defined in cmpt.c if missing, therefore it's generally safe to declare their parameters. */ #ifndef HAVE_STRERROR diff --git a/src/url.c b/src/url.c index 180a3bc0..8aed967d 100644 --- a/src/url.c +++ b/src/url.c @@ -1279,7 +1279,7 @@ mkalldirs (const char *path) { const char *p; char *t; - struct stat st; + struct_stat st; int res; p = path + strlen (path); diff --git a/src/utils.c b/src/utils.c index 2bb59935..7259d019 100644 --- a/src/utils.c +++ b/src/utils.c @@ -337,7 +337,7 @@ int remove_link (const char *file) { int err = 0; - struct stat st; + struct_stat st; if (lstat (file, &st) == 0 && S_ISLNK (st.st_mode)) { @@ -363,7 +363,7 @@ file_exists_p (const char *filename) #ifdef HAVE_ACCESS return access (filename, F_OK) >= 0; #else - struct stat buf; + struct_stat buf; return stat (filename, &buf) >= 0; #endif } @@ -373,7 +373,7 @@ file_exists_p (const char *filename) int file_non_directory_p (const char *path) { - struct stat buf; + struct_stat buf; /* Use lstat() rather than stat() so that symbolic links pointing to directories can be identified correctly. */ if (lstat (path, &buf) != 0) @@ -383,20 +383,28 @@ file_non_directory_p (const char *path) /* Return the size of file named by FILENAME, or -1 if it cannot be opened or seeked into. */ -long +wgint file_size (const char *filename) { - long size; +#if defined(HAVE_FSEEKO) && defined(HAVE_FTELLO) + wgint size; /* We use fseek rather than stat to determine the file size because - that way we can also verify whether the file is readable. - Inspired by the POST patch by Arnaud Wylie. */ + that way we can also verify that the file is readable without + explicitly checking for permissions. Inspired by the POST patch + by Arnaud Wylie. */ FILE *fp = fopen (filename, "rb"); if (!fp) return -1; - fseek (fp, 0, SEEK_END); - size = ftell (fp); + fseeko (fp, 0, SEEK_END); + size = ftello (fp); fclose (fp); return size; +#else + struct_stat st; + if (stat (filename, &st) < 0) + return -1; + return st.st_size; +#endif } /* stat file names named PREFIX.1, PREFIX.2, etc., until one that @@ -799,7 +807,7 @@ read_file (const char *file) { int fd; struct file_memory *fm; - long size; + wgint size; int inhibit_close = 0; /* Some magic in the finest tradition of Perl and its kin: if FILE @@ -819,7 +827,7 @@ read_file (const char *file) #ifdef HAVE_MMAP { - struct stat buf; + struct_stat buf; if (fstat (fd, &buf) < 0) goto mmap_lose; fm->length = buf.st_size; @@ -851,7 +859,7 @@ read_file (const char *file) fm->content = xmalloc (size); while (1) { - long nread; + wgint nread; if (fm->length > size / 2) { /* #### I'm not sure whether the whole exponential-growth @@ -1148,10 +1156,10 @@ legible_1 (const char *repr) return outbuf; } -/* Legible -- return a static pointer to the legibly printed long. */ +/* Legible -- return a static pointer to the legibly printed wgint. */ char * -legible (long l) +legible (wgint l) { char inbuf[24]; /* Print the number into the buffer. */ @@ -1185,9 +1193,9 @@ legible_large_int (LARGE_INT l) return legible_1 (inbuf); } -/* Count the digits in a (long) integer. */ +/* Count the digits in an integer number. */ int -numdigit (long number) +numdigit (wgint number) { int cnt = 1; if (number < 0) @@ -1200,15 +1208,6 @@ numdigit (long number) return cnt; } -/* Attempt to calculate INT_MAX on machines that don't bother to - define it. */ -#ifndef INT_MAX -# ifndef CHAR_BIT -# define CHAR_BIT 8 -# endif -# define INT_MAX ((int) ~((unsigned)1 << CHAR_BIT * sizeof (int) - 1)) -#endif - #define ONE_DIGIT(figure) *p++ = n / (figure) + '0' #define ONE_DIGIT_ADVANCE(figure) (ONE_DIGIT (figure), n %= (figure)) @@ -1223,7 +1222,7 @@ numdigit (long number) #define DIGITS_9(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_8 ((figure) / 10) #define DIGITS_10(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_9 ((figure) / 10) -/* DIGITS_<11-20> are only used on machines with 64-bit longs. */ +/* DIGITS_<11-20> are only used on machines with 64-bit numbers. */ #define DIGITS_11(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_10 ((figure) / 10) #define DIGITS_12(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_11 ((figure) / 10) @@ -1235,13 +1234,73 @@ numdigit (long number) #define DIGITS_18(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_17 ((figure) / 10) #define DIGITS_19(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_18 ((figure) / 10) -/* Print NUMBER to BUFFER in base 10. This should be completely - equivalent to `sprintf(buffer, "%ld", number)', only much faster. +/* It is annoying that we have three different syntaxes for 64-bit constants: + - nnnL for 64-bit systems, where they are of type long; + - nnnLL for 32-bit systems that support long long; + - nnnI64 for MS compiler on Windows, which doesn't support long long. */ + +#if SIZEOF_LONG > 4 +/* If long is large enough, use long constants. */ +# define C10000000000 10000000000L +# define C100000000000 100000000000L +# define C1000000000000 1000000000000L +# define C10000000000000 10000000000000L +# define C100000000000000 100000000000000L +# define C1000000000000000 1000000000000000L +# define C10000000000000000 10000000000000000L +# define C100000000000000000 100000000000000000L +# define C1000000000000000000 1000000000000000000L +#else +# if SIZEOF_LONG_LONG != 0 +/* Otherwise, if long long is available, use long long constants. */ +# define C10000000000 10000000000LL +# define C100000000000 100000000000LL +# define C1000000000000 1000000000000LL +# define C10000000000000 10000000000000LL +# define C100000000000000 100000000000000LL +# define C1000000000000000 1000000000000000LL +# define C10000000000000000 10000000000000000LL +# define C100000000000000000 100000000000000000LL +# define C1000000000000000000 1000000000000000000LL +# else +# if defined(_MSC_VER) || defined(__WATCOM__) +/* Otherwise, if __int64 is available (under Windows), use __int64 + constants. */ +# define C10000000000 10000000000I64 +# define C100000000000 100000000000I64 +# define C1000000000000 1000000000000I64 +# define C10000000000000 10000000000000I64 +# define C100000000000000 100000000000000I64 +# define C1000000000000000 1000000000000000I64 +# define C10000000000000000 10000000000000000I64 +# define C100000000000000000 100000000000000000I64 +# define C1000000000000000000 1000000000000000000I64 +# endif +# endif +#endif + +/* SPRINTF_WGINT is used by number_to_string to handle pathological + cases and to portably support strange sizes of wgint. */ +#if SIZEOF_LONG >= SIZEOF_WGINT +# define SPRINTF_WGINT(buf, n) sprintf(buf, "%ld", (long) (n)) +#else +# if SIZEOF_LONG_LONG >= SIZEOF_WGINT +# define SPRINTF_WGINT(buf, n) sprintf(buf, "%lld", (long long) (n)) +# else +# ifdef _MSC_VER +# define SPRINTF_WGINT(buf, n) sprintf(buf, "%I64", (__int64) (n)) +# endif +# endif +#endif + +/* Print NUMBER to BUFFER in base 10. This is equivalent to + `sprintf(buffer, "%lld", (long long) number)', only much faster and + portable to machines without long long. The speedup may make a difference in programs that frequently convert numbers to strings. Some implementations of sprintf, particularly the one in GNU libc, have been known to be extremely - slow compared to this function. + slow when converting integers to strings. Return the pointer to the location where the terminating zero was printed. (Equivalent to calling buffer+strlen(buffer) after the @@ -1254,25 +1313,25 @@ numdigit (long number) terminating '\0'. */ char * -number_to_string (char *buffer, long number) +number_to_string (char *buffer, wgint number) { char *p = buffer; - long n = number; + wgint n = number; -#if (SIZEOF_LONG != 4) && (SIZEOF_LONG != 8) +#if (SIZEOF_WGINT != 4) && (SIZEOF_WGINT != 8) /* We are running in a strange or misconfigured environment. Let sprintf cope with it. */ - sprintf (buffer, "%ld", n); + SPRINTF_WGINT (buffer, n); p += strlen (buffer); -#else /* (SIZEOF_LONG == 4) || (SIZEOF_LONG == 8) */ +#else /* (SIZEOF_WGINT == 4) || (SIZEOF_WGINT == 8) */ if (n < 0) { - if (n < -INT_MAX) + if (n < -WGINT_MAX) { /* We cannot print a '-' and assign -n to n because -n would overflow. Let sprintf deal with this border case. */ - sprintf (buffer, "%ld", n); + SPRINTF_WGINT (buffer, n); p += strlen (buffer); return p; } @@ -1290,24 +1349,26 @@ number_to_string (char *buffer, long number) else if (n < 10000000) { DIGITS_7 (1000000); } else if (n < 100000000) { DIGITS_8 (10000000); } else if (n < 1000000000) { DIGITS_9 (100000000); } -#if SIZEOF_LONG == 4 +#if SIZEOF_WGINT == 4 + /* wgint is four bytes long: we're done. */ /* ``if (1)'' serves only to preserve editor indentation. */ else if (1) { DIGITS_10 (1000000000); } -#else /* SIZEOF_LONG != 4 */ - else if (n < 10000000000L) { DIGITS_10 (1000000000L); } - else if (n < 100000000000L) { DIGITS_11 (10000000000L); } - else if (n < 1000000000000L) { DIGITS_12 (100000000000L); } - else if (n < 10000000000000L) { DIGITS_13 (1000000000000L); } - else if (n < 100000000000000L) { DIGITS_14 (10000000000000L); } - else if (n < 1000000000000000L) { DIGITS_15 (100000000000000L); } - else if (n < 10000000000000000L) { DIGITS_16 (1000000000000000L); } - else if (n < 100000000000000000L) { DIGITS_17 (10000000000000000L); } - else if (n < 1000000000000000000L) { DIGITS_18 (100000000000000000L); } - else { DIGITS_19 (1000000000000000000L); } -#endif /* SIZEOF_LONG != 4 */ +#else + /* wgint is 64 bits long -- make sure to process all the digits. */ + else if (n < C10000000000) { DIGITS_10 (1000000000); } + else if (n < C100000000000) { DIGITS_11 (C10000000000); } + else if (n < C1000000000000) { DIGITS_12 (C100000000000); } + else if (n < C10000000000000) { DIGITS_13 (C1000000000000); } + else if (n < C100000000000000) { DIGITS_14 (C10000000000000); } + else if (n < C1000000000000000) { DIGITS_15 (C100000000000000); } + else if (n < C10000000000000000) { DIGITS_16 (C1000000000000000); } + else if (n < C100000000000000000) { DIGITS_17 (C10000000000000000); } + else if (n < C1000000000000000000) { DIGITS_18 (C100000000000000000); } + else { DIGITS_19 (C1000000000000000000); } +#endif *p = '\0'; -#endif /* (SIZEOF_LONG == 4) || (SIZEOF_LONG == 8) */ +#endif /* (SIZEOF_WGINT == 4) || (SIZEOF_WGINT == 8) */ return p; } @@ -1334,6 +1395,50 @@ number_to_string (char *buffer, long number) #undef DIGITS_17 #undef DIGITS_18 #undef DIGITS_19 + +#define RING_SIZE 3 + +/* Print NUMBER to a statically allocated string and return a pointer + to the printed representation. + + This function is intended to be used in conjunction with printf. + It is hard to portably print wgint values: + a) you cannot use printf("%ld", number) because wgint can be long + long on 32-bit machines with LFS. + b) you cannot use printf("%lld", number) because NUMBER could be + long on 32-bit machines without LFS, or on 64-bit machines, + which do not require LFS. Also, Windows doesn't support %lld. + c) you cannot use printf("%j", (int_max_t) number) because not all + versions of printf support "%j", the most notable being the one + on Windows. + d) you cannot #define WGINT_FMT to the appropriate format and use + printf(WGINT_FMT, number) because that would break translations + for user-visible messages, such as printf("Downloaded: %d + bytes\n", number). + + What you should use instead is printf("%s", number_to_static_string + (number)). + + CAVEAT: since the function returns pointers to static data, you + must be careful to copy its result before calling it again. + However, to make it more useful with printf, the function maintains + an internal ring of static buffers to return. That way things like + printf("%s %s", number_to_static_string (num1), + number_to_static_string (num2)) work as expected. Three buffers + are currently used, which means that "%s %s %s" will work, but "%s + %s %s %s" won't. If you need to print more than three wgints, + bump the RING_SIZE (or rethink your message.) */ + +char * +number_to_static_string (wgint number) +{ + static char ring[RING_SIZE][24]; + static int ringpos; + char *buf = ring[ringpos]; + number_to_string (buf, number); + ringpos = (ringpos + 1) % RING_SIZE; + return buf; +} /* Support for timers. */ diff --git a/src/utils.h b/src/utils.h index ccc83279..08aa5667 100644 --- a/src/utils.h +++ b/src/utils.h @@ -45,7 +45,7 @@ struct hash_table; struct file_memory { char *content; - long length; + wgint length; int mmap_p; }; @@ -79,7 +79,7 @@ void touch PARAMS ((const char *, time_t)); int remove_link PARAMS ((const char *)); int file_exists_p PARAMS ((const char *)); int file_non_directory_p PARAMS ((const char *)); -long file_size PARAMS ((const char *)); +wgint file_size PARAMS ((const char *)); int make_directory PARAMS ((const char *)); char *unique_name PARAMS ((const char *, int)); char *file_merge PARAMS ((const char *, const char *)); @@ -109,10 +109,11 @@ int string_set_contains PARAMS ((struct hash_table *, const char *)); void string_set_free PARAMS ((struct hash_table *)); void free_keys_and_values PARAMS ((struct hash_table *)); -char *legible PARAMS ((long)); +char *legible PARAMS ((wgint)); char *legible_large_int PARAMS ((LARGE_INT)); -int numdigit PARAMS ((long)); -char *number_to_string PARAMS ((char *, long)); +int numdigit PARAMS ((wgint)); +char *number_to_string PARAMS ((char *, wgint)); +char *number_to_static_string PARAMS ((wgint)); struct wget_timer *wtimer_allocate PARAMS ((void)); struct wget_timer *wtimer_new PARAMS ((void)); diff --git a/src/wget.h b/src/wget.h index 241537a9..c41a6814 100644 --- a/src/wget.h +++ b/src/wget.h @@ -107,6 +107,24 @@ so, delete this exception statement from your version. */ # define GCC_FORMAT_ATTR(a, b) #endif /* not __GNUC__ */ +/* Define an integer type that works for LFS. off_t would be perfect + for this, but off_t is always 32-bit under Windows. */ +#ifndef WINDOWS +typedef off_t wgint; +# define SIZEOF_WGINT SIZEOF_OFF_T +#endif + +/* Define a strtol/strtoll clone that works with wgint. */ +#ifndef str_to_wgint /* mswindows.h defines its own alias */ +# if SIZEOF_WGINT == SIZEOF_LONG +# define str_to_wgint strtol +# define WGINT_MAX LONG_MAX +# else +# define str_to_wgint strtoll +# define WGINT_MAX LLONG_MAX +# endif +#endif + /* Everything uses this, so include them here directly. */ #include "xmalloc.h"