moparisthebest
/
wget
mirror of https://github.com/moparisthebest/wget
1
0
Fork 0
Code Issues Releases Wiki Activity

[svn] Remove VERY_LONG_TYPE; use LARGE_INT instead. Remove special code 

for handling VERY_LONG_TYPE overflows.
Make opt.quota a LARGE_INT.
This commit is contained in:
hniksic 2003-10-11 06:57:11 -07:00
parent a6f72ca66b
commit 711bf72609
12 changed files with 185 additions and 175 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
src/ChangeLog
View File

@ -1,3 +1,29 @@
2003-10-11 Hrvoje Niksic <hniksic@xemacs.org>
* utils.c (large_int_to_string): Use snprintf() to print the
number. This will work even on systems where libc doesn't
understand %lld, but the compiler does, because it will use our
snprintf replacement.
* init.c (parse_bytes_helper): New function.
(cmd_bytes): Use it to parse bytes, but cast the result to long.
(cmd_bytes_large): Ditto, but store the result to LARGE_INT. Used
for --quota so that --quota=10G works even on machines without
long long.
* options.h (struct options): Declare quota as LARGE_INT.
* retr.c (downloaded_exceeds_quota): Removed.
(downloaded_increase): Ditto.
(total_downloaded_bytes): New variable, replaces opt.downloaded,
which was the wrong place for it anyway. Updated callers of
downloaded_exceeds_quota and downloaded_increase to check this
variable directly.
* sysdep.h: Get rid of VERY_LONG_TYPE. Use LARGE_INT for the same
purpose, defined as `long', `long long' or `double', depending on
size of long and whether long long is available.
2003-10-11 Hrvoje Niksic <hniksic@xemacs.org>
* sysdep.h: Also check size of short for int32_t.

14
src/ftp.c
View File

@ -59,6 +59,8 @@ so, delete this exception statement from your version. */
extern int errno;
#endif
extern LARGE_INT total_downloaded_bytes;
/* File where the "ls -al" listing will be saved. */
#define LIST_FILENAME ".listing"
@ -1195,7 +1197,7 @@ ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con)
/* --dont-remove-listing was specified, so do count this towards the
number of bytes and files downloaded. */
{
downloaded_increase (len);
total_downloaded_bytes += len;
opt.numurls++;
}
@ -1210,7 +1212,7 @@ ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con)
downloaded if they're going to be deleted. People seeding proxies,
for instance, may want to know how many bytes and files they've
downloaded through it. */
downloaded_increase (len);
total_downloaded_bytes += len;
opt.numurls++;
if (opt.delete_after)
@ -1336,7 +1338,7 @@ ftp_retrieve_list (struct url *u, struct fileinfo *f, ccon *con)
{
char *old_target, *ofile;
if (downloaded_exceeds_quota ())
if (opt.quota && total_downloaded_bytes > opt.quota)
{
--depth;
return QUOTEXC;
@ -1540,7 +1542,7 @@ ftp_retrieve_dirs (struct url *u, struct fileinfo *f, ccon *con)
int size;
char *odir, *newdir;
if (downloaded_exceeds_quota ())
if (opt.quota && total_downloaded_bytes > opt.quota)
break;
if (f->type != FT_DIRECTORY)
continue;
@ -1586,7 +1588,7 @@ Not descending to `%s' as it is excluded/not-included.\n"), newdir);
/* Set the time-stamp? */
}
if (opt.quota && opt.downloaded > opt.quota)
if (opt.quota && total_downloaded_bytes > opt.quota)
return QUOTEXC;
else
return RETROK;
@ -1704,7 +1706,7 @@ ftp_retrieve_glob (struct url *u, ccon *con, int action)
}
}
freefileinfo (start);
if (downloaded_exceeds_quota ())
if (opt.quota && total_downloaded_bytes > opt.quota)
return QUOTEXC;
else
/* #### Should we return `res' here? */

13
src/http.c
View File

@ -53,6 +53,9 @@ so, delete this exception statement from your version. */
# include <time.h>
# endif
#endif
#ifndef errno
extern int errno;
#endif
#include "wget.h"
#include "utils.h"
@ -73,10 +76,8 @@ so, delete this exception statement from your version. */
#include "convert.h"
extern char *version_string;
extern LARGE_INT total_downloaded_bytes;
#ifndef errno
extern int errno;
#endif
static int cookies_loaded_p;
struct cookie_jar *wget_cookie_jar;
@ -1951,7 +1952,7 @@ The sizes do not match (local %ld) -- retrieving.\n"), local_size);
tms, u->url, hstat.len, hstat.contlen, locf, count);
}
++opt.numurls;
downloaded_increase (hstat.len);
total_downloaded_bytes += hstat.len;
/* Remember that we downloaded the file for later ".orig" code. */
if (*dt & ADDED_HTML_EXTENSION)
@ -1978,7 +1979,7 @@ The sizes do not match (local %ld) -- retrieving.\n"), local_size);
tms, u->url, hstat.len, locf, count);
}
++opt.numurls;
downloaded_increase (hstat.len);
total_downloaded_bytes += hstat.len;
/* Remember that we downloaded the file for later ".orig" code. */
if (*dt & ADDED_HTML_EXTENSION)
@ -2009,7 +2010,7 @@ The sizes do not match (local %ld) -- retrieving.\n"), local_size);
"%s URL:%s [%ld/%ld] -> \"%s\" [%d]\n",
tms, u->url, hstat.len, hstat.contlen, locf, count);
++opt.numurls;
downloaded_increase (hstat.len);
total_downloaded_bytes += hstat.len;
/* Remember that we downloaded the file for later ".orig" code. */
if (*dt & ADDED_HTML_EXTENSION)

123
src/init.c
View File

@ -84,6 +84,7 @@ static int enable_tilde_expansion;
CMD_DECLARE (cmd_boolean);
CMD_DECLARE (cmd_bytes);
CMD_DECLARE (cmd_bytes_large);
CMD_DECLARE (cmd_directory_vector);
CMD_DECLARE (cmd_lockable_boolean);
CMD_DECLARE (cmd_number);
@ -184,7 +185,7 @@ static struct {
{ "proxypasswd", &opt.proxy_passwd, cmd_string },
{ "proxyuser", &opt.proxy_user, cmd_string },
{ "quiet", &opt.quiet, cmd_boolean },
{ "quota", &opt.quota, cmd_bytes },
{ "quota", &opt.quota, cmd_bytes_large },
{ "randomwait", &opt.random_wait, cmd_boolean },
{ "readtimeout", &opt.read_timeout, cmd_time },
{ "reclevel", &opt.reclevel, cmd_number_inf },
@ -849,6 +850,63 @@ cmd_directory_vector (const char *com, const char *val, void *closure)
static int simple_atof PARAMS ((const char *, const char *, double *));
/* Enginge for cmd_bytes and cmd_bytes_large: converts a string such
as "100k" or "2.5G" to a floating point number. */
static int
parse_bytes_helper (const char *val, double *result)
{
double number, mult;
const char *end = val + strlen (val);
/* Check for "inf". */
if (0 == strcmp (val, "inf"))
{
*result = 0;
return 1;
}
/* Strip trailing whitespace. */
while (val < end && ISSPACE (end[-1]))
--end;
if (val == end)
return 0;
switch (TOLOWER (end[-1]))
{
case 'k':
--end, mult = 1024.0;
break;
case 'm':
--end, mult = 1048576.0;
break;
case 'g':
--end, mult = 1073741824.0;
break;
case 't':
--end, mult = 1099511627776.0;
break;
default:
/* Not a recognized suffix: assume it's a digit. (If not,
simple_atof will raise an error.) */
mult = 1;
}
/* Skip leading and trailing whitespace. */
while (val < end && ISSPACE (*val))
++val;
while (val < end && ISSPACE (end[-1]))
--end;
if (val == end)
return 0;
if (!simple_atof (val, end, &number))
return 0;
*result = number * mult;
return 1;
}
/* Parse VAL as a number and set its value to CLOSURE (which should
point to a long int).
@ -866,58 +924,33 @@ static int simple_atof PARAMS ((const char *, const char *, double *));
static int
cmd_bytes (const char *com, const char *val, void *closure)
{
long mult;
double number;
const char *end = val + strlen (val);
/* Check for "inf". */
if (0 == strcmp (val, "inf"))
double byte_value;
if (!parse_bytes_helper (val, &byte_value))
{
*(long *)closure = 0;
return 1;
}
/* Strip trailing whitespace. */
while (val < end && ISSPACE (end[-1]))
--end;
if (val == end)
{
err:
fprintf (stderr, _("%s: %s: Invalid byte value `%s'\n"),
exec_name, com, val);
return 0;
}
*(long *)closure = (long)byte_value;
return 1;
}
switch (TOLOWER (end[-1]))
/* Like cmd_bytes, but CLOSURE is interpreted as a pointer to
LARGE_INT. It works by converting the string to double, therefore
working with values up to 2^53-1 without loss of precision. This
value (8192 TB) is large enough to serve for a while. */
static int
cmd_bytes_large (const char *com, const char *val, void *closure)
{
double byte_value;
if (!parse_bytes_helper (val, &byte_value))
{
case 'k':
--end, mult = 1L<<10;
break;
case 'm':
--end, mult = 1L<<20;
break;
case 'g':
--end, mult = 1L<<30;
break;
default:
/* Not a recognized suffix: assume it belongs to the number.
(If not, atof simple_atof will raise an error.) */
mult = 1;
fprintf (stderr, _("%s: %s: Invalid byte value `%s'\n"),
exec_name, com, val);
return 0;
}
/* Skip leading and trailing whitespace. */
while (val < end && ISSPACE (*val))
++val;
while (val < end && ISSPACE (end[-1]))
--end;
if (val == end)
goto err;
if (!simple_atof (val, end, &number))
goto err;
*(long *)closure = (long)(number * mult);
*(LARGE_INT *)closure = (LARGE_INT)byte_value;
return 1;
}

20
src/main.c
View File

@ -50,6 +50,9 @@ so, delete this exception statement from your version. */
#endif /* HAVE_LOCALE_H */
#endif /* HAVE_NLS */
#include <errno.h>
#ifndef errno
extern int errno;
#endif
#include "wget.h"
#include "utils.h"
@ -73,14 +76,11 @@ so, delete this exception statement from your version. */
# define PATH_SEPARATOR '/'
#endif
extern char *version_string;
#ifndef errno
extern int errno;
#endif
struct options opt;
extern LARGE_INT total_downloaded_bytes;
extern char *version_string;
extern struct cookie_jar *wget_cookie_jar;
/* From log.c. */
@ -908,16 +908,14 @@ Can't timestamp and not clobber old files at the same time.\n"));
/* Print the downloaded sum. */
if (opt.recursive
|| nurl > 1
|| (opt.input_filename && opt.downloaded != 0))
|| (opt.input_filename && total_downloaded_bytes != 0))
{
logprintf (LOG_NOTQUIET,
_("\nFINISHED --%s--\nDownloaded: %s bytes in %d files\n"),
time_str (NULL),
(opt.downloaded_overflow ?
"<overflow>" : legible_very_long (opt.downloaded)),
time_str (NULL), legible_large_int (total_downloaded_bytes),
opt.numurls);
/* Print quota warning, if exceeded. */
if (downloaded_exceeds_quota ())
if (opt.quota && total_downloaded_bytes > opt.quota)
logprintf (LOG_NOTQUIET,
_("Download quota (%s bytes) EXCEEDED!\n"),
legible (opt.quota));

6
src/options.h
View File

@ -119,10 +119,8 @@ struct options
long limit_rate; /* Limit the download rate to this
many bps. */
long quota; /* Maximum number of bytes to
retrieve. */
VERY_LONG_TYPE downloaded; /* How much we downloaded already. */
int downloaded_overflow; /* Whether the above overflowed. */
LARGE_INT quota; /* Maximum file size to download and
store. */
int numurls; /* Number of successfully downloaded
URLs */

5
src/recur.c
View File

@ -59,6 +59,7 @@ extern int errno;
#endif
extern char *version_string;
extern LARGE_INT total_downloaded_bytes;
extern struct hash_table *dl_url_file_map;
extern struct hash_table *downloaded_html_set;
@ -224,7 +225,7 @@ retrieve_tree (const char *start_url)
int depth, html_allowed;
boolean dash_p_leaf_HTML = FALSE;
if (downloaded_exceeds_quota ())
if (opt.quota && total_downloaded_bytes > opt.quota)
break;
if (status == FWRITEERR)
break;
@ -404,7 +405,7 @@ retrieve_tree (const char *start_url)
url_free (start_url_parsed);
string_set_free (blacklist);
if (downloaded_exceeds_quota ())
if (opt.quota && total_downloaded_bytes > opt.quota)
return QUOTEXC;
else if (status == FWRITEERR)
return FWRITEERR;

39
src/retr.c
View File

@ -66,6 +66,9 @@ extern int errno;
/* See the comment in gethttp() why this is needed. */
int global_download_count;
/* Total size of downloaded files. Used to enforce quota. */
LARGE_INT total_downloaded_bytes;
static struct {
long chunk_bytes;
@ -576,7 +579,7 @@ retrieve_from_file (const char *file, int html, int *count)
if (cur_url->ignore_when_downloading)
continue;
if (downloaded_exceeds_quota ())
if (opt.quota && total_downloaded_bytes > opt.quota)
{
status = QUOTEXC;
break;
@ -614,40 +617,6 @@ printwhat (int n1, int n2)
logputs (LOG_VERBOSE, (n1 == n2) ? _("Giving up.\n\n") : _("Retrying.\n\n"));
}
/* Increment opt.downloaded by BY_HOW_MUCH. If an overflow occurs,
set opt.downloaded_overflow to 1. */
void
downloaded_increase (unsigned long by_how_much)
{
VERY_LONG_TYPE old;
if (opt.downloaded_overflow)
return;
old = opt.downloaded;
opt.downloaded += by_how_much;
if (opt.downloaded < old) /* carry flag, where are you when I
need you? */
{
/* Overflow. */
opt.downloaded_overflow = 1;
opt.downloaded = ~((VERY_LONG_TYPE)0);
}
}
/* Return non-zero if the downloaded amount of bytes exceeds the
desired quota. If quota is not set or if the amount overflowed, 0
is returned. */
int
downloaded_exceeds_quota (void)
{
if (!opt.quota)
return 0;
if (opt.downloaded_overflow)
/* We don't really know. (Wildly) assume not. */
return 0;
return opt.downloaded > opt.quota;
}
/* If opt.wait or opt.waitretry are specified, and if certain
conditions are met, sleep the appropriate number of seconds. See
the documentation of --wait and --waitretry for more information.

3
src/retr.h
View File

@ -43,9 +43,6 @@ char *retr_rate PARAMS ((long, double, int));
double calc_rate PARAMS ((long, double, int *));
void printwhat PARAMS ((int, int));
void downloaded_increase PARAMS ((unsigned long));
int downloaded_exceeds_quota PARAMS ((void));
void sleep_between_retrievals PARAMS ((int));
void rotate_backups PARAMS ((const char *));

44
src/sysdep.h
View File

@ -128,27 +128,30 @@ do { \
DEBUGP (("Closing fd %d\n", x)); \
} while (0)
/* Define a large ("very long") type useful for storing large
non-negative quantities that exceed sizes of normal download. Note
that this has nothing to do with large file support. For example,
one should be able to say `--quota=10G', large files
notwithstanding.
/* Define a large integral type useful for storing large sizes that
exceed sizes of one download, such as when printing the sum of all
downloads. Note that this has nothing to do with large file
support, yet.
On the machines where `long' is 64-bit, we use long. Otherwise, we
check whether `long long' is available and if yes, use that. If
long long is unavailable, we give up and just use `long'.
We use a 64-bit integral type where available, `double' otherwise.
It's hard to print LARGE_INT's portably, but fortunately it's
rarely needed. */
Note: you cannot use VERY_LONG_TYPE along with printf(). When you
need to print it, use very_long_to_string(). */
#if SIZEOF_LONG >= 8 || SIZEOF_LONG_LONG == 0
/* either long is "big enough", or long long is unavailable which
leaves long as the only choice. */
# define VERY_LONG_TYPE unsigned long
#else /* use long long */
/* long is smaller than 8 bytes, but long long is available. */
# define VERY_LONG_TYPE unsigned long long
#endif /* use long long */
#if SIZEOF_LONG >= 8
/* Long is large enough: use it. */
typedef long LARGE_INT;
# define LARGE_INT_FMT "%ld"
#else
# if SIZEOF_LONG_LONG == 8
/* Long long is large enough: use it. */
typedef long long LARGE_INT;
# define LARGE_INT_FMT "%lld"
# else
/* Use `double'. */
typedef double LARGE_INT;
# define LARGE_INT_FMT "%.0f"
# endif
#endif
/* These are defined in cmpt.c if missing, therefore it's generally
safe to declare their parameters. */
@ -167,6 +170,9 @@ char *strstr ();
#ifndef HAVE_STRPTIME
char *strptime ();
#endif
#ifndef HAVE_SNPRINTF
int snprintf ();
#endif
#ifndef HAVE_VSNPRINTF
int vsnprintf ();
#endif

65
src/utils.c
View File

@ -1290,13 +1290,13 @@ free_keys_and_values (struct hash_table *ht)
}
/* Engine for legible and legible_very_long; this function works on
strings. */
/* Engine for legible and legible_large_int; add thousand separators
to numbers printed in strings. */
static char *
legible_1 (const char *repr)
{
static char outbuf[128];
static char outbuf[48];
int i, i1, mod;
char *outptr;
const char *inptr;
@ -1304,7 +1304,9 @@ legible_1 (const char *repr)
/* Reset the pointers. */
outptr = outbuf;
inptr = repr;
/* If the number is negative, shift the pointers. */
/* Ignore the sign for the purpose of adding thousand
separators. */
if (*inptr == '-')
{
*outptr++ = '-';
@ -1329,6 +1331,7 @@ legible_1 (const char *repr)
}
/* Legible -- return a static pointer to the legibly printed long. */
char *
legible (long l)
{
@ -1338,53 +1341,29 @@ legible (long l)
return legible_1 (inbuf);
}
/* Write a string representation of NUMBER into the provided buffer.
We cannot use sprintf() because we cannot be sure whether the
platform supports printing of what we chose for VERY_LONG_TYPE.
/* Write a string representation of LARGE_INT NUMBER into the provided
buffer. The buffer should be able to accept 24 characters,
including the terminating zero.
Example: Gcc supports `long long' under many platforms, but on many
of those the native libc knows nothing of it and therefore cannot
print it.
How long BUFFER needs to be depends on the platform and the content
of NUMBER. For 64-bit VERY_LONG_TYPE (the most common case), 24
bytes are sufficient. Using more might be a good idea.
This function does not go through the hoops that long_to_string
goes to because it doesn't aspire to be fast. (It's called perhaps
once in a Wget run.) */
It would be dangerous to use sprintf, because the code wouldn't
work on a machine with gcc-provided long long support, but without
libc support for "%lld". However, such platforms will typically
not have snprintf and will use our version, which does support
"%lld" where long longs are available. */
static void
very_long_to_string (char *buffer, VERY_LONG_TYPE number)
large_int_to_string (char *buffer, LARGE_INT number)
{
int i = 0;
int j;
/* Print the number backwards... */
do
{
buffer[i++] = '0' + number % 10;
number /= 10;
}
while (number);
/* ...and reverse the order of the digits. */
for (j = 0; j < i / 2; j++)
{
char c = buffer[j];
buffer[j] = buffer[i - 1 - j];
buffer[i - 1 - j] = c;
}
buffer[i] = '\0';
snprintf (buffer, 24, LARGE_INT_FMT, number);
}
/* The same as legible(), but works on VERY_LONG_TYPE. See sysdep.h. */
/* The same as legible(), but works on LARGE_INT. */
char *
legible_very_long (VERY_LONG_TYPE l)
legible_large_int (LARGE_INT l)
{
char inbuf[128];
/* Print the number into the buffer. */
very_long_to_string (inbuf, l);
char inbuf[48];
large_int_to_string (inbuf, l);
return legible_1 (inbuf);
}

2
src/utils.h
View File

@ -102,7 +102,7 @@ void string_set_free PARAMS ((struct hash_table *));
void free_keys_and_values PARAMS ((struct hash_table *));
char *legible PARAMS ((long));
char *legible_very_long PARAMS ((VERY_LONG_TYPE));
char *legible_large_int PARAMS ((LARGE_INT));
int numdigit PARAMS ((long));
char *number_to_string PARAMS ((char *, long));