[svn] Remove VERY_LONG_TYPE; use LARGE_INT instead. Remove special code

for handling VERY_LONG_TYPE overflows.
Make opt.quota a LARGE_INT.

src/ChangeLog

@@ -1,3 +1,29 @@
+2003-10-11  Hrvoje Niksic  <hniksic@xemacs.org>
+
+	* utils.c (large_int_to_string): Use snprintf() to print the
+	number.  This will work even on systems where libc doesn't
+	understand %lld, but the compiler does, because it will use our
+	snprintf replacement.
+
+	* init.c (parse_bytes_helper): New function.
+	(cmd_bytes): Use it to parse bytes, but cast the result to long.
+	(cmd_bytes_large): Ditto, but store the result to LARGE_INT.  Used
+	for --quota so that --quota=10G works even on machines without
+	long long.
+
+	* options.h (struct options): Declare quota as LARGE_INT.
+
+	* retr.c (downloaded_exceeds_quota): Removed.
+	(downloaded_increase): Ditto.
+	(total_downloaded_bytes): New variable, replaces opt.downloaded,
+	which was the wrong place for it anyway.  Updated callers of
+	downloaded_exceeds_quota and downloaded_increase to check this
+	variable directly.
+
+	* sysdep.h: Get rid of VERY_LONG_TYPE.  Use LARGE_INT for the same
+	purpose, defined as `long', `long long' or `double', depending on
+	size of long and whether long long is available.
+
 2003-10-11  Hrvoje Niksic  <hniksic@xemacs.org>
 
 	* sysdep.h: Also check size of short for int32_t.

src/ftp.c

@@ -59,6 +59,8 @@ so, delete this exception statement from your version.  */
 extern int errno;
 #endif
 
+extern LARGE_INT total_downloaded_bytes;
+
 /* File where the "ls -al" listing will be saved.  */
 #define LIST_FILENAME ".listing"
 
@@ -1195,7 +1197,7 @@ ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con)
 	    /* --dont-remove-listing was specified, so do count this towards the
 	       number of bytes and files downloaded. */
 	    {
-	      downloaded_increase (len);
+	      total_downloaded_bytes += len;
 	      opt.numurls++;
 	    }
 
@@ -1210,7 +1212,7 @@ ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con)
 	     downloaded if they're going to be deleted.  People seeding proxies,
 	     for instance, may want to know how many bytes and files they've
 	     downloaded through it. */
-	  downloaded_increase (len);
+	  total_downloaded_bytes += len;
 	  opt.numurls++;
 
 	  if (opt.delete_after)
@@ -1336,7 +1338,7 @@ ftp_retrieve_list (struct url *u, struct fileinfo *f, ccon *con)
     {
       char *old_target, *ofile;
 
-      if (downloaded_exceeds_quota ())
+      if (opt.quota && total_downloaded_bytes > opt.quota)
 	{
 	  --depth;
 	  return QUOTEXC;
@@ -1540,7 +1542,7 @@ ftp_retrieve_dirs (struct url *u, struct fileinfo *f, ccon *con)
       int size;
       char *odir, *newdir;
 
-      if (downloaded_exceeds_quota ())
+      if (opt.quota && total_downloaded_bytes > opt.quota)
 	break;
       if (f->type != FT_DIRECTORY)
 	continue;
@@ -1586,7 +1588,7 @@ Not descending to `%s' as it is excluded/not-included.\n"), newdir);
       /* Set the time-stamp?  */
     }
 
-  if (opt.quota && opt.downloaded > opt.quota)
+  if (opt.quota && total_downloaded_bytes > opt.quota)
     return QUOTEXC;
   else
     return RETROK;
@@ -1704,7 +1706,7 @@ ftp_retrieve_glob (struct url *u, ccon *con, int action)
 	}
     }
   freefileinfo (start);
-  if (downloaded_exceeds_quota ())
+  if (opt.quota && total_downloaded_bytes > opt.quota)
     return QUOTEXC;
   else
     /* #### Should we return `res' here?  */

src/http.c

@@ -53,6 +53,9 @@ so, delete this exception statement from your version.  */
 #  include <time.h>
 # endif
 #endif
+#ifndef errno
+extern int errno;
+#endif
 
 #include "wget.h"
 #include "utils.h"
@@ -73,10 +76,8 @@ so, delete this exception statement from your version.  */
 #include "convert.h"
 
 extern char *version_string;
+extern LARGE_INT total_downloaded_bytes;
 
-#ifndef errno
-extern int errno;
-#endif
 
 static int cookies_loaded_p;
 struct cookie_jar *wget_cookie_jar;
@@ -1951,7 +1952,7 @@ The sizes do not match (local %ld) -- retrieving.\n"), local_size);
 			 tms, u->url, hstat.len, hstat.contlen, locf, count);
 	    }
 	  ++opt.numurls;
-	  downloaded_increase (hstat.len);
+	  total_downloaded_bytes += hstat.len;
 
 	  /* Remember that we downloaded the file for later ".orig" code. */
 	  if (*dt & ADDED_HTML_EXTENSION)
@@ -1978,7 +1979,7 @@ The sizes do not match (local %ld) -- retrieving.\n"), local_size);
 			     tms, u->url, hstat.len, locf, count);
 		}
 	      ++opt.numurls;
-	      downloaded_increase (hstat.len);
+	      total_downloaded_bytes += hstat.len;
 
 	      /* Remember that we downloaded the file for later ".orig" code. */
 	      if (*dt & ADDED_HTML_EXTENSION)
@@ -2009,7 +2010,7 @@ The sizes do not match (local %ld) -- retrieving.\n"), local_size);
 			 "%s URL:%s [%ld/%ld] -> \"%s\" [%d]\n",
 			 tms, u->url, hstat.len, hstat.contlen, locf, count);
 	      ++opt.numurls;
-	      downloaded_increase (hstat.len);
+	      total_downloaded_bytes += hstat.len;
 
 	      /* Remember that we downloaded the file for later ".orig" code. */
 	      if (*dt & ADDED_HTML_EXTENSION)

src/init.c

@@ -84,6 +84,7 @@ static int enable_tilde_expansion;
 
 CMD_DECLARE (cmd_boolean);
 CMD_DECLARE (cmd_bytes);
+CMD_DECLARE (cmd_bytes_large);
 CMD_DECLARE (cmd_directory_vector);
 CMD_DECLARE (cmd_lockable_boolean);
 CMD_DECLARE (cmd_number);
@@ -184,7 +185,7 @@ static struct {
   { "proxypasswd",	&opt.proxy_passwd,	cmd_string },
   { "proxyuser",	&opt.proxy_user,	cmd_string },
   { "quiet",		&opt.quiet,		cmd_boolean },
-  { "quota",		&opt.quota,		cmd_bytes },
+  { "quota",		&opt.quota,		cmd_bytes_large },
   { "randomwait",	&opt.random_wait,	cmd_boolean },
   { "readtimeout",	&opt.read_timeout,	cmd_time },
   { "reclevel",		&opt.reclevel,		cmd_number_inf },
@@ -849,6 +850,63 @@ cmd_directory_vector (const char *com, const char *val, void *closure)
 
 static int simple_atof PARAMS ((const char *, const char *, double *));
 
+/* Enginge for cmd_bytes and cmd_bytes_large: converts a string such
+   as "100k" or "2.5G" to a floating point number.  */
+
+static int
+parse_bytes_helper (const char *val, double *result)
+{
+  double number, mult;
+  const char *end = val + strlen (val);
+
+  /* Check for "inf".  */
+  if (0 == strcmp (val, "inf"))
+    {
+      *result = 0;
+      return 1;
+    }
+
+  /* Strip trailing whitespace.  */
+  while (val < end && ISSPACE (end[-1]))
+    --end;
+  if (val == end)
+    return 0;
+
+  switch (TOLOWER (end[-1]))
+    {
+    case 'k':
+      --end, mult = 1024.0;
+      break;
+    case 'm':
+      --end, mult = 1048576.0;
+      break;
+    case 'g':
+      --end, mult = 1073741824.0;
+      break;
+    case 't':
+      --end, mult = 1099511627776.0;
+      break;
+    default:
+      /* Not a recognized suffix: assume it's a digit.  (If not,
+	 simple_atof will raise an error.)  */
+      mult = 1;
+    }
+
+  /* Skip leading and trailing whitespace. */
+  while (val < end && ISSPACE (*val))
+    ++val;
+  while (val < end && ISSPACE (end[-1]))
+    --end;
+  if (val == end)
+    return 0;
+
+  if (!simple_atof (val, end, &number))
+    return 0;
+
+  *result = number * mult;
+  return 1;
+}
+
 /* Parse VAL as a number and set its value to CLOSURE (which should
    point to a long int).
 
@@ -866,58 +924,33 @@ static int simple_atof PARAMS ((const char *, const char *, double *));
 static int
 cmd_bytes (const char *com, const char *val, void *closure)
 {
-  long mult;
+  double byte_value;
-  double number;
+  if (!parse_bytes_helper (val, &byte_value))
-  const char *end = val + strlen (val);
-
-  /* Check for "inf".  */
-  if (0 == strcmp (val, "inf"))
     {
-      *(long *)closure = 0;
-      return 1;
-    }
-
-  /* Strip trailing whitespace.  */
-  while (val < end && ISSPACE (end[-1]))
-    --end;
-
-  if (val == end)
-    {
-    err:
       fprintf (stderr, _("%s: %s: Invalid byte value `%s'\n"),
 	       exec_name, com, val);
       return 0;
     }
-
+  *(long *)closure = (long)byte_value;
-  switch (TOLOWER (end[-1]))
+  return 1;
-    {
-    case 'k':
-      --end, mult = 1L<<10;
-      break;
-    case 'm':
-      --end, mult = 1L<<20;
-      break;
-    case 'g':
-      --end, mult = 1L<<30;
-      break;
-    default:
-      /* Not a recognized suffix: assume it belongs to the number.
-	 (If not, atof simple_atof will raise an error.)  */
-      mult = 1;
 }
 
-  /* Skip leading and trailing whitespace. */
+/* Like cmd_bytes, but CLOSURE is interpreted as a pointer to
-  while (val < end && ISSPACE (*val))
+   LARGE_INT.  It works by converting the string to double, therefore
-    ++val;
+   working with values up to 2^53-1 without loss of precision.  This
-  while (val < end && ISSPACE (end[-1]))
+   value (8192 TB) is large enough to serve for a while.  */
-    --end;
-  if (val == end)
-    goto err;
 
-  if (!simple_atof (val, end, &number))
+static int
-    goto err;
+cmd_bytes_large (const char *com, const char *val, void *closure)
-
+{
-  *(long *)closure = (long)(number * mult);
+  double byte_value;
+  if (!parse_bytes_helper (val, &byte_value))
+    {
+      fprintf (stderr, _("%s: %s: Invalid byte value `%s'\n"),
+	       exec_name, com, val);
+      return 0;
+    }
+  *(LARGE_INT *)closure = (LARGE_INT)byte_value;
   return 1;
 }
 

src/main.c

@@ -50,6 +50,9 @@ so, delete this exception statement from your version.  */
 #endif /* HAVE_LOCALE_H */
 #endif /* HAVE_NLS */
 #include <errno.h>
+#ifndef errno
+extern int errno;
+#endif
 
 #include "wget.h"
 #include "utils.h"
@@ -73,14 +76,11 @@ so, delete this exception statement from your version.  */
 # define PATH_SEPARATOR '/'
 #endif
 
-extern char *version_string;
-
-#ifndef errno
-extern int errno;
-#endif
-
 struct options opt;
 
+extern LARGE_INT total_downloaded_bytes;
+extern char *version_string;
+
 extern struct cookie_jar *wget_cookie_jar;
 
 /* From log.c.  */
@@ -908,16 +908,14 @@ Can't timestamp and not clobber old files at the same time.\n"));
   /* Print the downloaded sum.  */
   if (opt.recursive
       || nurl > 1
-      || (opt.input_filename && opt.downloaded != 0))
+      || (opt.input_filename && total_downloaded_bytes != 0))
     {
       logprintf (LOG_NOTQUIET,
 		 _("\nFINISHED --%s--\nDownloaded: %s bytes in %d files\n"),
-		 time_str (NULL),
+		 time_str (NULL), legible_large_int (total_downloaded_bytes),
-		 (opt.downloaded_overflow ?
-		  "<overflow>" : legible_very_long (opt.downloaded)),
 		 opt.numurls);
       /* Print quota warning, if exceeded.  */
-      if (downloaded_exceeds_quota ())
+      if (opt.quota && total_downloaded_bytes > opt.quota)
 	logprintf (LOG_NOTQUIET,
 		   _("Download quota (%s bytes) EXCEEDED!\n"),
 		   legible (opt.quota));

src/options.h

@@ -119,10 +119,8 @@ struct options
 
   long limit_rate;		/* Limit the download rate to this
 				   many bps. */
-  long quota;			/* Maximum number of bytes to
+  LARGE_INT quota;		/* Maximum file size to download and
-				   retrieve. */
+				   store. */
-  VERY_LONG_TYPE downloaded;	/* How much we downloaded already. */
-  int downloaded_overflow;	/* Whether the above overflowed. */
   int numurls;			/* Number of successfully downloaded
 				   URLs */
 

src/recur.c

@@ -59,6 +59,7 @@ extern int errno;
 #endif
 
 extern char *version_string;
+extern LARGE_INT total_downloaded_bytes;
 
 extern struct hash_table *dl_url_file_map;
 extern struct hash_table *downloaded_html_set;
@@ -224,7 +225,7 @@ retrieve_tree (const char *start_url)
       int depth, html_allowed;
       boolean dash_p_leaf_HTML = FALSE;
 
-      if (downloaded_exceeds_quota ())
+      if (opt.quota && total_downloaded_bytes > opt.quota)
 	break;
       if (status == FWRITEERR)
 	break;
@@ -404,7 +405,7 @@ retrieve_tree (const char *start_url)
     url_free (start_url_parsed);
   string_set_free (blacklist);
 
-  if (downloaded_exceeds_quota ())
+  if (opt.quota && total_downloaded_bytes > opt.quota)
     return QUOTEXC;
   else if (status == FWRITEERR)
     return FWRITEERR;

src/retr.c

@@ -66,6 +66,9 @@ extern int errno;
 /* See the comment in gethttp() why this is needed. */
 int global_download_count;
 
+/* Total size of downloaded files.  Used to enforce quota.  */
+LARGE_INT total_downloaded_bytes;
+
 
 static struct {
   long chunk_bytes;
@@ -576,7 +579,7 @@ retrieve_from_file (const char *file, int html, int *count)
       if (cur_url->ignore_when_downloading)
 	continue;
 
-      if (downloaded_exceeds_quota ())
+      if (opt.quota && total_downloaded_bytes > opt.quota)
 	{
 	  status = QUOTEXC;
 	  break;
@@ -614,40 +617,6 @@ printwhat (int n1, int n2)
   logputs (LOG_VERBOSE, (n1 == n2) ? _("Giving up.\n\n") : _("Retrying.\n\n"));
 }
 
-/* Increment opt.downloaded by BY_HOW_MUCH.  If an overflow occurs,
-   set opt.downloaded_overflow to 1. */
-void
-downloaded_increase (unsigned long by_how_much)
-{
-  VERY_LONG_TYPE old;
-  if (opt.downloaded_overflow)
-    return;
-  old = opt.downloaded;
-  opt.downloaded += by_how_much;
-  if (opt.downloaded < old)	/* carry flag, where are you when I
-                                   need you? */
-    {
-      /* Overflow. */
-      opt.downloaded_overflow = 1;
-      opt.downloaded = ~((VERY_LONG_TYPE)0);
-    }
-}
-
-/* Return non-zero if the downloaded amount of bytes exceeds the
-   desired quota.  If quota is not set or if the amount overflowed, 0
-   is returned. */
-int
-downloaded_exceeds_quota (void)
-{
-  if (!opt.quota)
-    return 0;
-  if (opt.downloaded_overflow)
-    /* We don't really know.  (Wildly) assume not. */
-    return 0;
-
-  return opt.downloaded > opt.quota;
-}
-
 /* If opt.wait or opt.waitretry are specified, and if certain
    conditions are met, sleep the appropriate number of seconds.  See
    the documentation of --wait and --waitretry for more information.

src/retr.h

@@ -43,9 +43,6 @@ char *retr_rate PARAMS ((long, double, int));
 double calc_rate PARAMS ((long, double, int *));
 void printwhat PARAMS ((int, int));
 
-void downloaded_increase PARAMS ((unsigned long));
-int downloaded_exceeds_quota PARAMS ((void));
-
 void sleep_between_retrievals PARAMS ((int));
 
 void rotate_backups PARAMS ((const char *));

src/sysdep.h

@@ -128,27 +128,30 @@ do {						\
   DEBUGP (("Closing fd %d\n", x));		\
 } while (0)
 
-/* Define a large ("very long") type useful for storing large
+/* Define a large integral type useful for storing large sizes that
-   non-negative quantities that exceed sizes of normal download.  Note
+   exceed sizes of one download, such as when printing the sum of all
-   that this has nothing to do with large file support.  For example,
+   downloads.  Note that this has nothing to do with large file
-   one should be able to say `--quota=10G', large files
+   support, yet.
-   notwithstanding.
 
-   On the machines where `long' is 64-bit, we use long.  Otherwise, we
+   We use a 64-bit integral type where available, `double' otherwise.
-   check whether `long long' is available and if yes, use that.  If
+   It's hard to print LARGE_INT's portably, but fortunately it's
-   long long is unavailable, we give up and just use `long'.
+   rarely needed.  */
 
-   Note: you cannot use VERY_LONG_TYPE along with printf().  When you
+#if SIZEOF_LONG >= 8
-   need to print it, use very_long_to_string().  */
+/* Long is large enough: use it.  */
-
+typedef long LARGE_INT;
-#if SIZEOF_LONG >= 8 || SIZEOF_LONG_LONG == 0
+# define LARGE_INT_FMT "%ld"
-/* either long is "big enough", or long long is unavailable which
+#else
-   leaves long as the only choice. */ 
+# if SIZEOF_LONG_LONG == 8
-# define VERY_LONG_TYPE   unsigned long
+/* Long long is large enough: use it.  */
-#else  /* use long long */
+typedef long long LARGE_INT;
-/* long is smaller than 8 bytes, but long long is available. */
+#  define LARGE_INT_FMT "%lld"
-# define VERY_LONG_TYPE   unsigned long long
+# else
-#endif /* use long long */
+/* Use `double'. */
+typedef double LARGE_INT;
+#  define LARGE_INT_FMT "%.0f"
+# endif
+#endif
 
 /* These are defined in cmpt.c if missing, therefore it's generally
    safe to declare their parameters.  */
@@ -167,6 +170,9 @@ char *strstr ();
 #ifndef HAVE_STRPTIME
 char *strptime ();
 #endif
+#ifndef HAVE_SNPRINTF
+int snprintf ();
+#endif
 #ifndef HAVE_VSNPRINTF
 int vsnprintf ();
 #endif

src/utils.c

@@ -1290,13 +1290,13 @@ free_keys_and_values (struct hash_table *ht)
 }
 
 
-/* Engine for legible and legible_very_long; this function works on
+/* Engine for legible and legible_large_int; add thousand separators
-   strings.  */
+   to numbers printed in strings.  */
 
 static char *
 legible_1 (const char *repr)
 {
-  static char outbuf[128];
+  static char outbuf[48];
   int i, i1, mod;
   char *outptr;
   const char *inptr;
@@ -1304,7 +1304,9 @@ legible_1 (const char *repr)
   /* Reset the pointers.  */
   outptr = outbuf;
   inptr = repr;
-  /* If the number is negative, shift the pointers.  */
+
+  /* Ignore the sign for the purpose of adding thousand
+     separators.  */
   if (*inptr == '-')
     {
       *outptr++ = '-';
@@ -1329,6 +1331,7 @@ legible_1 (const char *repr)
 }
 
 /* Legible -- return a static pointer to the legibly printed long.  */
+
 char *
 legible (long l)
 {
@@ -1338,53 +1341,29 @@ legible (long l)
   return legible_1 (inbuf);
 }
 
-/* Write a string representation of NUMBER into the provided buffer.
+/* Write a string representation of LARGE_INT NUMBER into the provided
-   We cannot use sprintf() because we cannot be sure whether the
+   buffer.  The buffer should be able to accept 24 characters,
-   platform supports printing of what we chose for VERY_LONG_TYPE.
+   including the terminating zero.
 
-   Example: Gcc supports `long long' under many platforms, but on many
+   It would be dangerous to use sprintf, because the code wouldn't
-   of those the native libc knows nothing of it and therefore cannot
+   work on a machine with gcc-provided long long support, but without
-   print it.
+   libc support for "%lld".  However, such platforms will typically
-
+   not have snprintf and will use our version, which does support
-   How long BUFFER needs to be depends on the platform and the content
+   "%lld" where long longs are available.  */
-   of NUMBER.  For 64-bit VERY_LONG_TYPE (the most common case), 24
-   bytes are sufficient.  Using more might be a good idea.
-
-   This function does not go through the hoops that long_to_string
-   goes to because it doesn't aspire to be fast.  (It's called perhaps
-   once in a Wget run.)  */
 
 static void
-very_long_to_string (char *buffer, VERY_LONG_TYPE number)
+large_int_to_string (char *buffer, LARGE_INT number)
 {
-  int i = 0;
+  snprintf (buffer, 24, LARGE_INT_FMT, number);
-  int j;
-
-  /* Print the number backwards... */
-  do
-    {
-      buffer[i++] = '0' + number % 10;
-      number /= 10;
-    }
-  while (number);
-
-  /* ...and reverse the order of the digits. */
-  for (j = 0; j < i / 2; j++)
-    {
-      char c = buffer[j];
-      buffer[j] = buffer[i - 1 - j];
-      buffer[i - 1 - j] = c;
-    }
-  buffer[i] = '\0';
 }
 
-/* The same as legible(), but works on VERY_LONG_TYPE.  See sysdep.h.  */
+/* The same as legible(), but works on LARGE_INT.  */
+
 char *
-legible_very_long (VERY_LONG_TYPE l)
+legible_large_int (LARGE_INT l)
 {
-  char inbuf[128];
+  char inbuf[48];
-  /* Print the number into the buffer.  */
+  large_int_to_string (inbuf, l);
-  very_long_to_string (inbuf, l);
   return legible_1 (inbuf);
 }
 

src/utils.h

@@ -102,7 +102,7 @@ void string_set_free PARAMS ((struct hash_table *));
 void free_keys_and_values PARAMS ((struct hash_table *));
 
 char *legible PARAMS ((long));
-char *legible_very_long PARAMS ((VERY_LONG_TYPE));
+char *legible_large_int PARAMS ((LARGE_INT));
 int numdigit PARAMS ((long));
 char *number_to_string PARAMS ((char *, long));