[svn] Use hash table for tag lookup in html-url.c and html-parse.c.

src/ChangeLog

@@ -1,3 +1,14 @@
+2003-10-09  Hrvoje Niksic  <hniksic@xemacs.org>
+
+	* html-url.c (init_interesting): Initialize interesting_tags and
+	interesting_attributes as hash tables.  This simplifies the code
+	immensely because hash tables handle allocation and remove
+	duplicates automatically.
+	(find_tag): Removed.
+	(collect_tags_mapper): Instead of calling find_tag, simply get the
+	entry from interesting_tags hash table, which is both simpler and
+	faster.
+
 2003-10-09  Hrvoje Niksic  <hniksic@xemacs.org>
 
 	* hash.c (hash_table_get): Declare hash-table argument as const.

src/html-parse.c

@@ -129,7 +129,18 @@ so, delete this exception statement from your version.  */
 # define ISALNUM(x) isalnum (x)
 # define TOLOWER(x) tolower (x)
 # define TOUPPER(x) toupper (x)
-#endif /* STANDALONE */
+
+struct hash_table {
+  int dummy;
+};
+static void *
+hash_table_get (const struct hash_table *ht, void *ptr)
+{
+  return ptr;
+}
+#else  /* not STANDALONE */
+# include "hash.h"
+#endif
 
 /* Pool support.  A pool is a resizable chunk of memory.  It is first
    allocated on the stack, and moved to the heap if it needs to be
@@ -390,24 +401,6 @@ convert_and_copy (struct pool *pool, const char *beg, const char *end, int flags
     }
 }
 
-/* Check whether the contents of [POS, POS+LENGTH) match any of the
-   strings in the ARRAY.  */
-static int
-array_allowed (const char **array, const char *beg, const char *end)
-{
-  int length = end - beg;
-  if (array)
-    {
-      for (; *array; array++)
-	if (length >= strlen (*array)
-	    && !strncasecmp (*array, beg, length))
-	  break;
-      if (!*array)
-	return 0;
-    }
-  return 1;
-}
-
 /* Originally we used to adhere to rfc 1866 here, and allowed only
    letters, digits, periods, and hyphens as names (of tags or
    attributes).  However, this broke too many pages which used
@@ -659,6 +652,19 @@ find_comment_end (const char *beg, const char *end)
   return NULL;
 }
 
+/* Return non-zero of the string inside [b, e) are present in hash
+   table HT.  */
+
+static int
+name_allowed (const struct hash_table *ht, const char *b, const char *e)
+{
+  char *copy;
+  if (!ht)
+    return 1;
+  BOUNDED_TO_ALLOCA (b, e, copy);
+  return hash_table_get (ht, copy) != NULL;
+}
+
 /* Advance P (a char pointer), with the explicit intent of being able
    to read the next character.  If this is not possible, go to finish.  */
 
@@ -708,8 +714,8 @@ void
 map_html_tags (const char *text, int size,
 	       void (*mapfun) (struct taginfo *, void *), void *maparg,
 	       int flags,
-	       const char **allowed_tag_names,
+	       const struct hash_table *allowed_tags,
-	       const char **allowed_attribute_names)
+	       const struct hash_table *allowed_attributes)
 {
   /* storage for strings passed to MAPFUN callback; if 256 bytes is
      too little, POOL_APPEND allocates more with malloc. */
@@ -793,7 +799,7 @@ map_html_tags (const char *text, int size,
     if (end_tag && *p != '>')
       goto backout_tag;
 
-    if (!array_allowed (allowed_tag_names, tag_name_begin, tag_name_end))
+    if (!name_allowed (allowed_tags, tag_name_begin, tag_name_end))
       /* We can't just say "goto look_for_tag" here because we need
          the loop below to properly advance over the tag's attributes.  */
       uninteresting_tag = 1;
@@ -934,9 +940,7 @@ map_html_tags (const char *text, int size,
 	/* If we aren't interested in the attribute, skip it.  We
            cannot do this test any sooner, because our text pointer
            needs to correctly advance over the attribute.  */
-	if (allowed_attribute_names
+	if (!name_allowed (allowed_attributes, attr_name_begin, attr_name_end))
-	    && !array_allowed (allowed_attribute_names, attr_name_begin,
-			       attr_name_end))
 	  continue;
 
 	GROW_ARRAY (pairs, attr_pair_size, nattrs + 1, attr_pair_resized,
@@ -1034,7 +1038,7 @@ int main ()
       x = (char *)xrealloc (x, size);
     }
 
-  map_html_tags (x, length, NULL, NULL, test_mapper, &tag_counter);
+  map_html_tags (x, length, test_mapper, &tag_counter, 0, NULL, NULL);
   printf ("TAGS: %d\n", tag_counter);
   printf ("Tag backouts:     %d\n", tag_backout_count);
   printf ("Comment backouts: %d\n", comment_backout_count);

src/html-parse.h

@@ -53,13 +53,16 @@ struct taginfo {
   const char *end_position;	/* end position of tag */
 };
 
+struct hash_table;		/* forward declaration */
+
 /* Flags for map_html_tags: */
 #define MHT_STRICT_COMMENTS  1  /* use strict comment interpretation */
 #define MHT_TRIM_VALUES      2  /* trim attribute values, e.g. interpret
                                    <a href=" foo "> as "foo" */
 
 void map_html_tags PARAMS ((const char *, int,
-			    void (*) (struct taginfo *, void *), void *,
+			    void (*) (struct taginfo *, void *), void *, int,
-			    int, const char **, const char **));
+			    const struct hash_table *,
+			    const struct hash_table *));
 
 #endif /* HTML_PARSE_H */

src/html-url.c

@@ -1,12 +1,12 @@
 /* Collect URLs from HTML source.
-   Copyright (C) 1998, 2000, 2001, 2002 Free Software Foundation, Inc.
+   Copyright (C) 1998, 2000, 2001, 2002, 2003 Free Software Foundation, Inc.
 
 This file is part of GNU Wget.
 
 GNU Wget is free software; you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
 the Free Software Foundation; either version 2 of the License, or
-(at your option) any later version.
+ (at your option) any later version.
 
 GNU Wget is distributed in the hope that it will be useful,
 but WITHOUT ANY WARRANTY; without even the implied warranty of
@@ -43,6 +43,7 @@ so, delete this exception statement from your version.  */
 #include "html-parse.h"
 #include "url.h"
 #include "utils.h"
+#include "hash.h"
 #include "convert.h"
 
 #ifndef errno
@@ -63,54 +64,58 @@ DECLARE_TAG_HANDLER (tag_handle_form);
 DECLARE_TAG_HANDLER (tag_handle_link);
 DECLARE_TAG_HANDLER (tag_handle_meta);
 
+enum {
+  TAG_A,
+  TAG_APPLET,
+  TAG_AREA,
+  TAG_BASE,
+  TAG_BGSOUND,
+  TAG_BODY,
+  TAG_EMBED,
+  TAG_FIG,
+  TAG_FORM,
+  TAG_FRAME,
+  TAG_IFRAME,
+  TAG_IMG,
+  TAG_INPUT,
+  TAG_LAYER,
+  TAG_LINK,
+  TAG_META,
+  TAG_OVERLAY,
+  TAG_SCRIPT,
+  TAG_TABLE,
+  TAG_TD,
+  TAG_TH
+};
+
 /* The list of known tags and functions used for handling them.  Most
    tags are simply harvested for URLs. */
-static struct {
+static struct known_tag {
+  int tagid;
   const char *name;
   tag_handler_t handler;
 } known_tags[] = {
-#define TAG_A		0
+  { TAG_A,	 "a",		tag_find_urls },
-  { "a",	tag_find_urls },
+  { TAG_APPLET,	 "applet",	tag_find_urls },
-#define TAG_APPLET	1
+  { TAG_AREA,	 "area",	tag_find_urls },
-  { "applet",	tag_find_urls },
+  { TAG_BASE,	 "base",	tag_handle_base },
-#define TAG_AREA	2
+  { TAG_BGSOUND, "bgsound",	tag_find_urls },
-  { "area",	tag_find_urls },
+  { TAG_BODY,	 "body",	tag_find_urls },
-#define TAG_BASE	3
+  { TAG_EMBED,	 "embed",	tag_find_urls },
-  { "base",	tag_handle_base },
+  { TAG_FIG,	 "fig",		tag_find_urls },
-#define TAG_BGSOUND	4
+  { TAG_FORM,	 "form",	tag_handle_form },
-  { "bgsound",	tag_find_urls },
+  { TAG_FRAME,	 "frame",	tag_find_urls },
-#define TAG_BODY	5
+  { TAG_IFRAME,	 "iframe",	tag_find_urls },
-  { "body",	tag_find_urls },
+  { TAG_IMG,	 "img",		tag_find_urls },
-#define TAG_EMBED	6
+  { TAG_INPUT,	 "input",	tag_find_urls },
-  { "embed",	tag_find_urls },
+  { TAG_LAYER,	 "layer",	tag_find_urls },
-#define TAG_FIG		7
+  { TAG_LINK,	 "link",	tag_handle_link },
-  { "fig",	tag_find_urls },
+  { TAG_META,	 "meta",	tag_handle_meta },
-#define TAG_FORM	8
+  { TAG_OVERLAY, "overlay",	tag_find_urls },
-  { "form",	tag_handle_form },
+  { TAG_SCRIPT,	 "script",	tag_find_urls },
-#define TAG_FRAME	9
+  { TAG_TABLE,	 "table",	tag_find_urls },
-  { "frame",	tag_find_urls },
+  { TAG_TD,	 "td",		tag_find_urls },
-#define TAG_IFRAME	10
+  { TAG_TH,	 "th",		tag_find_urls }
-  { "iframe",	tag_find_urls },
-#define TAG_IMG		11
-  { "img",	tag_find_urls },
-#define TAG_INPUT	12
-  { "input",	tag_find_urls },
-#define TAG_LAYER	13
-  { "layer",	tag_find_urls },
-#define TAG_LINK	14
-  { "link",	tag_handle_link },
-#define TAG_META	15
-  { "meta",	tag_handle_meta },
-#define TAG_OVERLAY	16
-  { "overlay",	tag_find_urls },
-#define TAG_SCRIPT	17
-  { "script",	tag_find_urls },
-#define TAG_TABLE	18
-  { "table",	tag_find_urls },
-#define TAG_TD		19
-  { "td",	tag_find_urls },
-#define TAG_TH		20
-  { "th",	tag_find_urls }
 };
 
 /* tag_url_attributes documents which attributes of which tags contain
@@ -162,8 +167,8 @@ static const char *additional_attributes[] = {
   "action"			/* used by tag_handle_form */
 };
 
-static const char **interesting_tags;
+struct hash_table *interesting_tags;
-static const char **interesting_attributes;
+struct hash_table *interesting_attributes;
 
 static void
 init_interesting (void)
@@ -175,125 +180,49 @@ init_interesting (void)
 
      Here we also make sure that what we put in interesting_tags
      matches the user's preferences as specified through --ignore-tags
-     and --follow-tags.
+     and --follow-tags.  */
 
-     This function is as large as this only because of the glorious
+  int i;
-     expressivity of the C programming language.  */
+  interesting_tags = make_nocase_string_hash_table (countof (known_tags));
 
-  {
+  /* First, add all the tags we know hot to handle, mapped to their
-    int i, ind = 0;
+     respective entries in known_tags.  */
-    int size = countof (known_tags);
+  for (i = 0; i < countof (known_tags); i++)
-    interesting_tags = (const char **)xmalloc ((size + 1) * sizeof (char *));
+    hash_table_put (interesting_tags, known_tags[i].name, known_tags + i);
 
-    for (i = 0; i < size; i++)
+  /* Then remove the tags ignored through --ignore-tags.  */
-      {
+  if (opt.ignore_tags)
-	const char *name = known_tags[i].name;
-
-	/* Normally here we could say:
-	   interesting_tags[i] = name;
-	   But we need to respect the settings of --ignore-tags and
-	   --follow-tags, so the code gets a bit hairier.  */
-
-	if (opt.ignore_tags)
-	  {
-	    /* --ignore-tags was specified.  Do not match these
-	       specific tags.  --ignore-tags takes precedence over
-	       --follow-tags, so we process --ignore first and fall
-	       through if there's no match. */
-	    int j, lose = 0;
-	    for (j = 0; opt.ignore_tags[j] != NULL; j++)
-	      /* Loop through all the tags this user doesn't care about. */
-	      if (strcasecmp(opt.ignore_tags[j], name) == EQ)
-		{
-		  lose = 1;
-		  break;
-		}
-	    if (lose)
-	      continue;
-	  }
-
-	if (opt.follow_tags)
-	  {
-	    /* --follow-tags was specified.  Only match these specific tags, so
-	       continue back to top of for if we don't match one of them. */
-	    int j, win = 0;
-	    for (j = 0; opt.follow_tags[j] != NULL; j++)
-	      /* Loop through all the tags this user cares about. */
-	      if (strcasecmp(opt.follow_tags[j], name) == EQ)
-		{
-		  win = 1;
-		  break;
-		}
-	    if (!win)
-	      continue;  /* wasn't one of the explicitly desired tags */
-	  }
-
-	/* If we get to here, --follow-tags isn't being used or the
-	   tag is among the ones that are followed, and --ignore-tags,
-	   if specified, didn't include this tag, so it's an
-	   "interesting" one. */
-	interesting_tags[ind++] = name;
-      }
-    interesting_tags[ind] = NULL;
-  }
-
-  /* The same for attributes, except we loop through tag_url_attributes.
-     Here we also need to make sure that the list of attributes is
-     unique, and to include the attributes from additional_attributes.  */
-  {
-    int i, ind;
-    const char **att = xmalloc ((countof (additional_attributes) + 1)
-				* sizeof (char *));
-    /* First copy the "additional" attributes. */
-    for (i = 0; i < countof (additional_attributes); i++)
-      att[i] = additional_attributes[i];
-    ind = i;
-    att[ind] = NULL;
-    for (i = 0; i < countof (tag_url_attributes); i++)
-      {
-	int j, seen = 0;
-	const char *look_for = tag_url_attributes[i].attr_name;
-	for (j = 0; j < ind - 1; j++)
-	  if (!strcmp (att[j], look_for))
-	    {
-	      seen = 1;
-	      break;
-	    }
-	if (!seen)
-	  {
-	    att = xrealloc (att, (ind + 2) * sizeof (*att));
-	    att[ind++] = look_for;
-	    att[ind] = NULL;
-	  }
-      }
-    interesting_attributes = att;
-  }
-}
-
-/* Find tag with name TAG_NAME in KNOWN_TAGS and return its index.  */
-
-static int
-find_tag (const char *tag_name)
-{
-  /* Originally implemented as linear search.  In Wget 1.9 known_tags
-     contains 21 elements, for which binary search requires max. 5
-     comparisons, whereas linear search performs 10 on average.  */
-
-  int lo = 0, hi = countof (known_tags) - 1;
-
-  while (lo <= hi)
     {
-      int mid = (lo + hi) >> 1;
+      char **ignored;
-      int cmp = strcasecmp (tag_name, known_tags[mid].name);
+      for (ignored = opt.ignore_tags; *ignored; ignored++)
-      if (cmp < 0)
+	hash_table_remove (interesting_tags, *ignored);
-	hi = mid - 1;
-      else if (cmp > 0)
-	lo = mid + 1;
-      else
-	return mid;
     }
 
-  return -1;
+  /* If --follow-tags is specified, use only those tags.  */
+  if (opt.follow_tags)
+    {
+      /* Create a new hash table with the intersection of tags in
+	 --follow-tags and known_tags, and use that as
+	 interesting_tags.  */
+      struct hash_table *intersect = make_nocase_string_hash_table (0);
+      char **followed;
+      for (followed = opt.follow_tags; *followed; followed++)
+	{
+	  struct known_tag *t = hash_table_get (interesting_tags, *followed);
+	  if (!t)
+	    continue;		/* ignore unknown tags in --follow-tags. */
+	  hash_table_put (intersect, *followed, t);
+	}
+      hash_table_destroy (interesting_tags);
+      interesting_tags = intersect;
+    }
+
+  /* Add the attributes we care about. */
+  interesting_attributes = make_nocase_string_hash_table (17);
+  for (i = 0; i < countof (additional_attributes); i++)
+    string_set_add (interesting_attributes, additional_attributes[i]);
+  for (i = 0; i < countof (tag_url_attributes); i++)
+    string_set_add (interesting_attributes, tag_url_attributes[i].attr_name);
 }
 
 /* Find the value of attribute named NAME in the taginfo TAG.  If the
@@ -427,10 +356,10 @@ append_one_url (const char *link_uri, int inlinep,
 static void
 tag_find_urls (int tagid, struct taginfo *tag, struct map_context *ctx)
 {
-  int i, attrind, first = -1;
+  int i, attrind;
-  int size = countof (tag_url_attributes);
+  int first = -1;
 
-  for (i = 0; i < size; i++)
+  for (i = 0; i < countof (tag_url_attributes); i++)
     if (tag_url_attributes[i].tagid == tagid)
       {
 	/* We've found the index of tag_url_attributes where the
@@ -454,11 +383,12 @@ tag_find_urls (int tagid, struct taginfo *tag, struct map_context *ctx)
       /* Find whether TAG/ATTRIND is a combination that contains a
 	 URL. */
       char *link = tag->attrs[attrind].value;
+      const int size = countof (tag_url_attributes);
 
       /* If you're cringing at the inefficiency of the nested loops,
-	 remember that they both iterate over a laughably small
+	 remember that they both iterate over a very small number of
-	 quantity of items.  The worst-case inner loop is for the IMG
+	 items.  The worst-case inner loop is for the IMG tag, which
-	 tag, which has three attributes.  */
+	 has three attributes.  */
       for (i = first; i < size && tag_url_attributes[i].tagid == tagid; i++)
 	{
 	  if (0 == strcasecmp (tag->attrs[attrind].name,
@@ -617,21 +547,20 @@ tag_handle_meta (int tagid, struct taginfo *tag, struct map_context *ctx)
     }
 }
 
-/* Examine name and attributes of TAG and take appropriate action
+/* Dispatch the tag handler appropriate for the tag we're mapping
-   according to the tag.  */
+   over.  See known_tags[] for definition of tag handlers.  */
 
 static void
 collect_tags_mapper (struct taginfo *tag, void *arg)
 {
   struct map_context *ctx = (struct map_context *)arg;
-  int tagid;
-  tag_handler_t handler;
 
-  tagid = find_tag (tag->name);
+  /* Find the tag in our table of tags.  This must not fail because
-  assert (tagid != -1);
+     map_html_tags only returns tags found in interesting_tags.  */
-  handler = known_tags[tagid].handler;
+  struct known_tag *t = hash_table_get (interesting_tags, tag->name);
+  assert (t != NULL);
 
-  handler (tagid, tag, ctx);
+  t->handler (t->tagid, tag, ctx);
 }
 
 /* Analyze HTML tags FILE and construct a list of URLs referenced from