moparisthebest/wget
1
0
Fork 0
mirror of https://github.com/moparisthebest/wget synced 2024-07-03 16:38:41 -04:00
Code Issues Releases Wiki Activity

[svn] Use hash table for tag lookup in html-url.c and html-parse.c.

Browse Source
This commit is contained in:
hniksic 2003-10-09 08:01:58 -07:00
parent b49e89e78a
commit 233ebb78de
4 changed files with 149 additions and 202 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
src/ChangeLog
View File

@ -1,3 +1,14 @@
2003-10-09 Hrvoje Niksic <hniksic@xemacs.org>
* html-url.c (init_interesting): Initialize interesting_tags and
interesting_attributes as hash tables. This simplifies the code
immensely because hash tables handle allocation and remove
duplicates automatically.
(find_tag): Removed.
(collect_tags_mapper): Instead of calling find_tag, simply get the
entry from interesting_tags hash table, which is both simpler and
faster.
2003-10-09 Hrvoje Niksic <hniksic@xemacs.org> 2003-10-09 Hrvoje Niksic <hniksic@xemacs.org>
* hash.c (hash_table_get): Declare hash-table argument as const. * hash.c (hash_table_get): Declare hash-table argument as const.

56
src/html-parse.c
View File

@ -129,7 +129,18 @@ so, delete this exception statement from your version. */
# define ISALNUM(x) isalnum (x) # define ISALNUM(x) isalnum (x)
# define TOLOWER(x) tolower (x) # define TOLOWER(x) tolower (x)
# define TOUPPER(x) toupper (x) # define TOUPPER(x) toupper (x)
#endif /* STANDALONE */
struct hash_table {
int dummy;
};
static void *
hash_table_get (const struct hash_table *ht, void *ptr)
{
return ptr;
}
#else /* not STANDALONE */
# include "hash.h"
#endif
/* Pool support. A pool is a resizable chunk of memory. It is first /* Pool support. A pool is a resizable chunk of memory. It is first
allocated on the stack, and moved to the heap if it needs to be allocated on the stack, and moved to the heap if it needs to be
@ -390,24 +401,6 @@ convert_and_copy (struct pool *pool, const char *beg, const char *end, int flags
} }
} }
/* Check whether the contents of [POS, POS+LENGTH) match any of the
strings in the ARRAY. */
static int
array_allowed (const char **array, const char *beg, const char *end)
{
int length = end - beg;
if (array)
{
for (; *array; array++)
if (length >= strlen (*array)
&& !strncasecmp (*array, beg, length))
break;
if (!*array)
return 0;
}
return 1;
}
/* Originally we used to adhere to rfc 1866 here, and allowed only /* Originally we used to adhere to rfc 1866 here, and allowed only
letters, digits, periods, and hyphens as names (of tags or letters, digits, periods, and hyphens as names (of tags or
attributes). However, this broke too many pages which used attributes). However, this broke too many pages which used
@ -659,6 +652,19 @@ find_comment_end (const char *beg, const char *end)
return NULL; return NULL;
} }
/* Return non-zero of the string inside [b, e) are present in hash
table HT. */
static int
name_allowed (const struct hash_table *ht, const char *b, const char *e)
{
char *copy;
if (!ht)
return 1;
BOUNDED_TO_ALLOCA (b, e, copy);
return hash_table_get (ht, copy) != NULL;
}
/* Advance P (a char pointer), with the explicit intent of being able /* Advance P (a char pointer), with the explicit intent of being able
to read the next character. If this is not possible, go to finish. */ to read the next character. If this is not possible, go to finish. */
@ -708,8 +714,8 @@ void
map_html_tags (const char *text, int size, map_html_tags (const char *text, int size,
void (*mapfun) (struct taginfo *, void *), void *maparg, void (*mapfun) (struct taginfo *, void *), void *maparg,
int flags, int flags,
const char **allowed_tag_names, const struct hash_table *allowed_tags,
const char **allowed_attribute_names) const struct hash_table *allowed_attributes)
{ {
/* storage for strings passed to MAPFUN callback; if 256 bytes is /* storage for strings passed to MAPFUN callback; if 256 bytes is
too little, POOL_APPEND allocates more with malloc. */ too little, POOL_APPEND allocates more with malloc. */
@ -793,7 +799,7 @@ map_html_tags (const char *text, int size,
if (end_tag && *p != '>') if (end_tag && *p != '>')
goto backout_tag; goto backout_tag;
if (!array_allowed (allowed_tag_names, tag_name_begin, tag_name_end)) if (!name_allowed (allowed_tags, tag_name_begin, tag_name_end))
/* We can't just say "goto look_for_tag" here because we need /* We can't just say "goto look_for_tag" here because we need
the loop below to properly advance over the tag's attributes. */ the loop below to properly advance over the tag's attributes. */
uninteresting_tag = 1; uninteresting_tag = 1;
@ -934,9 +940,7 @@ map_html_tags (const char *text, int size,
/* If we aren't interested in the attribute, skip it. We /* If we aren't interested in the attribute, skip it. We
cannot do this test any sooner, because our text pointer cannot do this test any sooner, because our text pointer
needs to correctly advance over the attribute. */ needs to correctly advance over the attribute. */
if (allowed_attribute_names if (!name_allowed (allowed_attributes, attr_name_begin, attr_name_end))
&& !array_allowed (allowed_attribute_names, attr_name_begin,
attr_name_end))
continue; continue;
GROW_ARRAY (pairs, attr_pair_size, nattrs + 1, attr_pair_resized, GROW_ARRAY (pairs, attr_pair_size, nattrs + 1, attr_pair_resized,
@ -1034,7 +1038,7 @@ int main ()
x = (char *)xrealloc (x, size); x = (char *)xrealloc (x, size);
} }
map_html_tags (x, length, NULL, NULL, test_mapper, &tag_counter); map_html_tags (x, length, test_mapper, &tag_counter, 0, NULL, NULL);
printf ("TAGS: %d\n", tag_counter); printf ("TAGS: %d\n", tag_counter);
printf ("Tag backouts: %d\n", tag_backout_count); printf ("Tag backouts: %d\n", tag_backout_count);
printf ("Comment backouts: %d\n", comment_backout_count); printf ("Comment backouts: %d\n", comment_backout_count);

7
src/html-parse.h
View File

@ -53,13 +53,16 @@ struct taginfo {
const char *end_position; /* end position of tag */ const char *end_position; /* end position of tag */
}; };
struct hash_table; /* forward declaration */
/* Flags for map_html_tags: */ /* Flags for map_html_tags: */
#define MHT_STRICT_COMMENTS 1 /* use strict comment interpretation */ #define MHT_STRICT_COMMENTS 1 /* use strict comment interpretation */
#define MHT_TRIM_VALUES 2 /* trim attribute values, e.g. interpret #define MHT_TRIM_VALUES 2 /* trim attribute values, e.g. interpret
<a href=" foo "> as "foo" */ <a href=" foo "> as "foo" */
void map_html_tags PARAMS ((const char *, int, void map_html_tags PARAMS ((const char *, int,
void (*) (struct taginfo *, void *), void *, void (*) (struct taginfo *, void *), void *, int,
int, const char **, const char **)); const struct hash_table *,
const struct hash_table *));
#endif /* HTML_PARSE_H */ #endif /* HTML_PARSE_H */

259
src/html-url.c
View File

@ -1,12 +1,12 @@
/* Collect URLs from HTML source. /* Collect URLs from HTML source.
Copyright (C) 1998, 2000, 2001, 2002 Free Software Foundation, Inc. Copyright (C) 1998, 2000, 2001, 2002, 2003 Free Software Foundation, Inc.
This file is part of GNU Wget. This file is part of GNU Wget.
GNU Wget is free software; you can redistribute it and/or modify GNU Wget is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or the Free Software Foundation; either version 2 of the License, or
(at your option) any later version. (at your option) any later version.
GNU Wget is distributed in the hope that it will be useful, GNU Wget is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of but WITHOUT ANY WARRANTY; without even the implied warranty of
@ -43,6 +43,7 @@ so, delete this exception statement from your version. */
#include "html-parse.h" #include "html-parse.h"
#include "url.h" #include "url.h"
#include "utils.h" #include "utils.h"
#include "hash.h"
#include "convert.h" #include "convert.h"
#ifndef errno #ifndef errno
@ -63,54 +64,58 @@ DECLARE_TAG_HANDLER (tag_handle_form);
DECLARE_TAG_HANDLER (tag_handle_link); DECLARE_TAG_HANDLER (tag_handle_link);
DECLARE_TAG_HANDLER (tag_handle_meta); DECLARE_TAG_HANDLER (tag_handle_meta);
enum {
TAG_A,
TAG_APPLET,
TAG_AREA,
TAG_BASE,
TAG_BGSOUND,
TAG_BODY,
TAG_EMBED,
TAG_FIG,
TAG_FORM,
TAG_FRAME,
TAG_IFRAME,
TAG_IMG,
TAG_INPUT,
TAG_LAYER,
TAG_LINK,
TAG_META,
TAG_OVERLAY,
TAG_SCRIPT,
TAG_TABLE,
TAG_TD,
TAG_TH
};
/* The list of known tags and functions used for handling them. Most /* The list of known tags and functions used for handling them. Most
tags are simply harvested for URLs. */ tags are simply harvested for URLs. */
static struct { static struct known_tag {
int tagid;
const char *name; const char *name;
tag_handler_t handler; tag_handler_t handler;
} known_tags[] = { } known_tags[] = {
#define TAG_A 0 { TAG_A, "a", tag_find_urls },
{ "a", tag_find_urls }, { TAG_APPLET, "applet", tag_find_urls },
#define TAG_APPLET 1 { TAG_AREA, "area", tag_find_urls },
{ "applet", tag_find_urls }, { TAG_BASE, "base", tag_handle_base },
#define TAG_AREA 2 { TAG_BGSOUND, "bgsound", tag_find_urls },
{ "area", tag_find_urls }, { TAG_BODY, "body", tag_find_urls },
#define TAG_BASE 3 { TAG_EMBED, "embed", tag_find_urls },
{ "base", tag_handle_base }, { TAG_FIG, "fig", tag_find_urls },
#define TAG_BGSOUND 4 { TAG_FORM, "form", tag_handle_form },
{ "bgsound", tag_find_urls }, { TAG_FRAME, "frame", tag_find_urls },
#define TAG_BODY 5 { TAG_IFRAME, "iframe", tag_find_urls },
{ "body", tag_find_urls }, { TAG_IMG, "img", tag_find_urls },
#define TAG_EMBED 6 { TAG_INPUT, "input", tag_find_urls },
{ "embed", tag_find_urls }, { TAG_LAYER, "layer", tag_find_urls },
#define TAG_FIG 7 { TAG_LINK, "link", tag_handle_link },
{ "fig", tag_find_urls }, { TAG_META, "meta", tag_handle_meta },
#define TAG_FORM 8 { TAG_OVERLAY, "overlay", tag_find_urls },
{ "form", tag_handle_form }, { TAG_SCRIPT, "script", tag_find_urls },
#define TAG_FRAME 9 { TAG_TABLE, "table", tag_find_urls },
{ "frame", tag_find_urls }, { TAG_TD, "td", tag_find_urls },
#define TAG_IFRAME 10 { TAG_TH, "th", tag_find_urls }
{ "iframe", tag_find_urls },
#define TAG_IMG 11
{ "img", tag_find_urls },
#define TAG_INPUT 12
{ "input", tag_find_urls },
#define TAG_LAYER 13
{ "layer", tag_find_urls },
#define TAG_LINK 14
{ "link", tag_handle_link },
#define TAG_META 15
{ "meta", tag_handle_meta },
#define TAG_OVERLAY 16
{ "overlay", tag_find_urls },
#define TAG_SCRIPT 17
{ "script", tag_find_urls },
#define TAG_TABLE 18
{ "table", tag_find_urls },
#define TAG_TD 19
{ "td", tag_find_urls },
#define TAG_TH 20
{ "th", tag_find_urls }
}; };
/* tag_url_attributes documents which attributes of which tags contain /* tag_url_attributes documents which attributes of which tags contain
@ -162,8 +167,8 @@ static const char *additional_attributes[] = {
"action" /* used by tag_handle_form */ "action" /* used by tag_handle_form */
}; };
static const char **interesting_tags; struct hash_table *interesting_tags;
static const char **interesting_attributes; struct hash_table *interesting_attributes;
static void static void
init_interesting (void) init_interesting (void)
@ -175,125 +180,49 @@ init_interesting (void)
Here we also make sure that what we put in interesting_tags Here we also make sure that what we put in interesting_tags
matches the user's preferences as specified through --ignore-tags matches the user's preferences as specified through --ignore-tags
and --follow-tags. and --follow-tags. */
This function is as large as this only because of the glorious int i;
expressivity of the C programming language. */ interesting_tags = make_nocase_string_hash_table (countof (known_tags));
{ /* First, add all the tags we know hot to handle, mapped to their
int i, ind = 0; respective entries in known_tags. */
int size = countof (known_tags); for (i = 0; i < countof (known_tags); i++)
interesting_tags = (const char **)xmalloc ((size + 1) * sizeof (char *)); hash_table_put (interesting_tags, known_tags[i].name, known_tags + i);
for (i = 0; i < size; i++)
{
const char *name = known_tags[i].name;
/* Normally here we could say:
interesting_tags[i] = name;
But we need to respect the settings of --ignore-tags and
--follow-tags, so the code gets a bit hairier. */
/* Then remove the tags ignored through --ignore-tags. */
if (opt.ignore_tags) if (opt.ignore_tags)
{ {
/* --ignore-tags was specified. Do not match these char **ignored;
specific tags. --ignore-tags takes precedence over for (ignored = opt.ignore_tags; *ignored; ignored++)
--follow-tags, so we process --ignore first and fall hash_table_remove (interesting_tags, *ignored);
through if there's no match. */
int j, lose = 0;
for (j = 0; opt.ignore_tags[j] != NULL; j++)
/* Loop through all the tags this user doesn't care about. */
if (strcasecmp(opt.ignore_tags[j], name) == EQ)
{
lose = 1;
break;
}
if (lose)
continue;
} }
/* If --follow-tags is specified, use only those tags. */
if (opt.follow_tags) if (opt.follow_tags)
{ {
/* --follow-tags was specified. Only match these specific tags, so /* Create a new hash table with the intersection of tags in
continue back to top of for if we don't match one of them. */ --follow-tags and known_tags, and use that as
int j, win = 0; interesting_tags. */
for (j = 0; opt.follow_tags[j] != NULL; j++) struct hash_table *intersect = make_nocase_string_hash_table (0);
/* Loop through all the tags this user cares about. */ char **followed;
if (strcasecmp(opt.follow_tags[j], name) == EQ) for (followed = opt.follow_tags; *followed; followed++)
{ {
win = 1; struct known_tag *t = hash_table_get (interesting_tags, *followed);
break; if (!t)
continue; /* ignore unknown tags in --follow-tags. */
hash_table_put (intersect, *followed, t);
} }
if (!win) hash_table_destroy (interesting_tags);
continue; /* wasn't one of the explicitly desired tags */ interesting_tags = intersect;
} }
/* If we get to here, --follow-tags isn't being used or the /* Add the attributes we care about. */
tag is among the ones that are followed, and --ignore-tags, interesting_attributes = make_nocase_string_hash_table (17);
if specified, didn't include this tag, so it's an
"interesting" one. */
interesting_tags[ind++] = name;
}
interesting_tags[ind] = NULL;
}
/* The same for attributes, except we loop through tag_url_attributes.
Here we also need to make sure that the list of attributes is
unique, and to include the attributes from additional_attributes. */
{
int i, ind;
const char **att = xmalloc ((countof (additional_attributes) + 1)
* sizeof (char *));
/* First copy the "additional" attributes. */
for (i = 0; i < countof (additional_attributes); i++) for (i = 0; i < countof (additional_attributes); i++)
att[i] = additional_attributes[i]; string_set_add (interesting_attributes, additional_attributes[i]);
ind = i;
att[ind] = NULL;
for (i = 0; i < countof (tag_url_attributes); i++) for (i = 0; i < countof (tag_url_attributes); i++)
{ string_set_add (interesting_attributes, tag_url_attributes[i].attr_name);
int j, seen = 0;
const char *look_for = tag_url_attributes[i].attr_name;
for (j = 0; j < ind - 1; j++)
if (!strcmp (att[j], look_for))
{
seen = 1;
break;
}
if (!seen)
{
att = xrealloc (att, (ind + 2) * sizeof (*att));
att[ind++] = look_for;
att[ind] = NULL;
}
}
interesting_attributes = att;
}
}
/* Find tag with name TAG_NAME in KNOWN_TAGS and return its index. */
static int
find_tag (const char *tag_name)
{
/* Originally implemented as linear search. In Wget 1.9 known_tags
contains 21 elements, for which binary search requires max. 5
comparisons, whereas linear search performs 10 on average. */
int lo = 0, hi = countof (known_tags) - 1;
while (lo <= hi)
{
int mid = (lo + hi) >> 1;
int cmp = strcasecmp (tag_name, known_tags[mid].name);
if (cmp < 0)
hi = mid - 1;
else if (cmp > 0)
lo = mid + 1;
else
return mid;
}
return -1;
} }
/* Find the value of attribute named NAME in the taginfo TAG. If the /* Find the value of attribute named NAME in the taginfo TAG. If the
@ -427,10 +356,10 @@ append_one_url (const char *link_uri, int inlinep,
static void static void
tag_find_urls (int tagid, struct taginfo *tag, struct map_context *ctx) tag_find_urls (int tagid, struct taginfo *tag, struct map_context *ctx)
{ {
int i, attrind, first = -1; int i, attrind;
int size = countof (tag_url_attributes); int first = -1;
for (i = 0; i < size; i++) for (i = 0; i < countof (tag_url_attributes); i++)
if (tag_url_attributes[i].tagid == tagid) if (tag_url_attributes[i].tagid == tagid)
{ {
/* We've found the index of tag_url_attributes where the /* We've found the index of tag_url_attributes where the
@ -454,11 +383,12 @@ tag_find_urls (int tagid, struct taginfo *tag, struct map_context *ctx)
/* Find whether TAG/ATTRIND is a combination that contains a /* Find whether TAG/ATTRIND is a combination that contains a
URL. */ URL. */
char *link = tag->attrs[attrind].value; char *link = tag->attrs[attrind].value;
const int size = countof (tag_url_attributes);
/* If you're cringing at the inefficiency of the nested loops, /* If you're cringing at the inefficiency of the nested loops,
remember that they both iterate over a laughably small remember that they both iterate over a very small number of
quantity of items. The worst-case inner loop is for the IMG items. The worst-case inner loop is for the IMG tag, which
tag, which has three attributes. */ has three attributes. */
for (i = first; i < size && tag_url_attributes[i].tagid == tagid; i++) for (i = first; i < size && tag_url_attributes[i].tagid == tagid; i++)
{ {
if (0 == strcasecmp (tag->attrs[attrind].name, if (0 == strcasecmp (tag->attrs[attrind].name,
@ -617,21 +547,20 @@ tag_handle_meta (int tagid, struct taginfo *tag, struct map_context *ctx)
} }
} }
/* Examine name and attributes of TAG and take appropriate action /* Dispatch the tag handler appropriate for the tag we're mapping
according to the tag. */ over. See known_tags[] for definition of tag handlers. */
static void static void
collect_tags_mapper (struct taginfo *tag, void *arg) collect_tags_mapper (struct taginfo *tag, void *arg)
{ {
struct map_context *ctx = (struct map_context *)arg; struct map_context *ctx = (struct map_context *)arg;
int tagid;
tag_handler_t handler;
tagid = find_tag (tag->name); /* Find the tag in our table of tags. This must not fail because
assert (tagid != -1); map_html_tags only returns tags found in interesting_tags. */
handler = known_tags[tagid].handler; struct known_tag *t = hash_table_get (interesting_tags, tag->name);
assert (t != NULL);
handler (tagid, tag, ctx); t->handler (t->tagid, tag, ctx);
} }
/* Analyze HTML tags FILE and construct a list of URLs referenced from /* Analyze HTML tags FILE and construct a list of URLs referenced from