mirror of
https://github.com/moparisthebest/wget
synced 2024-07-03 16:38:41 -04:00
Support If-Modified-Since header in timestamping mode.
* src/wget.h: Add IF_MODIFIED_SINCE enum for dt. Add TIMECONV_ERR enum to uerr_t. * src/http.c (time_to_rfc1123): Convert time_t do http time. * src/http.c (initialize_request): Include If-Modified-Since header if appropriate. * src/http.c (set_file_timestamp): Separate this code from check_file_output. * src/http.c (check_file_output): Use set_file_timestamp. * src/http.c (gethttp): Handle properly 304 return code and 200 if server ignores If-Modified-Since headers. * src/http.c (http_loop): Load filename to hstat if condget was requested, use IF_MODIFIED_SINCE if requested and current timestamp can be obtained.
This commit is contained in:
Hubert Tarasiuk
Giuseppe Scrivano
parent
0e8d2d4251
commit
8a8d138dcc
240
src/http.c
240
src/http.c
@ -1681,6 +1681,58 @@ read_response_body (struct http_stat *hs, int sock, FILE *fp, wgint contlen,
|
||||
} while (0)
|
||||
#endif /* def __VMS [else] */
|
||||
|
||||
/*
|
||||
Convert time_t to one of valid HTTP date formats
|
||||
ie. rfc1123-date.
|
||||
|
||||
HTTP-date = rfc1123-date | rfc850-date | asctime-date
|
||||
rfc1123-date = wkday "," SP date1 SP time SP "GMT"
|
||||
rfc850-date = weekday "," SP date2 SP time SP "GMT"
|
||||
asctime-date = wkday SP date3 SP time SP 4DIGIT
|
||||
date1 = 2DIGIT SP month SP 4DIGIT
|
||||
; day month year (e.g., 02 Jun 1982)
|
||||
date2 = 2DIGIT "-" month "-" 2DIGIT
|
||||
; day-month-year (e.g., 02-Jun-82)
|
||||
date3 = month SP ( 2DIGIT | ( SP 1DIGIT ))
|
||||
; month day (e.g., Jun 2)
|
||||
time = 2DIGIT ":" 2DIGIT ":" 2DIGIT
|
||||
; 00:00:00 - 23:59:59
|
||||
wkday = "Mon" | "Tue" | "Wed"
|
||||
| "Thu" | "Fri" | "Sat" | "Sun"
|
||||
weekday = "Monday" | "Tuesday" | "Wednesday"
|
||||
| "Thursday" | "Friday" | "Saturday" | "Sunday"
|
||||
month = "Jan" | "Feb" | "Mar" | "Apr"
|
||||
| "May" | "Jun" | "Jul" | "Aug"
|
||||
| "Sep" | "Oct" | "Nov" | "Dec"
|
||||
|
||||
source: RFC2616 */
|
||||
static uerr_t
|
||||
time_to_rfc1123 (time_t time, char *buf, size_t bufsize)
|
||||
{
|
||||
static const char *wkday[] = { "Sun", "Mon", "Tue", "Wed",
|
||||
"Thu", "Fri", "Sat" };
|
||||
static const char *month[] = { "Jan", "Feb", "Mar", "Apr",
|
||||
"May", "Jun", "Jul", "Aug",
|
||||
"Sep", "Oct", "Nov", "Dec" };
|
||||
/* rfc1123 example: Thu, 01 Jan 1998 22:12:57 GMT */
|
||||
static const char *time_format = "%s, %02d %s %04d %02d:%02d:%02d GMT";
|
||||
|
||||
struct tm *gtm = gmtime (&time);
|
||||
if (!gtm)
|
||||
{
|
||||
logprintf (LOG_NOTQUIET,
|
||||
_("gmtime failed. This is probably a bug.\n"));
|
||||
return TIMECONV_ERR;
|
||||
}
|
||||
|
||||
snprintf (buf, bufsize, time_format, wkday[gtm->tm_wday],
|
||||
gtm->tm_mday, month[gtm->tm_mon],
|
||||
gtm->tm_year + 1900, gtm->tm_hour,
|
||||
gtm->tm_min, gtm->tm_sec);
|
||||
|
||||
return RETROK;
|
||||
}
|
||||
|
||||
static struct request *
|
||||
initialize_request (struct url *u, struct http_stat *hs, int *dt, struct url *proxy,
|
||||
bool inhibit_keep_alive, bool *basic_auth_finished,
|
||||
@ -1723,6 +1775,20 @@ initialize_request (struct url *u, struct http_stat *hs, int *dt, struct url *pr
|
||||
/* ... but some HTTP/1.0 caches doesn't implement Cache-Control. */
|
||||
request_set_header (req, "Pragma", "no-cache", rel_none);
|
||||
}
|
||||
if (*dt & IF_MODIFIED_SINCE)
|
||||
{
|
||||
char strtime[32];
|
||||
uerr_t err = time_to_rfc1123 (hs->orig_file_tstamp, strtime, countof (strtime));
|
||||
|
||||
if (err != RETROK)
|
||||
{
|
||||
logputs (LOG_VERBOSE, _("Cannot convert timestamp to http format. "
|
||||
"Falling back to time 0 as last modification "
|
||||
"time.\n"));
|
||||
strcpy (strtime, "Thu, 01 Jan 1970 00:00:00 GMT");
|
||||
}
|
||||
request_set_header (req, "If-Modified-Since", xstrdup (strtime), rel_value);
|
||||
}
|
||||
if (hs->restval)
|
||||
request_set_header (req, "Range",
|
||||
aprintf ("bytes=%s-",
|
||||
@ -2025,58 +2091,8 @@ establish_connection (struct url *u, struct url **conn_ref,
|
||||
}
|
||||
|
||||
static uerr_t
|
||||
check_file_output (struct url *u, struct http_stat *hs,
|
||||
struct response *resp, char *hdrval, size_t hdrsize)
|
||||
set_file_timestamp (struct http_stat *hs)
|
||||
{
|
||||
/* Determine the local filename if needed. Notice that if -O is used
|
||||
* hstat.local_file is set by http_loop to the argument of -O. */
|
||||
if (!hs->local_file)
|
||||
{
|
||||
char *local_file = NULL;
|
||||
|
||||
/* Honor Content-Disposition whether possible. */
|
||||
if (!opt.content_disposition
|
||||
|| !resp_header_copy (resp, "Content-Disposition",
|
||||
hdrval, hdrsize)
|
||||
|| !parse_content_disposition (hdrval, &local_file))
|
||||
{
|
||||
/* The Content-Disposition header is missing or broken.
|
||||
* Choose unique file name according to given URL. */
|
||||
hs->local_file = url_file_name (u, NULL);
|
||||
}
|
||||
else
|
||||
{
|
||||
DEBUGP (("Parsed filename from Content-Disposition: %s\n",
|
||||
local_file));
|
||||
hs->local_file = url_file_name (u, local_file);
|
||||
}
|
||||
|
||||
xfree (local_file);
|
||||
}
|
||||
|
||||
/* TODO: perform this check only once. */
|
||||
if (!hs->existence_checked && file_exists_p (hs->local_file))
|
||||
{
|
||||
if (opt.noclobber && !opt.output_document)
|
||||
{
|
||||
/* If opt.noclobber is turned on and file already exists, do not
|
||||
retrieve the file. But if the output_document was given, then this
|
||||
test was already done and the file didn't exist. Hence the !opt.output_document */
|
||||
return RETRUNNEEDED;
|
||||
}
|
||||
else if (!ALLOW_CLOBBER)
|
||||
{
|
||||
char *unique = unique_name (hs->local_file, true);
|
||||
if (unique != hs->local_file)
|
||||
xfree (hs->local_file);
|
||||
hs->local_file = unique;
|
||||
}
|
||||
}
|
||||
hs->existence_checked = true;
|
||||
|
||||
/* Support timestamping */
|
||||
if (opt.timestamping && !hs->timestamp_checked)
|
||||
{
|
||||
size_t filename_len = strlen (hs->local_file);
|
||||
char *filename_plus_orig_suffix = alloca (filename_len + sizeof (ORIG_SFX));
|
||||
bool local_dot_orig_file_exists = false;
|
||||
@ -2131,7 +2147,68 @@ check_file_output (struct url *u, struct http_stat *hs,
|
||||
increase local time by 1 second for later comparison. */
|
||||
++hs->orig_file_tstamp;
|
||||
#endif
|
||||
hs->timestamp_checked = true;
|
||||
}
|
||||
|
||||
return RETROK;
|
||||
}
|
||||
|
||||
static uerr_t
|
||||
check_file_output (struct url *u, struct http_stat *hs,
|
||||
struct response *resp, char *hdrval, size_t hdrsize)
|
||||
{
|
||||
/* Determine the local filename if needed. Notice that if -O is used
|
||||
* hstat.local_file is set by http_loop to the argument of -O. */
|
||||
if (!hs->local_file)
|
||||
{
|
||||
char *local_file = NULL;
|
||||
|
||||
/* Honor Content-Disposition whether possible. */
|
||||
if (!opt.content_disposition
|
||||
|| !resp_header_copy (resp, "Content-Disposition",
|
||||
hdrval, hdrsize)
|
||||
|| !parse_content_disposition (hdrval, &local_file))
|
||||
{
|
||||
/* The Content-Disposition header is missing or broken.
|
||||
* Choose unique file name according to given URL. */
|
||||
hs->local_file = url_file_name (u, NULL);
|
||||
}
|
||||
else
|
||||
{
|
||||
DEBUGP (("Parsed filename from Content-Disposition: %s\n",
|
||||
local_file));
|
||||
hs->local_file = url_file_name (u, local_file);
|
||||
}
|
||||
|
||||
xfree (local_file);
|
||||
}
|
||||
|
||||
/* TODO: perform this check only once. */
|
||||
if (!hs->existence_checked && file_exists_p (hs->local_file))
|
||||
{
|
||||
if (opt.noclobber && !opt.output_document)
|
||||
{
|
||||
/* If opt.noclobber is turned on and file already exists, do not
|
||||
retrieve the file. But if the output_document was given, then this
|
||||
test was already done and the file didn't exist. Hence the !opt.output_document */
|
||||
return RETRUNNEEDED;
|
||||
}
|
||||
else if (!ALLOW_CLOBBER)
|
||||
{
|
||||
char *unique = unique_name (hs->local_file, true);
|
||||
if (unique != hs->local_file)
|
||||
xfree (hs->local_file);
|
||||
hs->local_file = unique;
|
||||
}
|
||||
}
|
||||
hs->existence_checked = true;
|
||||
|
||||
/* Support timestamping */
|
||||
if (opt.timestamping && !hs->timestamp_checked)
|
||||
{
|
||||
uerr_t timestamp_err = set_file_timestamp (hs);
|
||||
if (timestamp_err != RETROK)
|
||||
return timestamp_err;
|
||||
}
|
||||
return RETROK;
|
||||
}
|
||||
@ -2421,6 +2498,9 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy,
|
||||
POST). */
|
||||
bool head_only = !!(*dt & HEAD_ONLY);
|
||||
|
||||
/* Whether conditional get request will be issued. */
|
||||
bool cond_get = !!(*dt & IF_MODIFIED_SINCE);
|
||||
|
||||
char *head = NULL;
|
||||
struct response *resp = NULL;
|
||||
char hdrval[512];
|
||||
@ -3020,6 +3100,41 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy,
|
||||
}
|
||||
}
|
||||
|
||||
if (cond_get)
|
||||
{
|
||||
if (statcode == HTTP_STATUS_NOT_MODIFIED)
|
||||
{
|
||||
logprintf (LOG_VERBOSE,
|
||||
_("File %s not modified on server. Omitting download.\n\n"),
|
||||
quote (hs->local_file));
|
||||
*dt |= RETROKF;
|
||||
CLOSE_FINISH (sock);
|
||||
retval = RETRUNNEEDED;
|
||||
goto cleanup;
|
||||
}
|
||||
/* Handle the case when server ignores If-Modified-Since header. */
|
||||
else if (statcode == HTTP_STATUS_OK && hs->remote_time)
|
||||
{
|
||||
time_t tmr = http_atotm (hs->remote_time);
|
||||
|
||||
/* Check if the local file is up-to-date based on Last-Modified header
|
||||
and content length. */
|
||||
if (tmr != (time_t) - 1 && tmr <= hs->orig_file_tstamp
|
||||
&& (contlen == -1 || contlen == hs->orig_file_size))
|
||||
{
|
||||
logprintf (LOG_VERBOSE,
|
||||
_("Server ignored If-Modified-Since header for file %s.\n"
|
||||
"You might want to add --no-if-modified-since option."
|
||||
"\n\n"),
|
||||
quote (hs->local_file));
|
||||
*dt |= RETROKF;
|
||||
CLOSE_INVALIDATE (sock);
|
||||
retval = RETRUNNEEDED;
|
||||
goto cleanup;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (statcode == HTTP_STATUS_RANGE_NOT_SATISFIABLE
|
||||
|| (!opt.timestamping && hs->restval > 0 && statcode == HTTP_STATUS_OK
|
||||
&& contrange == 0 && contlen >= 0 && hs->restval >= contlen))
|
||||
@ -3263,15 +3378,30 @@ http_loop (struct url *u, struct url *original_url, char **newloc,
|
||||
if (opt.content_disposition && opt.always_rest)
|
||||
send_head_first = true;
|
||||
|
||||
/* Send preliminary HEAD request if -N is given and we have an existing
|
||||
* destination file. */
|
||||
if (!opt.output_document)
|
||||
file_name = url_file_name (opt.trustservernames ? u : original_url, NULL);
|
||||
else
|
||||
file_name = xstrdup (opt.output_document);
|
||||
if (opt.timestamping && (file_exists_p (file_name)
|
||||
|| opt.content_disposition))
|
||||
|
||||
if (opt.timestamping)
|
||||
{
|
||||
/* Use conditional get request if requested
|
||||
* and if timestamp is known at this moment. */
|
||||
if (opt.if_modified_since && file_exists_p (file_name) && !send_head_first)
|
||||
{
|
||||
*dt |= IF_MODIFIED_SINCE;
|
||||
{
|
||||
uerr_t timestamp_err = set_file_timestamp (&hstat);
|
||||
if (timestamp_err != RETROK)
|
||||
return timestamp_err;
|
||||
}
|
||||
}
|
||||
/* Send preliminary HEAD request if -N is given and we have existing
|
||||
* destination file or content disposition is enabled. */
|
||||
else if (file_exists_p (file_name) || opt.content_disposition)
|
||||
send_head_first = true;
|
||||
}
|
||||
|
||||
xfree (file_name);
|
||||
|
||||
/* THE loop */
|
||||
|
||||
@ -331,7 +331,8 @@ enum
|
||||
SEND_NOCACHE = 0x0008, /* send Pragma: no-cache directive */
|
||||
ACCEPTRANGES = 0x0010, /* Accept-ranges header was found */
|
||||
ADDED_HTML_EXTENSION = 0x0020, /* added ".html" extension due to -E */
|
||||
TEXTCSS = 0x0040 /* document is of type text/css */
|
||||
TEXTCSS = 0x0040, /* document is of type text/css */
|
||||
IF_MODIFIED_SINCE = 0x0080 /* use if-modified-since header */
|
||||
};
|
||||
|
||||
/* Universal error type -- used almost everywhere. Error reporting of
|
||||
@ -351,7 +352,8 @@ typedef enum
|
||||
RETRBADPATTERN, PROXERR,
|
||||
AUTHFAILED, QUOTEXC, WRITEFAILED, SSLINITFAILED, VERIFCERTERR,
|
||||
UNLINKERR, NEWLOCATION_KEEP_POST, CLOSEFAILED, ATTRMISSING, UNKNOWNATTR,
|
||||
WARC_ERR, WARC_TMP_FOPENERR, WARC_TMP_FWRITEERR
|
||||
WARC_ERR, WARC_TMP_FOPENERR, WARC_TMP_FWRITEERR,
|
||||
TIMECONV_ERR
|
||||
} uerr_t;
|
||||
|
||||
/* 2005-02-19 SMS.
|
||||
|
||||
Loading…
Reference in New Issue
Block a user