diff --git a/src/ChangeLog b/src/ChangeLog index 87e05571..f27a486f 100644 --- a/src/ChangeLog +++ b/src/ChangeLog @@ -1,3 +1,25 @@ +2003-11-21 Hrvoje Niksic + + * rbuf.c: Removed. + + * ftp-basic.c (ftp_response): Use fd_read_line. No longer use + struct rbuf. Updated all callers. + + * http.c (gethttp): Use fd_read_head to read all the headers in + one go. + (next_header): New function. + + * retr.c (fd_read_line): New function: reads a line from FD, + leaving the rest of the data unread. + (fd_read_head): New function. + + * connect.c (fd_peek): New function, implements peeking. + (poll_internal): New function. + (fd_read): Use it. + (fd_write): Ditto. + (fd_peek): Ditto. + (fd_register_transport): Allow registering a "peeker" callback. + 2003-11-20 Hrvoje Niksic * connect.c: Renamed xread/xwrite/xclose to diff --git a/src/Makefile.in b/src/Makefile.in index 2dabccc5..789f3477 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -75,7 +75,7 @@ GETOPT_OBJ = @GETOPT_OBJ@ OBJ = $(ALLOCA) cmpt$o connect$o convert$o cookies$o \ ftp$o ftp-basic$o ftp-ls$o $(OPIE_OBJ) $(GETOPT_OBJ) hash$o \ headers$o host$o html-parse$o html-url$o http$o init$o \ - log$o main$o $(MD5_OBJ) netrc$o progress$o rbuf$o recur$o \ + log$o main$o $(MD5_OBJ) netrc$o progress$o recur$o \ res$o retr$o safe-ctype$o snprintf$o $(SSL_OBJ) url$o \ utils$o version$o xmalloc$o @@ -157,12 +157,12 @@ cmpt$o: wget.h sysdep.h options.h safe-ctype.h connect$o: wget.h sysdep.h options.h safe-ctype.h utils.h connect.h host.h convert$o: wget.h convert.h url.h recur.h utils.h hash.h cookies$o: wget.h sysdep.h options.h safe-ctype.h cookies.h hash.h url.h utils.h -ftp-basic$o: wget.h sysdep.h options.h safe-ctype.h utils.h rbuf.h connect.h \ +ftp-basic$o: wget.h sysdep.h options.h safe-ctype.h utils.h connect.h \ host.h ftp.h -ftp-ls$o: wget.h sysdep.h options.h safe-ctype.h utils.h ftp.h rbuf.h host.h \ +ftp-ls$o: wget.h sysdep.h options.h safe-ctype.h utils.h ftp.h host.h \ url.h ftp-opie$o: wget.h sysdep.h options.h safe-ctype.h gen-md5.h -ftp$o: wget.h sysdep.h options.h safe-ctype.h utils.h url.h rbuf.h retr.h \ +ftp$o: wget.h sysdep.h options.h safe-ctype.h utils.h url.h retr.h \ ftp.h host.h connect.h netrc.h gen-md5$o: wget.h sysdep.h options.h safe-ctype.h gen-md5.h gen_sslfunc$o: wget.h sysdep.h options.h safe-ctype.h utils.h connect.h host.h \ @@ -170,30 +170,27 @@ gen_sslfunc$o: wget.h sysdep.h options.h safe-ctype.h utils.h connect.h host.h \ getopt$o: wget.h sysdep.h options.h safe-ctype.h getopt.h gnu-md5$o: wget.h sysdep.h options.h safe-ctype.h gnu-md5.h hash$o: wget.h sysdep.h options.h safe-ctype.h utils.h hash.h -headers$o: wget.h sysdep.h options.h safe-ctype.h connect.h host.h rbuf.h \ +headers$o: wget.h sysdep.h options.h safe-ctype.h connect.h host.h \ headers.h host$o: wget.h sysdep.h options.h safe-ctype.h utils.h host.h url.h hash.h html-parse$o: wget.h sysdep.h options.h safe-ctype.h html-parse.h html-url$o: wget.h sysdep.h options.h safe-ctype.h html-parse.h url.h utils.h -http$o: wget.h sysdep.h options.h safe-ctype.h utils.h url.h host.h rbuf.h \ +http$o: wget.h sysdep.h options.h safe-ctype.h utils.h url.h host.h \ retr.h headers.h connect.h host.h netrc.h gen_sslfunc.h \ cookies.h gen-md5.h init$o: wget.h sysdep.h options.h safe-ctype.h utils.h init.h host.h recur.h \ netrc.h cookies.h progress.h log$o: wget.h sysdep.h options.h safe-ctype.h utils.h -main$o: wget.h sysdep.h options.h safe-ctype.h utils.h init.h retr.h rbuf.h \ +main$o: wget.h sysdep.h options.h safe-ctype.h utils.h init.h retr.h \ recur.h host.h cookies.h url.h progress.h gen_sslfunc.h getopt.h gnu-md5$o: wget.h sysdep.h options.h safe-ctype.h gnu-md5.h mswindows$o: wget.h sysdep.h options.h safe-ctype.h utils.h url.h netrc$o: wget.h sysdep.h options.h safe-ctype.h utils.h netrc.h init.h -progress$o: wget.h sysdep.h options.h safe-ctype.h progress.h utils.h retr.h \ - rbuf.h -rbuf$o: wget.h sysdep.h options.h safe-ctype.h rbuf.h connect.h host.h \ - gen_sslfunc.h +progress$o: wget.h sysdep.h options.h safe-ctype.h progress.h utils.h retr.h recur$o: wget.h sysdep.h options.h safe-ctype.h url.h recur.h utils.h retr.h \ - rbuf.h ftp.h host.h hash.h + ftp.h host.h hash.h res$o: wget.h sysdep.h options.h safe-ctype.h utils.h hash.h url.h retr.h res.h -retr$o: wget.h sysdep.h options.h safe-ctype.h utils.h retr.h rbuf.h url.h \ +retr$o: wget.h sysdep.h options.h safe-ctype.h utils.h retr.h url.h \ recur.h ftp.h host.h connect.h hash.h snprintf$o: safe-ctype.h safe-ctype$o: safe-ctype.h diff --git a/src/connect.c b/src/connect.c index 9a402769..eeb4eb28 100644 --- a/src/connect.c +++ b/src/connect.c @@ -735,6 +735,16 @@ sock_poll (int fd, double timeout, int wait_for) return select_fd (fd, timeout, wait_for); } +static int +sock_peek (int fd, char *buf, int bufsize) +{ + int res; + do + res = recv (fd, buf, bufsize, MSG_PEEK); + while (res == -1 && errno == EINTR); + return res; +} + static void sock_close (int fd) { @@ -760,6 +770,7 @@ struct transport_info { fd_reader_t reader; fd_writer_t writer; fd_poller_t poller; + fd_peeker_t peeker; fd_closer_t closer; void *ctx; }; @@ -773,7 +784,8 @@ struct transport_info { void fd_register_transport (int fd, fd_reader_t reader, fd_writer_t writer, - fd_poller_t poller, fd_closer_t closer, void *ctx) + fd_poller_t poller, fd_peeker_t peeker, + fd_closer_t closer, void *ctx) { struct transport_info *info; @@ -786,6 +798,7 @@ fd_register_transport (int fd, fd_reader_t reader, fd_writer_t writer, info->reader = reader; info->writer = writer; info->poller = poller; + info->peeker = peeker; info->closer = closer; info->ctx = ctx; if (!transport_map) @@ -819,6 +832,26 @@ fd_register_transport (int fd, fd_reader_t reader, fd_writer_t writer, } \ } while (0) +static int +poll_internal (int fd, struct transport_info *info, int wf, double timeout) +{ + if (timeout == -1) + timeout = opt.read_timeout; + if (timeout) + { + int test; + if (info && info->poller) + test = info->poller (fd, timeout, wf, info->ctx); + else + test = sock_poll (fd, timeout, wf); + if (test == 0) + errno = ETIMEDOUT; + if (test <= 0) + return 0; + } + return 1; +} + /* Read no more than BUFSIZE bytes of data from FD, storing them to BUF. If TIMEOUT is non-zero, the operation aborts if no data is received after that many seconds. If TIMEOUT is -1, the value of @@ -829,26 +862,30 @@ fd_read (int fd, char *buf, int bufsize, double timeout) { struct transport_info *info; LAZY_RETRIEVE_INFO (info); - if (timeout == -1) - timeout = opt.read_timeout; - if (timeout) - { - int test; - if (info && info->poller) - test = info->poller (fd, timeout, WAIT_FOR_READ, info->ctx); - else - test = sock_poll (fd, timeout, WAIT_FOR_READ); - if (test == 0) - errno = ETIMEDOUT; - if (test <= 0) - return -1; - } + if (!poll_internal (fd, info, WAIT_FOR_READ, timeout)) + return -1; if (info && info->reader) return info->reader (fd, buf, bufsize, info->ctx); else return sock_read (fd, buf, bufsize); } +/* The same as xread, but don't actually read the data, just copy it + instead. */ + +int +fd_peek (int fd, char *buf, int bufsize, double timeout) +{ + struct transport_info *info; + LAZY_RETRIEVE_INFO (info); + if (!poll_internal (fd, info, WAIT_FOR_READ, timeout)) + return -1; + if (info && info->peeker) + return info->peeker (fd, buf, bufsize, info->ctx); + else + return sock_peek (fd, buf, bufsize); +} + /* Write the entire contents of BUF to FD. If TIMEOUT is non-zero, the operation aborts if no data is received after that many seconds. If TIMEOUT is -1, the value of opt.timeout is used for @@ -860,26 +897,14 @@ fd_write (int fd, char *buf, int bufsize, double timeout) int res; struct transport_info *info; LAZY_RETRIEVE_INFO (info); - if (timeout == -1) - timeout = opt.read_timeout; /* `write' may write less than LEN bytes, thus the loop keeps trying it until all was written, or an error occurred. */ res = 0; while (bufsize > 0) { - if (timeout) - { - int test; - if (info && info->poller) - test = info->poller (fd, timeout, WAIT_FOR_WRITE, info->ctx); - else - test = sock_poll (fd, timeout, WAIT_FOR_WRITE); - if (test == 0) - errno = ETIMEDOUT; - if (test <= 0) - return -1; - } + if (!poll_internal (fd, info, WAIT_FOR_WRITE, timeout)) + return -1; if (info && info->writer) res = info->writer (fd, buf, bufsize, info->ctx); else diff --git a/src/connect.h b/src/connect.h index d77d8725..f0ae5c07 100644 --- a/src/connect.h +++ b/src/connect.h @@ -70,13 +70,14 @@ int socket_has_inet6 PARAMS ((void)); typedef int (*fd_reader_t) PARAMS ((int, char *, int, void *)); typedef int (*fd_writer_t) PARAMS ((int, char *, int, void *)); typedef int (*fd_poller_t) PARAMS ((int, double, int, void *)); +typedef int (*fd_peeker_t) PARAMS ((int, char *, int, void *)); typedef void (*fd_closer_t) PARAMS ((int, void *)); -void fd_register_transport PARAMS ((int, - fd_reader_t, fd_writer_t, - fd_poller_t, fd_closer_t, +void fd_register_transport PARAMS ((int, fd_reader_t, fd_writer_t, + fd_poller_t, fd_peeker_t, fd_closer_t, void *)); int fd_read PARAMS ((int, char *, int, double)); int fd_write PARAMS ((int, char *, int, double)); +int fd_peek PARAMS ((int, char *, int, double)); void fd_close PARAMS ((int)); #endif /* CONNECT_H */ diff --git a/src/ftp-basic.c b/src/ftp-basic.c index 644e215f..409bacf1 100644 --- a/src/ftp-basic.c +++ b/src/ftp-basic.c @@ -6,7 +6,7 @@ This file is part of GNU Wget. GNU Wget is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or -(at your option) any later version. + (at your option) any later version. GNU Wget is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of @@ -46,10 +46,10 @@ so, delete this exception statement from your version. */ #include "wget.h" #include "utils.h" -#include "rbuf.h" #include "connect.h" #include "host.h" #include "ftp.h" +#include "retr.h" char ftp_last_respline[128]; @@ -59,45 +59,32 @@ char ftp_last_respline[128]; line is 0-terminated. All the response lines but the last one are skipped. The last line is determined as described in RFC959. */ uerr_t -ftp_response (struct rbuf *rbuf, char **line) +ftp_response (int fd, char **ret_line) { - int i; - int bufsize = 40; - - *line = (char *)xmalloc (bufsize); - do + while (1) { - for (i = 0; 1; i++) - { - int res; - if (i > bufsize - 1) - *line = (char *)xrealloc (*line, (bufsize <<= 1)); - res = RBUF_READCHAR (rbuf, *line + i); - /* RES is number of bytes read. */ - if (res == 1) - { - if ((*line)[i] == '\n') - { - (*line)[i] = '\0'; - /* Get rid of \r. */ - if (i > 0 && (*line)[i - 1] == '\r') - (*line)[i - 1] = '\0'; - break; - } - } - else - return FTPRERR; - } + char *line = fd_read_line (fd); + if (!line) + return FTPRERR; if (opt.server_response) - logprintf (LOG_ALWAYS, "%s\n", *line); + logputs (LOG_NOTQUIET, line); else - DEBUGP (("%s\n", *line)); + DEBUGP (("%s", line)); + if (ISDIGIT (line[0]) && ISDIGIT (line[1]) && ISDIGIT (line[2]) + && line[3] == ' ') + { + char *p = line + strlen (line); + if (p > line && p[-1] == '\n') + *--p = '\0'; + if (p > line && p[-1] == '\r') + *--p = '\0'; + strncpy (ftp_last_respline, line, sizeof (ftp_last_respline)); + ftp_last_respline[sizeof (ftp_last_respline) - 1] = '\0'; + *ret_line = line; + return FTPOK; + } + xfree (line); } - while (!(i >= 3 && ISDIGIT (**line) && ISDIGIT ((*line)[1]) && - ISDIGIT ((*line)[2]) && (*line)[3] == ' ')); - strncpy (ftp_last_respline, *line, sizeof (ftp_last_respline)); - ftp_last_respline[sizeof (ftp_last_respline) - 1] = '\0'; - return FTPOK; } /* Returns the malloc-ed FTP request, ending with , printing @@ -126,14 +113,14 @@ ftp_request (const char *command, const char *value) /* Sends the USER and PASS commands to the server, to control connection socket csock. */ uerr_t -ftp_login (struct rbuf *rbuf, const char *acc, const char *pass) +ftp_login (int csock, const char *acc, const char *pass) { uerr_t err; char *request, *respline; int nwritten; /* Get greeting. */ - err = ftp_response (rbuf, &respline); + err = ftp_response (csock, &respline); if (err != FTPOK) { xfree (respline); @@ -147,7 +134,7 @@ ftp_login (struct rbuf *rbuf, const char *acc, const char *pass) xfree (respline); /* Send USER username. */ request = ftp_request ("USER", acc); - nwritten = fd_write (RBUF_FD (rbuf), request, strlen (request), -1); + nwritten = fd_write (csock, request, strlen (request), -1); if (nwritten < 0) { xfree (request); @@ -155,7 +142,7 @@ ftp_login (struct rbuf *rbuf, const char *acc, const char *pass) } xfree (request); /* Get appropriate response. */ - err = ftp_response (rbuf, &respline); + err = ftp_response (csock, &respline); if (err != FTPOK) { xfree (respline); @@ -214,7 +201,7 @@ ftp_login (struct rbuf *rbuf, const char *acc, const char *pass) xfree (respline); /* Send PASS password. */ request = ftp_request ("PASS", pass); - nwritten = fd_write (RBUF_FD (rbuf), request, strlen (request), -1); + nwritten = fd_write (csock, request, strlen (request), -1); if (nwritten < 0) { xfree (request); @@ -222,7 +209,7 @@ ftp_login (struct rbuf *rbuf, const char *acc, const char *pass) } xfree (request); /* Get appropriate response. */ - err = ftp_response (rbuf, &respline); + err = ftp_response (csock, &respline); if (err != FTPOK) { xfree (respline); @@ -260,7 +247,7 @@ ip_address_to_port_repr (const ip_address *addr, int port, char *buf, server. Use acceptport after RETR, to get the socket of data connection. */ uerr_t -ftp_port (struct rbuf *rbuf, int *local_sock) +ftp_port (int csock, int *local_sock) { uerr_t err; char *request, *respline; @@ -270,11 +257,8 @@ ftp_port (struct rbuf *rbuf, int *local_sock) /* Must contain the argument of PORT (of the form a,b,c,d,e,f). */ char bytes[6 * 4 + 1]; - assert (rbuf != NULL); - assert (rbuf_initialized_p (rbuf)); - /* Get the address of this side of the connection. */ - if (!socket_ip_address (RBUF_FD (rbuf), &addr, ENDPOINT_LOCAL)) + if (!socket_ip_address (csock, &addr, ENDPOINT_LOCAL)) return FTPSYSERR; assert (addr.type == IPV4_ADDRESS); @@ -292,7 +276,7 @@ ftp_port (struct rbuf *rbuf, int *local_sock) /* Send PORT request. */ request = ftp_request ("PORT", bytes); - nwritten = fd_write (RBUF_FD (rbuf), request, strlen (request), -1); + nwritten = fd_write (csock, request, strlen (request), -1); if (nwritten < 0) { xfree (request); @@ -302,7 +286,7 @@ ftp_port (struct rbuf *rbuf, int *local_sock) xfree (request); /* Get appropriate response. */ - err = ftp_response (rbuf, &respline); + err = ftp_response (csock, &respline); if (err != FTPOK) { xfree (respline); @@ -357,7 +341,7 @@ ip_address_to_lprt_repr (const ip_address *addr, int port, char *buf, server. Use acceptport after RETR, to get the socket of data connection. */ uerr_t -ftp_lprt (struct rbuf *rbuf, int *local_sock) +ftp_lprt (int csock, int *local_sock) { uerr_t err; char *request, *respline; @@ -367,11 +351,8 @@ ftp_lprt (struct rbuf *rbuf, int *local_sock) /* Must contain the argument of LPRT (of the form af,n,h1,h2,...,hn,p1,p2). */ char bytes[21 * 4 + 1]; - assert (rbuf != NULL); - assert (rbuf_initialized_p (rbuf)); - /* Get the address of this side of the connection. */ - if (!socket_ip_address (RBUF_FD (rbuf), &addr, ENDPOINT_LOCAL)) + if (!socket_ip_address (csock, &addr, ENDPOINT_LOCAL)) return FTPSYSERR; assert (addr.type == IPV4_ADDRESS || addr.type == IPV6_ADDRESS); @@ -389,7 +370,7 @@ ftp_lprt (struct rbuf *rbuf, int *local_sock) /* Send PORT request. */ request = ftp_request ("LPRT", bytes); - nwritten = fd_write (RBUF_FD (rbuf), request, strlen (request), -1); + nwritten = fd_write (csock, request, strlen (request), -1); if (nwritten < 0) { xfree (request); @@ -398,7 +379,7 @@ ftp_lprt (struct rbuf *rbuf, int *local_sock) } xfree (request); /* Get appropriate response. */ - err = ftp_response (rbuf, &respline); + err = ftp_response (csock, &respline); if (err != FTPOK) { xfree (respline); @@ -439,7 +420,7 @@ ip_address_to_eprt_repr (const ip_address *addr, int port, char *buf, server. Use acceptport after RETR, to get the socket of data connection. */ uerr_t -ftp_eprt (struct rbuf *rbuf, int *local_sock) +ftp_eprt (int csock, int *local_sock) { uerr_t err; char *request, *respline; @@ -451,11 +432,8 @@ ftp_eprt (struct rbuf *rbuf, int *local_sock) * 1 char for af (1-2) and 5 chars for port (0-65535) */ char bytes[4 + INET6_ADDRSTRLEN + 1 + 5 + 1]; - assert (rbuf != NULL); - assert (rbuf_initialized_p(rbuf)); - /* Get the address of this side of the connection. */ - if (!socket_ip_address (RBUF_FD (rbuf), &addr, ENDPOINT_LOCAL)) + if (!socket_ip_address (csock, &addr, ENDPOINT_LOCAL)) return FTPSYSERR; assert (addr.type == IPV4_ADDRESS || addr.type == IPV6_ADDRESS); @@ -473,7 +451,7 @@ ftp_eprt (struct rbuf *rbuf, int *local_sock) /* Send PORT request. */ request = ftp_request ("EPRT", bytes); - nwritten = fd_write (RBUF_FD (rbuf), request, strlen (request), -1); + nwritten = fd_write (csock, request, strlen (request), -1); if (nwritten < 0) { xfree (request); @@ -482,7 +460,7 @@ ftp_eprt (struct rbuf *rbuf, int *local_sock) } xfree (request); /* Get appropriate response. */ - err = ftp_response (rbuf, &respline); + err = ftp_response (csock, &respline); if (err != FTPOK) { xfree (respline); @@ -504,15 +482,13 @@ ftp_eprt (struct rbuf *rbuf, int *local_sock) transfer. Reads the response from server and parses it. Reads the host and port addresses and returns them. */ uerr_t -ftp_pasv (struct rbuf *rbuf, ip_address *addr, int *port) +ftp_pasv (int csock, ip_address *addr, int *port) { char *request, *respline, *s; int nwritten, i; uerr_t err; unsigned char tmp[6]; - assert (rbuf != NULL); - assert (rbuf_initialized_p (rbuf)); assert (addr != NULL); assert (port != NULL); @@ -521,7 +497,7 @@ ftp_pasv (struct rbuf *rbuf, ip_address *addr, int *port) /* Form the request. */ request = ftp_request ("PASV", NULL); /* And send it. */ - nwritten = fd_write (RBUF_FD (rbuf), request, strlen (request), -1); + nwritten = fd_write (csock, request, strlen (request), -1); if (nwritten < 0) { xfree (request); @@ -529,7 +505,7 @@ ftp_pasv (struct rbuf *rbuf, ip_address *addr, int *port) } xfree (request); /* Get the server response. */ - err = ftp_response (rbuf, &respline); + err = ftp_response (csock, &respline); if (err != FTPOK) { xfree (respline); @@ -573,7 +549,7 @@ ftp_pasv (struct rbuf *rbuf, ip_address *addr, int *port) transfer. Reads the response from server and parses it. Reads the host and port addresses and returns them. */ uerr_t -ftp_lpsv (struct rbuf *rbuf, ip_address *addr, int *port) +ftp_lpsv (int csock, ip_address *addr, int *port) { char *request, *respline, *s; int nwritten, i, af, addrlen, portlen; @@ -581,8 +557,6 @@ ftp_lpsv (struct rbuf *rbuf, ip_address *addr, int *port) unsigned char tmp[16]; unsigned char tmpprt[2]; - assert (rbuf != NULL); - assert (rbuf_initialized_p(rbuf)); assert (addr != NULL); assert (port != NULL); @@ -592,7 +566,7 @@ ftp_lpsv (struct rbuf *rbuf, ip_address *addr, int *port) request = ftp_request ("LPSV", NULL); /* And send it. */ - nwritten = fd_write (RBUF_FD (rbuf), request, strlen (request), -1); + nwritten = fd_write (csock, request, strlen (request), -1); if (nwritten < 0) { xfree (request); @@ -601,7 +575,7 @@ ftp_lpsv (struct rbuf *rbuf, ip_address *addr, int *port) xfree (request); /* Get the server response. */ - err = ftp_response (rbuf, &respline); + err = ftp_response (csock, &respline); if (err != FTPOK) { xfree (respline); @@ -739,15 +713,13 @@ ftp_lpsv (struct rbuf *rbuf, ip_address *addr, int *port) transfer. Reads the response from server and parses it. Reads the host and port addresses and returns them. */ uerr_t -ftp_epsv (struct rbuf *rbuf, ip_address *ip, int *port) +ftp_epsv (int csock, ip_address *ip, int *port) { char *request, *respline, *start, delim, *s; int nwritten, i; uerr_t err; int tport; - assert (rbuf != NULL); - assert (rbuf_initialized_p(rbuf)); assert (ip != NULL); assert (port != NULL); @@ -759,7 +731,7 @@ ftp_epsv (struct rbuf *rbuf, ip_address *ip, int *port) request = ftp_request ("EPSV", (ip->type == IPV4_ADDRESS ? "1" : "2")); /* And send it. */ - nwritten = fd_write (RBUF_FD (rbuf), request, strlen (request), -1); + nwritten = fd_write (csock, request, strlen (request), -1); if (nwritten < 0) { xfree (request); @@ -768,7 +740,7 @@ ftp_epsv (struct rbuf *rbuf, ip_address *ip, int *port) xfree (request); /* Get the server response. */ - err = ftp_response (rbuf, &respline); + err = ftp_response (csock, &respline); if (err != FTPOK) { xfree (respline); @@ -847,7 +819,7 @@ ftp_epsv (struct rbuf *rbuf, ip_address *ip, int *port) /* Sends the TYPE request to the server. */ uerr_t -ftp_type (struct rbuf *rbuf, int type) +ftp_type (int csock, int type) { char *request, *respline; int nwritten; @@ -859,7 +831,7 @@ ftp_type (struct rbuf *rbuf, int type) stype[1] = 0; /* Send TYPE request. */ request = ftp_request ("TYPE", stype); - nwritten = fd_write (RBUF_FD (rbuf), request, strlen (request), -1); + nwritten = fd_write (csock, request, strlen (request), -1); if (nwritten < 0) { xfree (request); @@ -867,7 +839,7 @@ ftp_type (struct rbuf *rbuf, int type) } xfree (request); /* Get appropriate response. */ - err = ftp_response (rbuf, &respline); + err = ftp_response (csock, &respline); if (err != FTPOK) { xfree (respline); @@ -886,7 +858,7 @@ ftp_type (struct rbuf *rbuf, int type) /* Changes the working directory by issuing a CWD command to the server. */ uerr_t -ftp_cwd (struct rbuf *rbuf, const char *dir) +ftp_cwd (int csock, const char *dir) { char *request, *respline; int nwritten; @@ -894,7 +866,7 @@ ftp_cwd (struct rbuf *rbuf, const char *dir) /* Send CWD request. */ request = ftp_request ("CWD", dir); - nwritten = fd_write (RBUF_FD (rbuf), request, strlen (request), -1); + nwritten = fd_write (csock, request, strlen (request), -1); if (nwritten < 0) { xfree (request); @@ -902,7 +874,7 @@ ftp_cwd (struct rbuf *rbuf, const char *dir) } xfree (request); /* Get appropriate response. */ - err = ftp_response (rbuf, &respline); + err = ftp_response (csock, &respline); if (err != FTPOK) { xfree (respline); @@ -925,7 +897,7 @@ ftp_cwd (struct rbuf *rbuf, const char *dir) /* Sends REST command to the FTP server. */ uerr_t -ftp_rest (struct rbuf *rbuf, long offset) +ftp_rest (int csock, long offset) { char *request, *respline; int nwritten; @@ -934,7 +906,7 @@ ftp_rest (struct rbuf *rbuf, long offset) number_to_string (numbuf, offset); request = ftp_request ("REST", numbuf); - nwritten = fd_write (RBUF_FD (rbuf), request, strlen (request), -1); + nwritten = fd_write (csock, request, strlen (request), -1); if (nwritten < 0) { xfree (request); @@ -942,7 +914,7 @@ ftp_rest (struct rbuf *rbuf, long offset) } xfree (request); /* Get appropriate response. */ - err = ftp_response (rbuf, &respline); + err = ftp_response (csock, &respline); if (err != FTPOK) { xfree (respline); @@ -960,7 +932,7 @@ ftp_rest (struct rbuf *rbuf, long offset) /* Sends RETR command to the FTP server. */ uerr_t -ftp_retr (struct rbuf *rbuf, const char *file) +ftp_retr (int csock, const char *file) { char *request, *respline; int nwritten; @@ -968,7 +940,7 @@ ftp_retr (struct rbuf *rbuf, const char *file) /* Send RETR request. */ request = ftp_request ("RETR", file); - nwritten = fd_write (RBUF_FD (rbuf), request, strlen (request), -1); + nwritten = fd_write (csock, request, strlen (request), -1); if (nwritten < 0) { xfree (request); @@ -976,7 +948,7 @@ ftp_retr (struct rbuf *rbuf, const char *file) } xfree (request); /* Get appropriate response. */ - err = ftp_response (rbuf, &respline); + err = ftp_response (csock, &respline); if (err != FTPOK) { xfree (respline); @@ -1000,7 +972,7 @@ ftp_retr (struct rbuf *rbuf, const char *file) /* Sends the LIST command to the server. If FILE is NULL, send just `LIST' (no space). */ uerr_t -ftp_list (struct rbuf *rbuf, const char *file) +ftp_list (int csock, const char *file) { char *request, *respline; int nwritten; @@ -1008,7 +980,7 @@ ftp_list (struct rbuf *rbuf, const char *file) /* Send LIST request. */ request = ftp_request ("LIST", file); - nwritten = fd_write (RBUF_FD (rbuf), request, strlen (request), -1); + nwritten = fd_write (csock, request, strlen (request), -1); if (nwritten < 0) { xfree (request); @@ -1016,7 +988,7 @@ ftp_list (struct rbuf *rbuf, const char *file) } xfree (request); /* Get appropriate respone. */ - err = ftp_response (rbuf, &respline); + err = ftp_response (csock, &respline); if (err != FTPOK) { xfree (respline); @@ -1039,7 +1011,7 @@ ftp_list (struct rbuf *rbuf, const char *file) /* Sends the SYST command to the server. */ uerr_t -ftp_syst (struct rbuf *rbuf, enum stype *server_type) +ftp_syst (int csock, enum stype *server_type) { char *request, *respline; int nwritten; @@ -1047,7 +1019,7 @@ ftp_syst (struct rbuf *rbuf, enum stype *server_type) /* Send SYST request. */ request = ftp_request ("SYST", NULL); - nwritten = fd_write (RBUF_FD (rbuf), request, strlen (request), -1); + nwritten = fd_write (csock, request, strlen (request), -1); if (nwritten < 0) { xfree (request); @@ -1056,7 +1028,7 @@ ftp_syst (struct rbuf *rbuf, enum stype *server_type) xfree (request); /* Get appropriate response. */ - err = ftp_response (rbuf, &respline); + err = ftp_response (csock, &respline); if (err != FTPOK) { xfree (respline); @@ -1096,7 +1068,7 @@ ftp_syst (struct rbuf *rbuf, enum stype *server_type) /* Sends the PWD command to the server. */ uerr_t -ftp_pwd (struct rbuf *rbuf, char **pwd) +ftp_pwd (int csock, char **pwd) { char *request, *respline; int nwritten; @@ -1104,7 +1076,7 @@ ftp_pwd (struct rbuf *rbuf, char **pwd) /* Send PWD request. */ request = ftp_request ("PWD", NULL); - nwritten = fd_write (RBUF_FD (rbuf), request, strlen (request), -1); + nwritten = fd_write (csock, request, strlen (request), -1); if (nwritten < 0) { xfree (request); @@ -1112,7 +1084,7 @@ ftp_pwd (struct rbuf *rbuf, char **pwd) } xfree (request); /* Get appropriate response. */ - err = ftp_response (rbuf, &respline); + err = ftp_response (csock, &respline); if (err != FTPOK) { xfree (respline); @@ -1142,7 +1114,7 @@ ftp_pwd (struct rbuf *rbuf, char **pwd) /* Sends the SIZE command to the server, and returns the value in 'size'. * If an error occurs, size is set to zero. */ uerr_t -ftp_size (struct rbuf *rbuf, const char *file, long int *size) +ftp_size (int csock, const char *file, long int *size) { char *request, *respline; int nwritten; @@ -1150,7 +1122,7 @@ ftp_size (struct rbuf *rbuf, const char *file, long int *size) /* Send PWD request. */ request = ftp_request ("SIZE", file); - nwritten = fd_write (RBUF_FD (rbuf), request, strlen (request), -1); + nwritten = fd_write (csock, request, strlen (request), -1); if (nwritten < 0) { xfree (request); @@ -1159,7 +1131,7 @@ ftp_size (struct rbuf *rbuf, const char *file, long int *size) } xfree (request); /* Get appropriate response. */ - err = ftp_response (rbuf, &respline); + err = ftp_response (csock, &respline); if (err != FTPOK) { xfree (respline); diff --git a/src/ftp.c b/src/ftp.c index 873556aa..fdac6d49 100644 --- a/src/ftp.c +++ b/src/ftp.c @@ -47,7 +47,6 @@ so, delete this exception statement from your version. */ #include "wget.h" #include "utils.h" #include "url.h" -#include "rbuf.h" #include "retr.h" #include "ftp.h" #include "connect.h" @@ -71,7 +70,7 @@ typedef struct { int st; /* connection status */ int cmd; /* command code */ - struct rbuf rbuf; /* control connection buffer */ + int csock; /* control connection socket */ double dltime; /* time of the download in msecs */ enum stype rs; /* remote system reported by ftp server */ char *id; /* initial directory */ @@ -128,14 +127,14 @@ ftp_expected_bytes (const char *s) * It is merely a wrapper around ftp_epsv, ftp_lpsv and ftp_pasv. */ static uerr_t -ftp_do_pasv (struct rbuf *rbuf, ip_address *addr, int *port) +ftp_do_pasv (int csock, ip_address *addr, int *port) { uerr_t err; /* We need to determine the address family and need to call getpeername, so while we're at it, store the address to ADDR. ftp_pasv and ftp_lpsv can simply override it. */ - if (!socket_ip_address (RBUF_FD (rbuf), addr, ENDPOINT_PEER)) + if (!socket_ip_address (csock, addr, ENDPOINT_PEER)) abort (); /* If our control connection is over IPv6, then we first try EPSV and then @@ -146,19 +145,19 @@ ftp_do_pasv (struct rbuf *rbuf, ip_address *addr, int *port) case IPV4_ADDRESS: if (!opt.server_response) logputs (LOG_VERBOSE, "==> PASV ... "); - err = ftp_pasv (rbuf, addr, port); + err = ftp_pasv (csock, addr, port); break; case IPV6_ADDRESS: if (!opt.server_response) logputs (LOG_VERBOSE, "==> EPSV ... "); - err = ftp_epsv (rbuf, addr, port); + err = ftp_epsv (csock, addr, port); /* If EPSV is not supported try LPSV */ if (err == FTPNOPASV) { if (!opt.server_response) logputs (LOG_VERBOSE, "==> LPSV ... "); - err = ftp_lpsv (rbuf, addr, port); + err = ftp_lpsv (csock, addr, port); } break; default: @@ -173,15 +172,12 @@ ftp_do_pasv (struct rbuf *rbuf, ip_address *addr, int *port) * It is merely a wrapper around ftp_eprt, ftp_lprt and ftp_port. */ static uerr_t -ftp_do_port (struct rbuf *rbuf, int *local_sock) +ftp_do_port (int csock, int *local_sock) { uerr_t err; ip_address cip; - assert (rbuf != NULL); - assert (rbuf_initialized_p (rbuf)); - - if (!socket_ip_address (RBUF_FD (rbuf), &cip, ENDPOINT_PEER)) + if (!socket_ip_address (csock, &cip, ENDPOINT_PEER)) abort (); /* If our control connection is over IPv6, then we first try EPRT and then @@ -192,19 +188,19 @@ ftp_do_port (struct rbuf *rbuf, int *local_sock) case IPV4_ADDRESS: if (!opt.server_response) logputs (LOG_VERBOSE, "==> PORT ... "); - err = ftp_port (rbuf, local_sock); + err = ftp_port (csock, local_sock); break; case IPV6_ADDRESS: if (!opt.server_response) logputs (LOG_VERBOSE, "==> EPRT ... "); - err = ftp_eprt (rbuf, local_sock); + err = ftp_eprt (csock, local_sock); /* If EPRT is not supported try LPRT */ if (err == FTPPORTERR) { if (!opt.server_response) logputs (LOG_VERBOSE, "==> LPRT ... "); - err = ftp_lprt (rbuf, local_sock); + err = ftp_lprt (csock, local_sock); } break; default: @@ -215,19 +211,19 @@ ftp_do_port (struct rbuf *rbuf, int *local_sock) #else static uerr_t -ftp_do_pasv (struct rbuf *rbuf, ip_address *addr, int *port) +ftp_do_pasv (int csock, ip_address *addr, int *port) { if (!opt.server_response) logputs (LOG_VERBOSE, "==> PASV ... "); - return ftp_pasv (rbuf, addr, port); + return ftp_pasv (csock, addr, port); } static uerr_t -ftp_do_port (struct rbuf *rbuf, int *local_sock) +ftp_do_port (int csock, int *local_sock) { if (!opt.server_response) logputs (LOG_VERBOSE, "==> PORT ... "); - return ftp_port (rbuf, local_sock); + return ftp_port (csock, local_sock); } #endif @@ -268,7 +264,7 @@ getftp (struct url *u, long *len, long restval, ccon *con) con->dltime = 0; if (!(cmd & DO_LOGIN)) - csock = RBUF_FD (&con->rbuf); + csock = con->csock; else /* cmd & DO_LOGIN */ { char type_char; @@ -294,20 +290,11 @@ getftp (struct url *u, long *len, long restval, ccon *con) return (retryable_socket_connect_error (errno) ? CONERROR : CONIMPOSSIBLE); - if (cmd & LEAVE_PENDING) - rbuf_initialize (&con->rbuf, csock); - else - rbuf_uninitialize (&con->rbuf); - - /* Since this is a new connection, we may safely discard - anything left in the buffer. */ - rbuf_discard (&con->rbuf); - /* Second: Login with proper USER/PASS sequence. */ logprintf (LOG_VERBOSE, _("Logging in as %s ... "), user); if (opt.server_response) logputs (LOG_ALWAYS, "\n"); - err = ftp_login (&con->rbuf, logname, passwd); + err = ftp_login (csock, logname, passwd); if (con->proxy) xfree (logname); @@ -320,14 +307,12 @@ getftp (struct url *u, long *len, long restval, ccon *con) logputs (LOG_NOTQUIET, _("\ Error in server response, closing control connection.\n")); fd_close (csock); - rbuf_uninitialize (&con->rbuf); return err; break; case FTPSRVERR: logputs (LOG_VERBOSE, "\n"); logputs (LOG_NOTQUIET, _("Error in server greeting.\n")); fd_close (csock); - rbuf_uninitialize (&con->rbuf); return err; break; case WRITEFAILED: @@ -335,21 +320,18 @@ Error in server response, closing control connection.\n")); logputs (LOG_NOTQUIET, _("Write failed, closing control connection.\n")); fd_close (csock); - rbuf_uninitialize (&con->rbuf); return err; break; case FTPLOGREFUSED: logputs (LOG_VERBOSE, "\n"); logputs (LOG_NOTQUIET, _("The server refuses login.\n")); fd_close (csock); - rbuf_uninitialize (&con->rbuf); return FTPLOGREFUSED; break; case FTPLOGINC: logputs (LOG_VERBOSE, "\n"); logputs (LOG_NOTQUIET, _("Login incorrect.\n")); fd_close (csock); - rbuf_uninitialize (&con->rbuf); return FTPLOGINC; break; case FTPOK: @@ -364,7 +346,7 @@ Error in server response, closing control connection.\n")); /* Third: Get the system type */ if (!opt.server_response) logprintf (LOG_VERBOSE, "==> SYST ... "); - err = ftp_syst (&con->rbuf, &con->rs); + err = ftp_syst (csock, &con->rs); /* FTPRERR */ switch (err) { @@ -373,7 +355,6 @@ Error in server response, closing control connection.\n")); logputs (LOG_NOTQUIET, _("\ Error in server response, closing control connection.\n")); fd_close (csock); - rbuf_uninitialize (&con->rbuf); return err; break; case FTPSRVERR: @@ -395,7 +376,7 @@ Error in server response, closing control connection.\n")); if (!opt.server_response) logprintf (LOG_VERBOSE, "==> PWD ... "); - err = ftp_pwd(&con->rbuf, &con->id); + err = ftp_pwd(csock, &con->id); /* FTPRERR */ switch (err) { @@ -404,7 +385,6 @@ Error in server response, closing control connection.\n")); logputs (LOG_NOTQUIET, _("\ Error in server response, closing control connection.\n")); fd_close (csock); - rbuf_uninitialize (&con->rbuf); return err; break; case FTPSRVERR : @@ -449,7 +429,7 @@ Error in server response, closing control connection.\n")); type_char = ftp_process_type (u->params); if (!opt.server_response) logprintf (LOG_VERBOSE, "==> TYPE %c ... ", type_char); - err = ftp_type (&con->rbuf, type_char); + err = ftp_type (csock, type_char); /* FTPRERR, WRITEFAILED, FTPUNKNOWNTYPE */ switch (err) { @@ -458,7 +438,6 @@ Error in server response, closing control connection.\n")); logputs (LOG_NOTQUIET, _("\ Error in server response, closing control connection.\n")); fd_close (csock); - rbuf_uninitialize (&con->rbuf); return err; break; case WRITEFAILED: @@ -466,7 +445,6 @@ Error in server response, closing control connection.\n")); logputs (LOG_NOTQUIET, _("Write failed, closing control connection.\n")); fd_close (csock); - rbuf_uninitialize (&con->rbuf); return err; break; case FTPUNKNOWNTYPE: @@ -475,7 +453,6 @@ Error in server response, closing control connection.\n")); _("Unknown type `%c', closing control connection.\n"), type_char); fd_close (csock); - rbuf_uninitialize (&con->rbuf); return err; case FTPOK: /* Everything is OK. */ @@ -564,7 +541,7 @@ Error in server response, closing control connection.\n")); if (!opt.server_response) logprintf (LOG_VERBOSE, "==> CWD %s ... ", target); - err = ftp_cwd (&con->rbuf, target); + err = ftp_cwd (csock, target); /* FTPRERR, WRITEFAILED, FTPNSFOD */ switch (err) { @@ -573,7 +550,6 @@ Error in server response, closing control connection.\n")); logputs (LOG_NOTQUIET, _("\ Error in server response, closing control connection.\n")); fd_close (csock); - rbuf_uninitialize (&con->rbuf); return err; break; case WRITEFAILED: @@ -581,7 +557,6 @@ Error in server response, closing control connection.\n")); logputs (LOG_NOTQUIET, _("Write failed, closing control connection.\n")); fd_close (csock); - rbuf_uninitialize (&con->rbuf); return err; break; case FTPNSFOD: @@ -589,7 +564,6 @@ Error in server response, closing control connection.\n")); logprintf (LOG_NOTQUIET, _("No such directory `%s'.\n\n"), u->dir); fd_close (csock); - rbuf_uninitialize (&con->rbuf); return err; break; case FTPOK: @@ -614,7 +588,7 @@ Error in server response, closing control connection.\n")); logprintf (LOG_VERBOSE, "==> SIZE %s ... ", u->file); } - err = ftp_size(&con->rbuf, u->file, len); + err = ftp_size(csock, u->file, len); /* FTPRERR */ switch (err) { @@ -624,7 +598,6 @@ Error in server response, closing control connection.\n")); logputs (LOG_NOTQUIET, _("\ Error in server response, closing control connection.\n")); fd_close (csock); - rbuf_uninitialize (&con->rbuf); return err; break; case FTPOK: @@ -645,7 +618,7 @@ Error in server response, closing control connection.\n")); { ip_address passive_addr; int passive_port; - err = ftp_do_pasv (&con->rbuf, &passive_addr, &passive_port); + err = ftp_do_pasv (csock, &passive_addr, &passive_port); /* FTPRERR, WRITEFAILED, FTPNOPASV, FTPINVPASV */ switch (err) { @@ -654,7 +627,6 @@ Error in server response, closing control connection.\n")); logputs (LOG_NOTQUIET, _("\ Error in server response, closing control connection.\n")); fd_close (csock); - rbuf_uninitialize (&con->rbuf); return err; break; case WRITEFAILED: @@ -662,7 +634,6 @@ Error in server response, closing control connection.\n")); logputs (LOG_NOTQUIET, _("Write failed, closing control connection.\n")); fd_close (csock); - rbuf_uninitialize (&con->rbuf); return err; break; case FTPNOPASV: @@ -690,7 +661,6 @@ Error in server response, closing control connection.\n")); { int save_errno = errno; fd_close (csock); - rbuf_uninitialize (&con->rbuf); logprintf (LOG_VERBOSE, _("couldn't connect to %s port %hu: %s\n"), pretty_print_address (&passive_addr), passive_port, strerror (save_errno)); @@ -706,7 +676,7 @@ Error in server response, closing control connection.\n")); if (!pasv_mode_open) /* Try to use a port command if PASV failed */ { - err = ftp_do_port (&con->rbuf, &local_sock); + err = ftp_do_port (csock, &local_sock); /* FTPRERR, WRITEFAILED, bindport (FTPSYSERR), HOSTERR, FTPPORTERR */ switch (err) @@ -718,7 +688,6 @@ Error in server response, closing control connection.\n")); fd_close (csock); fd_close (dtsock); fd_close (local_sock); - rbuf_uninitialize (&con->rbuf); return err; break; case WRITEFAILED: @@ -728,7 +697,6 @@ Error in server response, closing control connection.\n")); fd_close (csock); fd_close (dtsock); fd_close (local_sock); - rbuf_uninitialize (&con->rbuf); return err; break; case CONSOCKERR: @@ -737,7 +705,6 @@ Error in server response, closing control connection.\n")); fd_close (csock); fd_close (dtsock); fd_close (local_sock); - rbuf_uninitialize (&con->rbuf); return err; break; case FTPSYSERR: @@ -753,7 +720,6 @@ Error in server response, closing control connection.\n")); fd_close (csock); fd_close (dtsock); fd_close (local_sock); - rbuf_uninitialize (&con->rbuf); return err; break; case FTPOK: @@ -773,7 +739,7 @@ Error in server response, closing control connection.\n")); { if (!opt.server_response) logprintf (LOG_VERBOSE, "==> REST %ld ... ", restval); - err = ftp_rest (&con->rbuf, restval); + err = ftp_rest (csock, restval); /* FTPRERR, WRITEFAILED, FTPRESTFAIL */ switch (err) @@ -785,7 +751,6 @@ Error in server response, closing control connection.\n")); fd_close (csock); fd_close (dtsock); fd_close (local_sock); - rbuf_uninitialize (&con->rbuf); return err; break; case WRITEFAILED: @@ -795,7 +760,6 @@ Error in server response, closing control connection.\n")); fd_close (csock); fd_close (dtsock); fd_close (local_sock); - rbuf_uninitialize (&con->rbuf); return err; break; case FTPRESTFAIL: @@ -810,7 +774,6 @@ Error in server response, closing control connection.\n")); fd_close (csock); fd_close (dtsock); fd_close (local_sock); - rbuf_uninitialize (&con->rbuf); return CONTNOTSUPPORTED; } logputs (LOG_VERBOSE, _("\nREST failed, starting from scratch.\n")); @@ -838,7 +801,6 @@ Error in server response, closing control connection.\n")); fd_close (csock); fd_close (dtsock); fd_close (local_sock); - rbuf_uninitialize (&con->rbuf); return RETRFINISHED; } @@ -852,7 +814,7 @@ Error in server response, closing control connection.\n")); } } - err = ftp_retr (&con->rbuf, u->file); + err = ftp_retr (csock, u->file); /* FTPRERR, WRITEFAILED, FTPNSFOD */ switch (err) { @@ -863,7 +825,6 @@ Error in server response, closing control connection.\n")); fd_close (csock); fd_close (dtsock); fd_close (local_sock); - rbuf_uninitialize (&con->rbuf); return err; break; case WRITEFAILED: @@ -873,7 +834,6 @@ Error in server response, closing control connection.\n")); fd_close (csock); fd_close (dtsock); fd_close (local_sock); - rbuf_uninitialize (&con->rbuf); return err; break; case FTPNSFOD: @@ -903,7 +863,7 @@ Error in server response, closing control connection.\n")); /* As Maciej W. Rozycki (macro@ds2.pg.gda.pl) says, `LIST' without arguments is better than `LIST .'; confirmed by RFC959. */ - err = ftp_list (&con->rbuf, NULL); + err = ftp_list (csock, NULL); /* FTPRERR, WRITEFAILED */ switch (err) { @@ -914,7 +874,6 @@ Error in server response, closing control connection.\n")); fd_close (csock); fd_close (dtsock); fd_close (local_sock); - rbuf_uninitialize (&con->rbuf); return err; break; case WRITEFAILED: @@ -924,7 +883,6 @@ Error in server response, closing control connection.\n")); fd_close (csock); fd_close (dtsock); fd_close (local_sock); - rbuf_uninitialize (&con->rbuf); return err; break; case FTPNSFOD: @@ -987,7 +945,6 @@ Error in server response, closing control connection.\n")); { logprintf (LOG_NOTQUIET, "%s: %s\n", con->target, strerror (errno)); fd_close (csock); - rbuf_uninitialize (&con->rbuf); fd_close (dtsock); fd_close (local_sock); return FOPENERR; @@ -1031,8 +988,8 @@ Error in server response, closing control connection.\n")); } /* Get the contents of the document. */ - res = get_contents (dtsock, fp, len, restval, expected_bytes, &con->rbuf, - 0, &con->dltime); + res = fd_read_body (dtsock, fp, len, restval, expected_bytes, 0, + &con->dltime); tms = time_str (NULL); tmrate = retr_rate (*len - restval, con->dltime, 0); /* Close data connection socket. */ @@ -1058,7 +1015,6 @@ Error in server response, closing control connection.\n")); logprintf (LOG_NOTQUIET, _("%s: %s, closing control connection.\n"), con->target, strerror (errno)); fd_close (csock); - rbuf_uninitialize (&con->rbuf); return FWRITEERR; } else if (res == -1) @@ -1070,9 +1026,7 @@ Error in server response, closing control connection.\n")); } /* Get the server to tell us if everything is retrieved. */ - err = ftp_response (&con->rbuf, &respline); - /* ...and empty the buffer. */ - rbuf_discard (&con->rbuf); + err = ftp_response (csock, &respline); if (err != FTPOK) { xfree (respline); @@ -1086,7 +1040,6 @@ Error in server response, closing control connection.\n")); whole file was retrieved nevertheless (but that is for ftp_loop_internal to decide). */ fd_close (csock); - rbuf_uninitialize (&con->rbuf); return FTPRETRINT; } /* err != FTPOK */ /* If retrieval failed for any reason, return FTPRETRINT, but do not @@ -1115,7 +1068,6 @@ Error in server response, closing control connection.\n")); /* I should probably send 'QUIT' and check for a reply, but this is faster. #### Is it OK, though? */ fd_close (csock); - rbuf_uninitialize (&con->rbuf); } /* If it was a listing, and opt.server_response is true, print it out. */ @@ -1192,14 +1144,14 @@ ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con) { con->cmd = 0; con->cmd |= (DO_RETR | LEAVE_PENDING); - if (rbuf_initialized_p (&con->rbuf)) + if (con->csock != -1) con->cmd &= ~ (DO_LOGIN | DO_CWD); else con->cmd |= (DO_LOGIN | DO_CWD); } else /* not on your own */ { - if (rbuf_initialized_p (&con->rbuf)) + if (con->csock != -1) con->cmd &= ~DO_LOGIN; else con->cmd |= DO_LOGIN; @@ -1248,7 +1200,7 @@ ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con) len = 0; err = getftp (u, &len, restval, con); - if (!rbuf_initialized_p (&con->rbuf)) + if (con->csock != -1) con->st &= ~DONE_CWD; else con->st |= DONE_CWD; @@ -1294,8 +1246,8 @@ ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con) if (con->st & ON_YOUR_OWN) { - fd_close (RBUF_FD (&con->rbuf)); - rbuf_uninitialize (&con->rbuf); + fd_close (con->csock); + con->csock = -1; } if (!opt.spider) logprintf (LOG_VERBOSE, _("%s (%s) - `%s' saved [%ld]\n\n"), @@ -1354,10 +1306,10 @@ ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con) return RETROK; } while (!opt.ntry || (count < opt.ntry)); - if (rbuf_initialized_p (&con->rbuf) && (con->st & ON_YOUR_OWN)) + if (con->csock != -1 && (con->st & ON_YOUR_OWN)) { - fd_close (RBUF_FD (&con->rbuf)); - rbuf_uninitialize (&con->rbuf); + fd_close (con->csock); + con->csock = -1; } return TRYLIMEXC; } @@ -1448,7 +1400,7 @@ ftp_retrieve_list (struct url *u, struct fileinfo *f, ccon *con) con->cmd &= ~DO_CWD; con->cmd |= (DO_RETR | LEAVE_PENDING); - if (!rbuf_initialized_p (&con->rbuf)) + if (con->csock < 0) con->cmd |= DO_LOGIN; else con->cmd &= ~DO_LOGIN; @@ -1849,7 +1801,7 @@ ftp_loop (struct url *u, int *dt, struct url *proxy) memset (&con, 0, sizeof (con)); - rbuf_uninitialize (&con.rbuf); + con.csock = -1; con.st = ON_YOUR_OWN; con.rs = ST_UNIX; con.id = NULL; @@ -1916,8 +1868,8 @@ ftp_loop (struct url *u, int *dt, struct url *proxy) if (res == RETROK) *dt |= RETROKF; /* If a connection was left, quench it. */ - if (rbuf_initialized_p (&con.rbuf)) - fd_close (RBUF_FD (&con.rbuf)); + if (con.csock != -1) + fd_close (con.csock); xfree_null (con.id); con.id = NULL; xfree_null (con.target); diff --git a/src/ftp.h b/src/ftp.h index f7d79429..aaaff474 100644 --- a/src/ftp.h +++ b/src/ftp.h @@ -30,9 +30,6 @@ so, delete this exception statement from your version. */ #ifndef FTP_H #define FTP_H -/* Need it for struct rbuf. */ -#include "rbuf.h" - #include "host.h" /* System types. */ @@ -46,24 +43,24 @@ enum stype ST_OTHER }; -uerr_t ftp_response PARAMS ((struct rbuf *, char **)); -uerr_t ftp_login PARAMS ((struct rbuf *, const char *, const char *)); -uerr_t ftp_port PARAMS ((struct rbuf *, int *)); -uerr_t ftp_pasv PARAMS ((struct rbuf *, ip_address *, int *)); +uerr_t ftp_response PARAMS ((int, char **)); +uerr_t ftp_login PARAMS ((int, const char *, const char *)); +uerr_t ftp_port PARAMS ((int, int *)); +uerr_t ftp_pasv PARAMS ((int, ip_address *, int *)); #ifdef ENABLE_IPV6 -uerr_t ftp_lprt PARAMS ((struct rbuf *, int *)); -uerr_t ftp_lpsv PARAMS ((struct rbuf *, ip_address *, int *)); -uerr_t ftp_eprt PARAMS ((struct rbuf *, int *)); -uerr_t ftp_epsv PARAMS ((struct rbuf *, ip_address *, int *)); +uerr_t ftp_lprt PARAMS ((int, int *)); +uerr_t ftp_lpsv PARAMS ((int, ip_address *, int *)); +uerr_t ftp_eprt PARAMS ((int, int *)); +uerr_t ftp_epsv PARAMS ((int, ip_address *, int *)); #endif -uerr_t ftp_type PARAMS ((struct rbuf *, int)); -uerr_t ftp_cwd PARAMS ((struct rbuf *, const char *)); -uerr_t ftp_retr PARAMS ((struct rbuf *, const char *)); -uerr_t ftp_rest PARAMS ((struct rbuf *, long)); -uerr_t ftp_list PARAMS ((struct rbuf *, const char *)); -uerr_t ftp_syst PARAMS ((struct rbuf *, enum stype *)); -uerr_t ftp_pwd PARAMS ((struct rbuf *, char **)); -uerr_t ftp_size PARAMS ((struct rbuf *, const char *, long int *)); +uerr_t ftp_type PARAMS ((int, int)); +uerr_t ftp_cwd PARAMS ((int, const char *)); +uerr_t ftp_retr PARAMS ((int, const char *)); +uerr_t ftp_rest PARAMS ((int, long)); +uerr_t ftp_list PARAMS ((int, const char *)); +uerr_t ftp_syst PARAMS ((int, enum stype *)); +uerr_t ftp_pwd PARAMS ((int, char **)); +uerr_t ftp_size PARAMS ((int, const char *, long int *)); #ifdef USE_OPIE const char *skey_response PARAMS ((int, const char *, const char *)); diff --git a/src/gen_sslfunc.c b/src/gen_sslfunc.c index 4cbab6ac..98e75a1d 100644 --- a/src/gen_sslfunc.c +++ b/src/gen_sslfunc.c @@ -300,6 +300,19 @@ ssl_poll (int fd, double timeout, int wait_for, void *ctx) return select_fd (fd, timeout, wait_for); } +static int +ssl_peek (int fd, char *buf, int bufsize, void *ctx) +{ + int ret; + SSL *ssl = (SSL *) ctx; + do + ret = SSL_peek (ssl, buf, bufsize); + while (ret == -1 + && SSL_get_error (ssl, ret) == SSL_ERROR_SYSCALL + && errno == EINTR); + return ret; +} + static void ssl_close (int fd, void *ctx) { @@ -335,9 +348,10 @@ ssl_connect (int fd) /* Register FD with Wget's transport layer, i.e. arrange that SSL-enabled functions are used for reading, writing, and polling. - That way the rest of Wget can use fd_read, fd_write, and friends - and not care what happens underneath. */ - fd_register_transport (fd, ssl_read, ssl_write, ssl_poll, ssl_close, ssl); + That way the rest of Wget can keep using xread, xwrite, and + friends and not care what happens underneath. */ + fd_register_transport (fd, ssl_read, ssl_write, ssl_poll, ssl_peek, + ssl_close, ssl); DEBUGP (("Connected %d to SSL 0x%0lx\n", fd, (unsigned long) ssl)); return ssl; diff --git a/src/headers.c b/src/headers.c index 5ceea754..c942ca3f 100644 --- a/src/headers.c +++ b/src/headers.c @@ -39,7 +39,6 @@ so, delete this exception statement from your version. */ #include "wget.h" #include "connect.h" -#include "rbuf.h" #include "headers.h" /* This file contains the generic routines for work with headers. @@ -62,74 +61,7 @@ so, delete this exception statement from your version. */ The public functions are header_get() and header_process(), which see. */ - -/* Get a header from read-buffer RBUF and return it in *HDR. - As defined in RFC2068 and elsewhere, a header can be folded into - multiple lines if the continuation line begins with a space or - horizontal TAB. Also, this function will accept a header ending - with just LF instead of CRLF. - - The header may be of arbitrary length; the function will allocate - as much memory as necessary for it to fit. It need not contain a - `:', thus you can use it to retrieve, say, HTTP status line. - - All trailing whitespace is stripped from the header, and it is - zero-terminated. */ -int -header_get (struct rbuf *rbuf, char **hdr, enum header_get_flags flags) -{ - int i; - int bufsize = 80; - - *hdr = (char *)xmalloc (bufsize); - for (i = 0; 1; i++) - { - int res; - /* #### Use DO_REALLOC? */ - if (i > bufsize - 1) - *hdr = (char *)xrealloc (*hdr, (bufsize <<= 1)); - res = RBUF_READCHAR (rbuf, *hdr + i); - if (res == 1) - { - if ((*hdr)[i] == '\n') - { - if (!((flags & HG_NO_CONTINUATIONS) - || i == 0 - || (i == 1 && (*hdr)[0] == '\r'))) - { - char next; - /* If the header is non-empty, we need to check if - it continues on to the other line. We do that by - peeking at the next character. */ - res = rbuf_peek (rbuf, &next); - if (res == 0) - return HG_EOF; - else if (res == -1) - return HG_ERROR; - /* If the next character is HT or SP, just continue. */ - if (next == '\t' || next == ' ') - continue; - } - - /* Strip trailing whitespace. (*hdr)[i] is the newline; - decrement I until it points to the last available - whitespace. */ - while (i > 0 && ISSPACE ((*hdr)[i - 1])) - --i; - (*hdr)[i] = '\0'; - break; - } - } - else if (res == 0) - return HG_EOF; - else - return HG_ERROR; - } - DEBUGP (("%s\n", *hdr)); - return HG_OK; -} - /* Check whether HEADER begins with NAME and, if yes, skip the `:' and the whitespace, and call PROCFUN with the arguments of HEADER's contents (after the `:' and space) and ARG. Otherwise, return 0. */ diff --git a/src/headers.h b/src/headers.h index 6f69472f..782ad359 100644 --- a/src/headers.h +++ b/src/headers.h @@ -37,7 +37,6 @@ enum { enum header_get_flags { HG_NONE = 0, HG_NO_CONTINUATIONS = 0x2 }; -int header_get PARAMS ((struct rbuf *, char **, enum header_get_flags)); int header_process PARAMS ((const char *, const char *, int (*) (const char *, void *), void *)); diff --git a/src/http.c b/src/http.c index ea4ccf24..de1cb084 100644 --- a/src/http.c +++ b/src/http.c @@ -61,7 +61,6 @@ extern int errno; #include "utils.h" #include "url.h" #include "host.h" -#include "rbuf.h" #include "retr.h" #include "headers.h" #include "connect.h" @@ -233,6 +232,24 @@ post_file (int sock, const char *file_name, long promised_size) DEBUGP (("done]\n")); return 0; } + +static const char * +next_header (const char *h) +{ + const char *end = NULL; + const char *p = h; + do + { + p = strchr (p, '\n'); + if (!p) + return end; + end = ++p; + } + while (*p == ' ' || *p == '\t'); + + return end; +} + /* Functions to be used as arguments to header_process(): */ @@ -598,19 +615,20 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy) char *pragma_h, *referer, *useragent, *range, *wwwauth; char *authenticate_h; char *proxyauth; - char *all_headers; char *port_maybe; char *request_keep_alive; - int sock, hcount, all_length, statcode; + int sock, hcount, statcode; int write_error; long contlen, contrange; struct url *conn; FILE *fp; int auth_tried_already; - struct rbuf rbuf; int using_ssl = 0; char *cookies = NULL; + char *head; + const char *hdr_beg, *hdr_end; + /* Whether this connection will be kept alive after the HTTP request is done. */ int keep_alive; @@ -989,70 +1007,43 @@ Accept: %s\r\n\ statcode = -1; *dt &= ~RETROKF; - /* Before reading anything, initialize the rbuf. */ - rbuf_initialize (&rbuf, sock); - all_headers = NULL; - all_length = 0; - DEBUGP (("\n---response begin---\n")); - /* Header-fetching loop. */ - hcount = 0; - while (1) + head = fd_read_head (sock); + if (!head) { - char *hdr; - int status; - - ++hcount; - /* Get the header. */ - status = header_get (&rbuf, &hdr, - /* Disallow continuations for status line. */ - (hcount == 1 ? HG_NO_CONTINUATIONS : HG_NONE)); - - /* Check for errors. */ - if (status == HG_EOF && *hdr) + logputs (LOG_VERBOSE, "\n"); + if (errno == 0) { - /* This used to be an unconditional error, but that was - somewhat controversial, because of a large number of - broken CGI's that happily "forget" to send the second EOL - before closing the connection of a HEAD request. - - So, the deal is to check whether the header is empty - (*hdr is zero if it is); if yes, it means that the - previous header was fully retrieved, and that -- most - probably -- the request is complete. "...be liberal in - what you accept." Oh boy. */ - logputs (LOG_VERBOSE, "\n"); - logputs (LOG_NOTQUIET, _("End of file while parsing headers.\n")); - xfree (hdr); - xfree_null (type); - xfree_null (all_headers); + logputs (LOG_NOTQUIET, _("No data received.\n")); CLOSE_INVALIDATE (sock); return HEOF; } - else if (status == HG_ERROR) + else { - logputs (LOG_VERBOSE, "\n"); logprintf (LOG_NOTQUIET, _("Read error (%s) in headers.\n"), strerror (errno)); - xfree (hdr); - xfree_null (type); - xfree_null (all_headers); CLOSE_INVALIDATE (sock); return HERR; } + } - /* If the headers are to be saved to a file later, save them to - memory now. */ - if (opt.save_headers) - { - int lh = strlen (hdr); - all_headers = (char *)xrealloc (all_headers, all_length + lh + 2); - memcpy (all_headers + all_length, hdr, lh); - all_length += lh; - all_headers[all_length++] = '\n'; - all_headers[all_length] = '\0'; - } + /* Loop through the headers and process them. */ + + hcount = 0; + for (hdr_beg = head; + (hdr_end = next_header (hdr_beg)); + hdr_beg = hdr_end) + { + char *hdr = strdupdelim (hdr_beg, hdr_end); + { + char *tmp = hdr + strlen (hdr); + if (tmp > hdr && tmp[-1] == '\n') + *--tmp = '\0'; + if (tmp > hdr && tmp[-1] == '\r') + *--tmp = '\0'; + } + ++hcount; /* Check for status line. */ if (hcount == 1) @@ -1257,7 +1248,6 @@ Accept: %s\r\n\ CLOSE_INVALIDATE (sock); /* would be CLOSE_FINISH, but there might be more bytes in the body. */ xfree_null (type); - xfree_null (all_headers); return NEWLOCATION; } } @@ -1328,7 +1318,6 @@ Accept: %s\r\n\ /* Mark as successfully retrieved. */ *dt |= RETROKF; xfree_null (type); - xfree_null (all_headers); CLOSE_INVALIDATE (sock); /* would be CLOSE_FINISH, but there might be more bytes in the body. */ return RETRUNNEEDED; @@ -1343,7 +1332,6 @@ Accept: %s\r\n\ Continued download failed on this file, which conflicts with `-c'.\n\ Refusing to truncate existing file `%s'.\n\n"), *hs->local_file); xfree_null (type); - xfree_null (all_headers); CLOSE_INVALIDATE (sock); return CONTNOTSUPPORTED; } @@ -1359,7 +1347,6 @@ Refusing to truncate existing file `%s'.\n\n"), *hs->local_file); /* This means the whole request was somehow misunderstood by the server. Bail out. */ xfree_null (type); - xfree_null (all_headers); CLOSE_INVALIDATE (sock); return RANGEERR; } @@ -1408,7 +1395,6 @@ Refusing to truncate existing file `%s'.\n\n"), *hs->local_file); hs->len = 0L; hs->res = 0; xfree_null (type); - xfree_null (all_headers); CLOSE_INVALIDATE (sock); /* would be CLOSE_FINISH, but there might be more bytes in the body. */ return RETRFINISHED; @@ -1426,7 +1412,6 @@ Refusing to truncate existing file `%s'.\n\n"), *hs->local_file); logprintf (LOG_NOTQUIET, "%s: %s\n", *hs->local_file, strerror (errno)); CLOSE_INVALIDATE (sock); /* would be CLOSE_FINISH, but there might be more bytes in the body. */ - xfree_null (all_headers); return FOPENERR; } } @@ -1466,12 +1451,12 @@ Refusing to truncate existing file `%s'.\n\n"), *hs->local_file); /* #### This confuses the code that checks for file size. There should be some overhead information. */ if (opt.save_headers) - fwrite (all_headers, 1, all_length, fp); + fwrite (head, 1, strlen (head), fp); /* Get the contents of the document. */ - hs->res = get_contents (sock, fp, &hs->len, hs->restval, + hs->res = fd_read_body (sock, fp, &hs->len, hs->restval, (contlen != -1 ? contlen : 0), - &rbuf, keep_alive, &hs->dltime); + keep_alive, &hs->dltime); if (hs->res >= 0) CLOSE_FINISH (sock); @@ -1490,7 +1475,6 @@ Refusing to truncate existing file `%s'.\n\n"), *hs->local_file); if (flush_res == EOF) hs->res = -2; } - xfree_null (all_headers); if (hs->res == -2) return FWRITEERR; return RETRFINISHED; diff --git a/src/rbuf.c b/src/rbuf.c deleted file mode 100644 index b50e0e2a..00000000 --- a/src/rbuf.c +++ /dev/null @@ -1,127 +0,0 @@ -/* Buffering read. - Copyright (C) 1995, 1996, 1997, 1998 Free Software Foundation, Inc. - -This file is part of GNU Wget. - -GNU Wget is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; either version 2 of the License, or -(at your option) any later version. - -GNU Wget is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with Wget; if not, write to the Free Software -Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. - -In addition, as a special exception, the Free Software Foundation -gives permission to link the code of its release of Wget with the -OpenSSL project's "OpenSSL" library (or with modified versions of it -that use the same license as the "OpenSSL" library), and distribute -the linked executables. You must obey the GNU General Public License -in all respects for all of the code used other than "OpenSSL". If you -modify this file, you may extend this exception to your version of the -file, but you are not obligated to do so. If you do not wish to do -so, delete this exception statement from your version. */ - -/* This is a simple implementation of buffering IO-read functions. */ - -#include - -#include -#ifdef HAVE_STRING_H -# include -#else -# include -#endif - -#include "wget.h" -#include "rbuf.h" -#include "connect.h" - -void -rbuf_initialize (struct rbuf *rbuf, int fd) -{ - rbuf->fd = fd; - rbuf->buffer_pos = rbuf->buffer; - rbuf->buffer_left = 0; -} - -int -rbuf_initialized_p (struct rbuf *rbuf) -{ - return rbuf->fd != -1; -} - -void -rbuf_uninitialize (struct rbuf *rbuf) -{ - rbuf->fd = -1; -} - -int -rbuf_read_bufferful (struct rbuf *rbuf) -{ - return fd_read (rbuf->fd, rbuf->buffer, sizeof (rbuf->buffer), -1); -} - -/* Currently unused -- see RBUF_READCHAR. */ -#if 0 -/* Function version of RBUF_READCHAR. */ -int -rbuf_readchar (struct rbuf *rbuf, char *store) -{ - return RBUF_READCHAR (rbuf, store); -} -#endif - -/* Like rbuf_readchar(), only don't move the buffer position. */ -int -rbuf_peek (struct rbuf *rbuf, char *store) -{ - if (!rbuf->buffer_left) - { - int res; - rbuf->buffer_pos = rbuf->buffer; - rbuf->buffer_left = 0; - res = fd_read (rbuf->fd, rbuf->buffer, sizeof (rbuf->buffer), -1); - if (res <= 0) - return res; - rbuf->buffer_left = res; - } - *store = *rbuf->buffer_pos; - return 1; -} - -#define MIN(p,q) (((p) <= (q)) ? (p) : (q)) - -/* Flush RBUF's buffer to WHERE. Flush MAXSIZE bytes at most. - Returns the number of bytes actually copied. If the buffer is - empty, 0 is returned. */ -int -rbuf_flush (struct rbuf *rbuf, char *where, int maxsize) -{ - if (!rbuf->buffer_left) - return 0; - else - { - int howmuch = MIN (rbuf->buffer_left, maxsize); - - if (where) - memcpy (where, rbuf->buffer_pos, howmuch); - rbuf->buffer_left -= howmuch; - rbuf->buffer_pos += howmuch; - return howmuch; - } -} - -/* Discard any cached data in RBUF. */ -void -rbuf_discard (struct rbuf *rbuf) -{ - rbuf->buffer_left = 0; - rbuf->buffer_pos = rbuf->buffer; -} diff --git a/src/rbuf.h b/src/rbuf.h deleted file mode 100644 index b0426377..00000000 --- a/src/rbuf.h +++ /dev/null @@ -1,83 +0,0 @@ -/* Declarations for rbuf.c. - Copyright (C) 1998 Free Software Foundation, Inc. - -This file is part of GNU Wget. - -GNU Wget is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; either version 2 of the License, or -(at your option) any later version. - -GNU Wget is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with Wget; if not, write to the Free Software -Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. - -In addition, as a special exception, the Free Software Foundation -gives permission to link the code of its release of Wget with the -OpenSSL project's "OpenSSL" library (or with modified versions of it -that use the same license as the "OpenSSL" library), and distribute -the linked executables. You must obey the GNU General Public License -in all respects for all of the code used other than "OpenSSL". If you -modify this file, you may extend this exception to your version of the -file, but you are not obligated to do so. If you do not wish to do -so, delete this exception statement from your version. */ - -#ifndef RBUF_H -#define RBUF_H - -/* Retrieval stream */ -struct rbuf -{ - int fd; - char buffer[4096]; /* the input buffer */ - char *buffer_pos; /* current position in the buffer */ - size_t buffer_left; /* number of bytes left in the buffer: - buffer_left = buffer_end - buffer_pos */ - int internal_dont_touch_this; /* used by RBUF_READCHAR macro */ -}; - -/* Read a character from RBUF. If there is anything in the buffer, - the character is returned from the buffer. Otherwise, refill the - buffer and return the first character. - - The return value is the same as with read(2). On buffered read, - the function returns 1. - - #### That return value is totally screwed up, and is a direct - result of historical implementation of header code. The macro - should return the character or EOF, and in case of error store it - to rbuf->err or something. */ - -#define RBUF_READCHAR(rbuf, store) \ -((rbuf)->buffer_left \ - ? (--(rbuf)->buffer_left, \ - *((char *) (store)) = *(rbuf)->buffer_pos++, 1) \ - : ((rbuf)->buffer_pos = (rbuf)->buffer, \ - ((((rbuf)->internal_dont_touch_this \ - = rbuf_read_bufferful (rbuf)) <= 0) \ - ? (rbuf)->internal_dont_touch_this \ - : ((rbuf)->buffer_left = (rbuf)->internal_dont_touch_this - 1, \ - *((char *) (store)) = *(rbuf)->buffer_pos++, \ - 1)))) - -/* Return the file descriptor of RBUF. */ -#define RBUF_FD(rbuf) ((rbuf)->fd) - -/* Function declarations */ -void rbuf_initialize PARAMS ((struct rbuf *, int)); -int rbuf_initialized_p PARAMS ((struct rbuf *)); -void rbuf_uninitialize PARAMS ((struct rbuf *)); -int rbuf_readchar PARAMS ((struct rbuf *, char *)); -int rbuf_peek PARAMS ((struct rbuf *, char *)); -int rbuf_flush PARAMS ((struct rbuf *, char *, int)); -void rbuf_discard PARAMS ((struct rbuf *)); - -/* Internal, but used by the macro. */ -int rbuf_read_bufferful PARAMS ((struct rbuf *)); - -#endif /* RBUF_H */ diff --git a/src/retr.c b/src/retr.c index 1fc3f8b0..5d9795ad 100644 --- a/src/retr.c +++ b/src/retr.c @@ -133,9 +133,7 @@ limit_bandwidth (long bytes, struct wget_timer *timer) /* Reads the contents of file descriptor FD, until it is closed, or a read error occurs. The data is read in 8K chunks, and stored to - stream fp, which should have been open for writing. If BUF is - non-NULL and its file descriptor is equal to FD, flush RBUF first. - This function will *not* use the rbuf_* functions! + stream fp, which should have been open for writing. The EXPECTED argument is passed to show_progress() unchanged, but otherwise ignored. @@ -147,14 +145,11 @@ limit_bandwidth (long bytes, struct wget_timer *timer) The function exits and returns codes of 0, -1 and -2 if the connection was closed, there was a read error, or if it could not - write to the output stream, respectively. + write to the output stream, respectively. */ - IMPORTANT: The function flushes the contents of the buffer in - rbuf_flush() before actually reading from fd. If you wish to read - from fd immediately, flush or discard the buffer. */ int -get_contents (int fd, FILE *fp, long *len, long restval, long expected, - struct rbuf *rbuf, int use_expected, double *elapsed) +fd_read_body (int fd, FILE *out, long *len, long restval, long expected, + int use_expected, double *elapsed) { int res = 0; @@ -181,26 +176,6 @@ get_contents (int fd, FILE *fp, long *len, long restval, long expected, progress_interactive = progress_interactive_p (progress); } - if (rbuf && RBUF_FD (rbuf) == fd) - { - int sz = 0; - while ((res = rbuf_flush (rbuf, dlbuf, sizeof (dlbuf))) != 0) - { - fwrite (dlbuf, 1, res, fp); - *len += res; - sz += res; - } - if (sz) - fflush (fp); - if (ferror (fp)) - { - res = -2; - goto out; - } - if (progress) - progress_update (progress, sz, 0); - } - if (opt.limit_rate) limit_bandwidth_reset (); wtimer_reset (timer); @@ -253,14 +228,14 @@ get_contents (int fd, FILE *fp, long *len, long restval, long expected, wtimer_update (timer); if (res > 0) { - fwrite (dlbuf, 1, res, fp); + fwrite (dlbuf, 1, res, out); /* Always flush the contents of the network packet. This should not hinder performance: fast downloads will be received in 16K chunks (which stdio would write out anyway), and slow downloads won't be limited by disk performance. */ - fflush (fp); - if (ferror (fp)) + fflush (out); + if (ferror (out)) { res = -2; goto out; @@ -292,6 +267,157 @@ get_contents (int fd, FILE *fp, long *len, long restval, long expected, return res; } +typedef const char *(*finder_t) PARAMS ((const char *, int, int)); + +/* Driver for fd_read_line and fd_read_head: keeps reading data until + a terminator (as decided by FINDER) occurs in the data. The trick + is that the data is first peeked at, and only then actually read. + That way the data after the terminator is never read. */ + +static char * +fd_read_until (int fd, finder_t finder, int bufsize) +{ + int size = bufsize, tail = 0; + char *buf = xmalloc (size); + + while (1) + { + const char *end; + int pklen, rdlen, remain; + + /* First, peek at the available data. */ + + pklen = fd_peek (fd, buf + tail, size - tail, -1); + if (pklen < 0) + { + xfree (buf); + return NULL; + } + end = finder (buf, tail, pklen); + if (end) + { + /* The data contains the terminator: we'll read the data up + to the end of the terminator. */ + remain = end - (buf + tail); + /* Note +1 for trailing \0. */ + if (size < tail + remain + 1) + { + size = tail + remain + 1; + buf = xrealloc (buf, size); + } + } + else + /* No terminator: simply read the data we know is (or should + be) available. */ + remain = pklen; + + /* Now, read the data. Note that we make no assumptions about + how much data we'll get. (Some TCP stacks are notorious for + read returning less data than the previous MSG_PEEK.) */ + + rdlen = fd_read (fd, buf + tail, remain, 0); + if (rdlen < 0) + { + xfree_null (buf); + return NULL; + } + if (rdlen == 0) + { + if (tail == 0) + { + /* EOF without anything having been read */ + xfree (buf); + errno = 0; + return NULL; + } + /* Return what we received so far. */ + if (size < tail + 1) + { + size = tail + 1; /* expand the buffer to receive the + terminating \0 */ + buf = xrealloc (buf, size); + } + buf[tail] = '\0'; + return buf; + } + tail += rdlen; + if (end && rdlen == remain) + { + /* The end was seen and the data read -- we got what we came + for. */ + buf[tail] = '\0'; + return buf; + } + + /* Keep looping until all the data arrives. */ + + if (tail == size) + { + size <<= 1; + buf = xrealloc (buf, size); + } + } +} + +static const char * +line_terminator (const char *buf, int tail, int peeklen) +{ + const char *p = memchr (buf + tail, '\n', peeklen); + if (p) + /* p+1 because we want the line to include '\n' */ + return p + 1; + return NULL; +} + +/* Read one line from FD and return it. The line is allocated using + malloc. + + If an error occurs, or if no data can be read, NULL is returned. + In the former case errno indicates the error condition, and in the + latter case, errno is NULL. */ + +char * +fd_read_line (int fd) +{ + return fd_read_until (fd, line_terminator, 128); +} + +static const char * +head_terminator (const char *buf, int tail, int peeklen) +{ + const char *start, *end; + if (tail < 4) + start = buf; + else + start = buf + tail - 4; + end = buf + tail + peeklen; + + for (; start < end - 1; start++) + if (*start == '\n') + { + if (start < end - 2 + && start[1] == '\r' + && start[2] == '\n') + return start + 3; + if (start[1] == '\n') + return start + 2; + } + return NULL; +} + +/* Read the request head from FD and return it. The chunk of data is + allocated using malloc. + + If an error occurs, or if no data can be read, NULL is returned. + In the former case errno indicates the error condition, and in the + latter case, errno is NULL. */ + +char * +fd_read_head (int fd) +{ + return fd_read_until (fd, head_terminator, 512); +} + /* Return a printed representation of the download rate, as appropriate for the speed. If PAD is non-zero, strings will be padded to the width of 7 characters (xxxx.xx). */ diff --git a/src/retr.h b/src/retr.h index 95090721..05434473 100644 --- a/src/retr.h +++ b/src/retr.h @@ -30,10 +30,10 @@ so, delete this exception statement from your version. */ #ifndef RETR_H #define RETR_H -#include "rbuf.h" +char *fd_read_line PARAMS ((int)); +char *fd_read_head PARAMS ((int)); -int get_contents PARAMS ((int, FILE *, long *, long, long, struct rbuf *, - int, double *)); +int fd_read_body PARAMS ((int, FILE *, long *, long, long, int, double *)); uerr_t retrieve_url PARAMS ((const char *, char **, char **, const char *, int *)); diff --git a/windows/Makefile.src b/windows/Makefile.src index 27a73f4e..617a8181 100644 --- a/windows/Makefile.src +++ b/windows/Makefile.src @@ -67,13 +67,13 @@ SRC = cmpt.c safe-ctype.c convert.c connect.c host.c http.c netrc.c \ ftp-basic.c ftp.c ftp-ls.c ftp-opie.c getopt.c hash.c headers.c \ html-parse.c html-url.c progress.c retr.c recur.c res.c url.c cookies.c \ init.c utils.c main.c version.c xmalloc.c mswindows.c \ - gen-md5.c gnu-md5.c rbuf.c log.c $(SSLSRC) + gen-md5.c gnu-md5.c log.c $(SSLSRC) OBJ = cmpt$o safe-ctype$o convert$o connect$o host$o http$o netrc$o \ ftp-basic$o ftp$o ftp-ls$o ftp-opie$o getopt$o hash$o headers$o \ html-parse$o html-url$o progress$o retr$o recur$o res$o url$o cookies$o \ init$o utils$o main$o version$o xmalloc$o mswindows$o \ - gen-md5$o gnu-md5$o rbuf$o log$o $(SSLOBJ) + gen-md5$o gnu-md5$o log$o $(SSLOBJ) .SUFFIXES: .c .obj diff --git a/windows/Makefile.src.bor b/windows/Makefile.src.bor index aaba63c3..91f982df 100644 --- a/windows/Makefile.src.bor +++ b/windows/Makefile.src.bor @@ -10,7 +10,7 @@ CFLAGS=-DWINDOWS -DHAVE_CONFIG_H -I. -w- -O2 ## variables OBJS=cmpt.obj connect.obj convert.obj ftp.obj ftp-basic.obj \ ftp-ls.obj ftp-opie.obj getopt.obj headers.obj host.obj html-parse.obj html-url.obj \ - http.obj init.obj log.obj main.obj gnu-md5.obj netrc.obj rbuf.obj \ + http.obj init.obj log.obj main.obj gnu-md5.obj netrc.obj \ safe-ctype.obj hash.obj progress.obj gen-md5.obj cookies.obj \ recur.obj res.obj retr.obj url.obj utils.obj version.obj xmalloc.obj \ mswindows.obj @@ -44,7 +44,6 @@ main.obj+ mswindows.obj+ netrc.obj+ progress.obj+ -rbuf.obj+ recur.obj+ res.obj+ retr.obj+ diff --git a/windows/Makefile.src.mingw b/windows/Makefile.src.mingw index 64f73341..0fddcf3a 100644 --- a/windows/Makefile.src.mingw +++ b/windows/Makefile.src.mingw @@ -23,7 +23,7 @@ LIBS= -lwsock32 OBJ_EXT=.o OBJS=cmpt${OBJ_EXT} convert${OBJ_EXT} connect${OBJ_EXT} ftp${OBJ_EXT} ftp-basic${OBJ_EXT} \ ftp-ls${OBJ_EXT} ftp-opie${OBJ_EXT} getopt${OBJ_EXT} headers${OBJ_EXT} host${OBJ_EXT} html-parse${OBJ_EXT} html-url${OBJ_EXT} \ - http${OBJ_EXT} init${OBJ_EXT} log${OBJ_EXT} main${OBJ_EXT} gnu-md5${OBJ_EXT} netrc${OBJ_EXT} rbuf${OBJ_EXT} \ + http${OBJ_EXT} init${OBJ_EXT} log${OBJ_EXT} main${OBJ_EXT} gnu-md5${OBJ_EXT} netrc${OBJ_EXT} \ safe-ctype${OBJ_EXT} hash${OBJ_EXT} progress${OBJ_EXT} gen-md5${OBJ_EXT} cookies${OBJ_EXT} \ recur${OBJ_EXT} res${OBJ_EXT} retr${OBJ_EXT} url${OBJ_EXT} utils${OBJ_EXT} \ version${OBJ_EXT} xmalloc${OBJ_EXT} mswindows${OBJ_EXT} diff --git a/windows/Makefile.watcom b/windows/Makefile.watcom index 44d0b5ba..75077d01 100644 --- a/windows/Makefile.watcom +++ b/windows/Makefile.watcom @@ -53,7 +53,7 @@ CFLAGS+= /os /d2 OBJS = cmpt.obj convert.obj connect.obj cookies.obj ftp.obj ftp-basic.obj & ftp-ls.obj ftp-opie.obj getopt.obj hash.obj headers.obj host.obj html-parse.obj html-url.obj & - http.obj init.obj log.obj main.obj gen-md5.obj gnu-md5.obj netrc.obj progress.obj rbuf.obj & + http.obj init.obj log.obj main.obj gen-md5.obj gnu-md5.obj netrc.obj progress.obj & recur.obj res.obj retr.obj safe-ctype.obj url.obj utils.obj version.obj mswindows.obj LIBFILES = diff --git a/windows/wget.dep b/windows/wget.dep index dae1399a..e2c2851a 100644 --- a/windows/wget.dep +++ b/windows/wget.dep @@ -4,29 +4,28 @@ cmpt$o: cmpt.c config.h wget.h sysdep.h mswindows.h options.h safe-ctype.h connect$o: connect.c config.h wget.h sysdep.h mswindows.h options.h safe-ctype.h utils.h connect.h host.h convert$o: convert.c config.h wget.h convert.h url.h recur.h utils.h hash.h cookies$o: cookies.c config.h wget.h sysdep.h mswindows.h options.h safe-ctype.h cookies.h hash.h url.h utils.h -ftp-basic$o: ftp-basic.c config.h wget.h sysdep.h mswindows.h options.h safe-ctype.h utils.h rbuf.h connect.h host.h ftp.h +ftp-basic$o: ftp-basic.c config.h wget.h sysdep.h mswindows.h options.h safe-ctype.h utils.h connect.h host.h ftp.h ftp-ls$o: ftp-ls.c config.h wget.h sysdep.h mswindows.h options.h safe-ctype.h utils.h ftp.h url.h ftp-opie$o: ftp-opie.c config.h wget.h sysdep.h mswindows.h options.h safe-ctype.h gen-md5.h -ftp$o: ftp.c config.h wget.h sysdep.h mswindows.h options.h safe-ctype.h utils.h url.h rbuf.h retr.h ftp.h connect.h host.h netrc.h +ftp$o: ftp.c config.h wget.h sysdep.h mswindows.h options.h safe-ctype.h utils.h url.h retr.h ftp.h connect.h host.h netrc.h gen-md5$o: gen-md5.c config.h wget.h sysdep.h mswindows.h options.h safe-ctype.h gen-md5.h gen_sslfunc$o: gen_sslfunc.c config.h wget.h sysdep.h mswindows.h options.h safe-ctype.h utils.h connect.h host.h url.h getopt$o: getopt.c config.h wget.h sysdep.h mswindows.h options.h safe-ctype.h getopt.h gnu-md5$o: gnu-md5.c config.h wget.h sysdep.h mswindows.h options.h safe-ctype.h gnu-md5.h hash$o: hash.c config.h wget.h sysdep.h mswindows.h options.h safe-ctype.h utils.h hash.h -headers$o: headers.c config.h wget.h sysdep.h mswindows.h options.h safe-ctype.h connect.h host.h rbuf.h headers.h +headers$o: headers.c config.h wget.h sysdep.h mswindows.h options.h safe-ctype.h connect.h host.h headers.h host$o: host.c config.h wget.h sysdep.h mswindows.h options.h safe-ctype.h utils.h host.h url.h hash.h html-parse$o: html-parse.c config.h wget.h sysdep.h mswindows.h options.h safe-ctype.h html-parse.h html-url$o: html-url.c config.h wget.h sysdep.h mswindows.h options.h safe-ctype.h html-parse.h url.h utils.h -http$o: http.c config.h wget.h sysdep.h mswindows.h options.h safe-ctype.h utils.h url.h host.h rbuf.h retr.h headers.h connect.h netrc.h gen-md5.h +http$o: http.c config.h wget.h sysdep.h mswindows.h options.h safe-ctype.h utils.h url.h host.h retr.h headers.h connect.h netrc.h gen-md5.h init$o: init.c config.h wget.h sysdep.h mswindows.h options.h safe-ctype.h utils.h init.h host.h recur.h netrc.h cookies.h progress.h log$o: log.c config.h wget.h sysdep.h mswindows.h options.h safe-ctype.h utils.h -main$o: main.c config.h wget.h sysdep.h mswindows.h options.h safe-ctype.h utils.h getopt.h init.h retr.h rbuf.h recur.h host.h gen_sslfunc.h getopt.h +main$o: main.c config.h wget.h sysdep.h mswindows.h options.h safe-ctype.h utils.h getopt.h init.h retr.h recur.h host.h gen_sslfunc.h getopt.h mswindows$o: mswindows.c config.h wget.h sysdep.h mswindows.h options.h safe-ctype.h utils.h url.h netrc$o: netrc.c config.h wget.h sysdep.h mswindows.h options.h safe-ctype.h utils.h netrc.h init.h -progress$o: progress.c config.h wget.h sysdep.h mswindows.h options.h safe-ctype.h utils.h retr.h rbuf.h -rbuf$o: rbuf.c config.h wget.h sysdep.h mswindows.h options.h safe-ctype.h rbuf.h connect.h host.h -recur$o: recur.c config.h wget.h sysdep.h mswindows.h options.h safe-ctype.h url.h recur.h utils.h retr.h rbuf.h ftp.h host.h hash.h -retr$o: retr.c config.h wget.h sysdep.h mswindows.h options.h safe-ctype.h utils.h retr.h rbuf.h url.h recur.h ftp.h host.h connect.h hash.h +progress$o: progress.c config.h wget.h sysdep.h mswindows.h options.h safe-ctype.h utils.h retr.h +recur$o: recur.c config.h wget.h sysdep.h mswindows.h options.h safe-ctype.h url.h recur.h utils.h retr.h ftp.h host.h hash.h +retr$o: retr.c config.h wget.h sysdep.h mswindows.h options.h safe-ctype.h utils.h retr.h url.h recur.h ftp.h host.h connect.h hash.h safe-ctype$o: safe-ctype.c config.h safe-ctype.h snprintf$o: snprintf.c config.h safe-ctype.h url$o: url.c config.h wget.h sysdep.h mswindows.h options.h safe-ctype.h utils.h url.h host.h hash.h