From 44872aefc2d54f297caf2b0cc887df321bc9d791 Mon Sep 17 00:00:00 2001 From: Viktor Szakats Date: Fri, 19 Feb 2021 13:57:19 +0000 Subject: [PATCH] http: add support to read and store the referrer header - add CURLINFO_REFERER libcurl option - add --write-out '%{referer}' command-line option - extend --xattr command-line option to fill user.xdg.referrer.url extended attribute with the referrer (if there was any) Closes #6591 --- docs/cmdline-opts/write-out.d | 3 +++ docs/examples/Makefile.inc | 1 + docs/libcurl/curl_easy_getinfo.3 | 5 ++++- docs/libcurl/opts/CURLOPT_REFERER.3 | 3 ++- docs/libcurl/opts/Makefile.inc | 3 ++- docs/libcurl/symbols-in-versions | 1 + include/curl/curl.h | 3 ++- lib/getinfo.c | 4 ++++ packages/OS400/README.OS400 | 1 + packages/OS400/curl.inc.in | 2 ++ src/tool_writeout.c | 1 + src/tool_writeout.h | 3 ++- src/tool_xattr.c | 9 +++++---- tests/data/test1067 | 5 ++++- tests/data/test970 | 2 +- 15 files changed, 35 insertions(+), 11 deletions(-) diff --git a/docs/cmdline-opts/write-out.d b/docs/cmdline-opts/write-out.d index be8f75ad5..0579cae69 100644 --- a/docs/cmdline-opts/write-out.d +++ b/docs/cmdline-opts/write-out.d @@ -93,6 +93,9 @@ When an HTTP request was made without --location to follow redirects (or when --max-redir is met), this variable will show the actual URL a redirect \fIwould\fP have gone to. (Added in 7.18.2) .TP +.B referer +The Referer: header, if there was any. (Added in 7.76.0) +.TP .B remote_ip The remote IP address of the most recently done connection - can be either IPv4 or IPv6 (Added in 7.29.0) diff --git a/docs/examples/Makefile.inc b/docs/examples/Makefile.inc index defaad6d6..84a2bdac6 100644 --- a/docs/examples/Makefile.inc +++ b/docs/examples/Makefile.inc @@ -43,6 +43,7 @@ check_PROGRAMS = \ getinfo \ getinmemory \ getredirect \ + getreferrer \ http-post \ http2-download \ http2-pushinmemory \ diff --git a/docs/libcurl/curl_easy_getinfo.3 b/docs/libcurl/curl_easy_getinfo.3 index d3dfcf26b..845e303e3 100644 --- a/docs/libcurl/curl_easy_getinfo.3 +++ b/docs/libcurl/curl_easy_getinfo.3 @@ -5,7 +5,7 @@ .\" * | (__| |_| | _ <| |___ .\" * \___|\___/|_| \_\_____| .\" * -.\" * Copyright (C) 1998 - 2020, Daniel Stenberg, , et al. +.\" * Copyright (C) 1998 - 2021, Daniel Stenberg, , et al. .\" * .\" * This software is licensed as described in the file COPYING, which .\" * you should have received as part of this distribution. The terms @@ -50,6 +50,9 @@ See \fICURLINFO_EFFECTIVE_URL(3)\fP .IP CURLINFO_RESPONSE_CODE Last received response code. See \fICURLINFO_RESPONSE_CODE(3)\fP +.IP CURLINFO_REFERER +Referrer header. +See \fICURLINFO_REFERER(3)\fP .IP CURLINFO_HTTP_CONNECTCODE Last proxy CONNECT response code. See \fICURLINFO_HTTP_CONNECTCODE(3)\fP diff --git a/docs/libcurl/opts/CURLOPT_REFERER.3 b/docs/libcurl/opts/CURLOPT_REFERER.3 index c3a0ed781..ce3ed173c 100644 --- a/docs/libcurl/opts/CURLOPT_REFERER.3 +++ b/docs/libcurl/opts/CURLOPT_REFERER.3 @@ -5,7 +5,7 @@ .\" * | (__| |_| | _ <| |___ .\" * \___|\___/|_| \_\_____| .\" * -.\" * Copyright (C) 1998 - 2020, Daniel Stenberg, , et al. +.\" * Copyright (C) 1998 - 2021, Daniel Stenberg, , et al. .\" * .\" * This software is licensed as described in the file COPYING, which .\" * you should have received as part of this distribution. The terms @@ -58,3 +58,4 @@ Returns CURLE_OK if HTTP support is enabled, CURLE_UNKNOWN_OPTION if not, or CURLE_OUT_OF_MEMORY if there was insufficient heap space. .SH "SEE ALSO" .BR CURLOPT_USERAGENT "(3), " CURLOPT_HTTPHEADER "(3), " +.BR CURLINFO_REFERER "(3), " diff --git a/docs/libcurl/opts/Makefile.inc b/docs/libcurl/opts/Makefile.inc index 5cec7cb4e..69e241862 100644 --- a/docs/libcurl/opts/Makefile.inc +++ b/docs/libcurl/opts/Makefile.inc @@ -5,7 +5,7 @@ # | (__| |_| | _ <| |___ # \___|\___/|_| \_\_____| # -# Copyright (C) 1998 - 2020, Daniel Stenberg, , et al. +# Copyright (C) 1998 - 2021, Daniel Stenberg, , et al. # # This software is licensed as described in the file COPYING, which # you should have received as part of this distribution. The terms @@ -64,6 +64,7 @@ man_MANS = \ CURLINFO_REDIRECT_TIME.3 \ CURLINFO_REDIRECT_TIME_T.3 \ CURLINFO_REDIRECT_URL.3 \ + CURLINFO_REFERER.3 \ CURLINFO_REQUEST_SIZE.3 \ CURLINFO_RESPONSE_CODE.3 \ CURLINFO_RETRY_AFTER.3 \ diff --git a/docs/libcurl/symbols-in-versions b/docs/libcurl/symbols-in-versions index 7da5c4cfa..5ff3a3503 100644 --- a/docs/libcurl/symbols-in-versions +++ b/docs/libcurl/symbols-in-versions @@ -271,6 +271,7 @@ CURLINFO_REDIRECT_COUNT 7.9.7 CURLINFO_REDIRECT_TIME 7.9.7 CURLINFO_REDIRECT_TIME_T 7.61.0 CURLINFO_REDIRECT_URL 7.18.2 +CURLINFO_REFERER 7.76.0 CURLINFO_REQUEST_SIZE 7.4.1 CURLINFO_RESPONSE_CODE 7.10.8 CURLINFO_RETRY_AFTER 7.66.0 diff --git a/include/curl/curl.h b/include/curl/curl.h index b53df7851..bed8068b0 100644 --- a/include/curl/curl.h +++ b/include/curl/curl.h @@ -2761,8 +2761,9 @@ typedef enum { CURLINFO_RETRY_AFTER = CURLINFO_OFF_T + 57, CURLINFO_EFFECTIVE_METHOD = CURLINFO_STRING + 58, CURLINFO_PROXY_ERROR = CURLINFO_LONG + 59, + CURLINFO_REFERER = CURLINFO_STRING + 60, - CURLINFO_LASTONE = 59 + CURLINFO_LASTONE = 60 } CURLINFO; /* CURLINFO_RESPONSE_CODE is the new name for the option previously known as diff --git a/lib/getinfo.c b/lib/getinfo.c index f531631b8..22bd3a135 100644 --- a/lib/getinfo.c +++ b/lib/getinfo.c @@ -145,6 +145,10 @@ static CURLcode getinfo_char(struct Curl_easy *data, CURLINFO info, option had been enabled! */ *param_charp = data->info.wouldredirect; break; + case CURLINFO_REFERER: + /* Return the referrer header for this request, or NULL if unset */ + *param_charp = data->change.referer; + break; case CURLINFO_PRIMARY_IP: /* Return the ip address of the most recent (primary) connection */ *param_charp = data->info.conn_primary_ip; diff --git a/packages/OS400/README.OS400 b/packages/OS400/README.OS400 index 95a558411..6ea4ac29e 100644 --- a/packages/OS400/README.OS400 +++ b/packages/OS400/README.OS400 @@ -178,6 +178,7 @@ use: CURLINFO_CONTENT_TYPE CURLINFO_FTP_ENTRY_PATH CURLINFO_REDIRECT_URL + CURLINFO_REFERER CURLINFO_PRIMARY_IP CURLINFO_RTSP_SESSION_ID CURLINFO_LOCAL_IP diff --git a/packages/OS400/curl.inc.in b/packages/OS400/curl.inc.in index ec052dc76..c77e2066f 100644 --- a/packages/OS400/curl.inc.in +++ b/packages/OS400/curl.inc.in @@ -1690,6 +1690,8 @@ d c X'0010003A' d CURLINFO_PROXY_ERROR... CURLINFO_LONG + 59 d c X'0020003B' + d CURLINFO_REFERER... CURLINFO_STRING + 60 + d c X'0010003C' * d CURLINFO_HTTP_CODE... Old ...RESPONSE_CODE d c X'00200002' diff --git a/src/tool_writeout.c b/src/tool_writeout.c index e914dfa7b..529677818 100644 --- a/src/tool_writeout.c +++ b/src/tool_writeout.c @@ -83,6 +83,7 @@ static const struct writeoutvar variables[] = { {"proxy_ssl_verify_result", VAR_PROXY_SSL_VERIFY_RESULT, CURLINFO_PROXY_SSL_VERIFYRESULT, writeLong}, {"redirect_url", VAR_REDIRECT_URL, CURLINFO_REDIRECT_URL, writeString}, + {"referer", VAR_REFERER, CURLINFO_REFERER, writeString}, {"remote_ip", VAR_PRIMARY_IP, CURLINFO_PRIMARY_IP, writeString}, {"remote_port", VAR_PRIMARY_PORT, CURLINFO_PRIMARY_PORT, writeLong}, {"response_code", VAR_HTTP_CODE, CURLINFO_RESPONSE_CODE, writeLong}, diff --git a/src/tool_writeout.h b/src/tool_writeout.h index 2cf7d012a..b8f51ffaa 100644 --- a/src/tool_writeout.h +++ b/src/tool_writeout.h @@ -7,7 +7,7 @@ * | (__| |_| | _ <| |___ * \___|\___/|_| \_\_____| * - * Copyright (C) 1998 - 2020, Daniel Stenberg, , et al. + * Copyright (C) 1998 - 2021, Daniel Stenberg, , et al. * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms @@ -54,6 +54,7 @@ typedef enum { VAR_REDIRECT_COUNT, VAR_REDIRECT_TIME, VAR_REDIRECT_URL, + VAR_REFERER, VAR_REQUEST_SIZE, VAR_SCHEME, VAR_SIZE_DOWNLOAD, diff --git a/src/tool_xattr.c b/src/tool_xattr.c index b6a00cb01..08e92fa50 100644 --- a/src/tool_xattr.c +++ b/src/tool_xattr.c @@ -5,7 +5,7 @@ * | (__| |_| | _ <| |___ * \___|\___/|_| \_\_____| * - * Copyright (C) 1998 - 2020, Daniel Stenberg, , et al. + * Copyright (C) 1998 - 2021, Daniel Stenberg, , et al. * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms @@ -44,9 +44,10 @@ static const struct xattr_mapping { /* mappings proposed by * https://freedesktop.org/wiki/CommonExtendedAttributes/ */ - { "user.xdg.origin.url", CURLINFO_EFFECTIVE_URL }, - { "user.mime_type", CURLINFO_CONTENT_TYPE }, - { NULL, CURLINFO_NONE } /* last element, abort loop here */ + { "user.xdg.referrer.url", CURLINFO_REFERER }, + { "user.xdg.origin.url", CURLINFO_EFFECTIVE_URL }, + { "user.mime_type", CURLINFO_CONTENT_TYPE }, + { NULL, CURLINFO_NONE } /* last element, abort here */ }; /* returns TRUE if a new URL is returned, that then needs to be freed */ diff --git a/tests/data/test1067 b/tests/data/test1067 index b464c71ca..5790f6b86 100644 --- a/tests/data/test1067 +++ b/tests/data/test1067 @@ -53,7 +53,7 @@ http HTTP Location: following with auto-referer -http://%HOSTIP:%HTTPPORT/want/1067 -L --referer "firstone.html;auto" +http://%HOSTIP:%HTTPPORT/want/1067 --silent --location --referer "firstone.html;auto" --write-out "%{stderr}|%{referer}|" @@ -73,5 +73,8 @@ Accept: */* Referer: http://%HOSTIP:%HTTPPORT/want/1067 + +|http://%HOSTIP:%HTTPPORT/want/1067| + diff --git a/tests/data/test970 b/tests/data/test970 index 6a89a5455..44c640c94 100644 --- a/tests/data/test970 +++ b/tests/data/test970 @@ -59,7 +59,7 @@ Accept: */* -{"content_type":"text/html","errormsg":null,"exitcode":0,"filename_effective":"log/out970","ftp_entry_path":null,"http_code":200,"http_connect":000,"http_version":"1.1","local_ip":"127.0.0.1","local_port":13,"method":"GET","num_connects":1,"num_headers":9,"num_redirects":0,"proxy_ssl_verify_result":0,"redirect_url":null,"remote_ip":"%HOSTIP","remote_port":%HTTPPORT,"response_code":200,"scheme":"HTTP","size_download":445,"size_header":4019,"size_request":4019,"size_upload":0,"speed_download":13,"speed_upload":13,"ssl_verify_result":0,"time_appconnect":0.000013,"time_connect":0.000013,"time_namelookup":0.000013,"time_pretransfer":0.000013,"time_redirect":0.000013,"time_starttransfer":0.000013,"time_total":0.000013,"url":"http://%HOSTIP:%HTTPPORT/970","url_effective":"http://%HOSTIP:%HTTPPORT/970","urlnum":0,"curl_version":"curl-unit-test-fake-version"} +{"content_type":"text/html","errormsg":null,"exitcode":0,"filename_effective":"log/out970","ftp_entry_path":null,"http_code":200,"http_connect":000,"http_version":"1.1","local_ip":"127.0.0.1","local_port":13,"method":"GET","num_connects":1,"num_headers":9,"num_redirects":0,"proxy_ssl_verify_result":0,"redirect_url":null,"referer":null,"remote_ip":"%HOSTIP","remote_port":%HTTPPORT,"response_code":200,"scheme":"HTTP","size_download":445,"size_header":4019,"size_request":4019,"size_upload":0,"speed_download":13,"speed_upload":13,"ssl_verify_result":0,"time_appconnect":0.000013,"time_connect":0.000013,"time_namelookup":0.000013,"time_pretransfer":0.000013,"time_redirect":0.000013,"time_starttransfer":0.000013,"time_total":0.000013,"url":"http://%HOSTIP:%HTTPPORT/970","url_effective":"http://%HOSTIP:%HTTPPORT/970","urlnum":0,"curl_version":"curl-unit-test-fake-version"}