From cdcd923375fd9f2c55221812e96c01f6dcd25e1b Mon Sep 17 00:00:00 2001
From: luccioman <luccioman@users.noreply.github.com>
Date: Tue, 28 Feb 2017 18:11:54 +0100
Subject: [PATCH] Privacy enhancement : added settings to control referrer
 policy.

HTTP "Referer" header sent by the browser when using YaCy can now be
controlled either with the referrer meta tag as a global policy, or only
for search result links by adding the attribute rel="noreferrer".

To improve privacy with the less possible regressions, the default is
set as meta tag with value "origin-when-cross-origin" : internal YaCy
links behavior is not affected, but when visiting external websites
referrer url is not empty but stripped from query parameters and path.

Older browsers, Safari, MS IE and Edge do not support the referrer meta
tag, so the standard but less flexible noreferrer link type can also be
enabled as an alternative.

User-friendly settings page to be implemented.
---
 defaults/yacy.init                                   | 12 ++++++++++++
 htroot/env/templates/metas.template                  |  1 +
 htroot/yacysearchitem.html                           | 12 ++++++------
 htroot/yacysearchitem.java                           |  9 +++++++--
 .../net/yacy/http/servlets/YaCyDefaultServlet.java   |  1 +
 source/net/yacy/search/SwitchboardConstants.java     | 12 ++++++++++++
 6 files changed, 39 insertions(+), 8 deletions(-)

diff --git a/defaults/yacy.init b/defaults/yacy.init
index 6d978fc68..2aa3109e3 100644
--- a/defaults/yacy.init
+++ b/defaults/yacy.init
@@ -89,6 +89,12 @@ server.maxTrackingCount = 1000
 # maximum number of hosts that are tracked
 server.maxTrackingHostCount = 100
 
+# Global HTTP Referrer policy delivered by meta tag (see https://www.w3.org/TR/referrer-policy/ for available policies)
+# Can be left empty : the browser should then fallback to the default "no-referrer-when-downgrade" policy
+# Be careful, some policies will also affect YaCy internal links : "no-referrer", "same-origin", "origin" and "strict-origin". This can be useful 
+# for example when proxyURL.useforresults is set to true, to control the referrer information initially sent by YaCy proxy when visiting search results
+referrer.meta.policy=origin-when-cross-origin
+
 # maximum file sizes: since some users experience problems with too large files
 # the file size of database files can be limited. Larger files can be used to get a
 # better IO performance and to use less RAM; however, if the size must be limited
@@ -834,6 +840,12 @@ search.target = _self
 search.target.special = _self
 search.target.special.pattern =
 
+# When true, override the global referrer.meta.policy value and add the standard noreferrer link type to search results links : 
+# this instructs the browser that it should not send any referrer information at all when visiting them
+# Be careful : some websites might reject requests with no referrer
+# Supported by more browsers than the meta referrer tag
+search.result.noreferrer=false
+
 # search result lines may show additional information for each search hit
 # these information pieces may be switched on or off
 search.result.show.date = true
diff --git a/htroot/env/templates/metas.template b/htroot/env/templates/metas.template
index bc5ce7a10..30e62e4d9 100644
--- a/htroot/env/templates/metas.template
+++ b/htroot/env/templates/metas.template
@@ -4,6 +4,7 @@
   <meta name="keywords" content="YaCy HTTP search engine spider indexer java network open free download Mac Windows Linux Software development" />
   <meta name="description" content="Software HTTP Freeware Home Page" />
   <meta name="copyright" content="Michael Christen et al." />
+  <meta name="referrer" content="#[referrer.meta.policy]#"/>
   
   <!-- Ensure proper rendering and touch zooming on mobile devices -->
   <meta name="viewport" content="width=device-width, initial-scale=1">
diff --git a/htroot/yacysearchitem.html b/htroot/yacysearchitem.html
index ef08933f4..73d08678c 100644
--- a/htroot/yacysearchitem.html
+++ b/htroot/yacysearchitem.html
@@ -4,7 +4,7 @@
   #(favicon)#::
   <img width="16" height="16" src="#[faviconUrl]#" id="f#[urlhash]#" class="favicon" style="width:16px; height:16px;" alt="" />
   #(/favicon)#  
-  <a href="#[link]#" target="#[target]#">#[title]#</a></h4>
+  <a href="#[link]#" target="#[target]#" #(noreferrer)#::rel="noreferrer"#(/noreferrer)#>#[title]#</a></h4>
   <div class="urlactions">
     #(heuristic)#::
     <img width="16" height="9" src="env/grafics/heuristic_redundant.gif" title="heuristic:#[name]# (redundant)" style="width:16px; height:9px;" alt="heuristic:#[name]# (redundant)"/>::
@@ -24,7 +24,7 @@
     #(/authorized)#
   </div>
   <p class="snippet"><span class="snippetLoaded" id="h#[urlhash]#">#[description]#</span></p>
-  <p class="url"><a href="#[link]#" id="url#[urlhash]#" target="#[target]#">#[urlname]#</a></p>
+  <p class="url"><a href="#[link]#" id="url#[urlhash]#" target="#[target]#" #(noreferrer)#::rel="noreferrer"#(/noreferrer)#>#[urlname]#</a></p>
   <p class="urlinfo">
   #(showDate)#::#[date]##(/showDate)#
   #(showEvent)#::on #[date]##(/showEvent)#
@@ -45,14 +45,14 @@
     <a href="#[hrefFullPreview]#" target="#[target]#" class="thumblink" onclick="return hs.expand(this)">
       <img src="#[hrefCache]#" width="#[width]#" height="#[height]#" style="#[style]#" alt="#[name]#" />
     </a>
-    <div class="highslide-caption"><a href="#[href]#" target="#[target]#">#[name]#</a><br /><a href="#[source]#" target="#[target]#">#[sourcedom]#</a></div>
+    <div class="highslide-caption"><a href="#[href]#" target="#[target]#" #(noreferrer)#::rel="noreferrer"#(/noreferrer)#>#[name]#</a><br /><a href="#[source]#" target="#[target]#" #(noreferrer)#::rel="noreferrer"#(/noreferrer)#>#[sourcedom]#</a></div>
   </div>#(/item)#
   ::
-  #(item)#::<tr class="#(col)#TableCellLight::TableCellDark#(/col)#"><td>#[name]#</td><td><a href="#[href]#" target="#[target]#">#[hrefshort]#</a></tr>#(/item)#
+  #(item)#::<tr class="#(col)#TableCellLight::TableCellDark#(/col)#"><td>#[name]#</td><td><a href="#[href]#" target="#[target]#" #(noreferrer)#::rel="noreferrer"#(/noreferrer)#>#[hrefshort]#</a></tr>#(/item)#
   ::
-  #(item)#::<tr class="#(col)#TableCellLight::TableCellDark#(/col)#"><td>#[name]#</td><td><a href="#[href]#" target="#[target]#">#[hrefshort]#</a></tr>#(/item)#
+  #(item)#::<tr class="#(col)#TableCellLight::TableCellDark#(/col)#"><td>#[name]#</td><td><a href="#[href]#" target="#[target]#" #(noreferrer)#::rel="noreferrer"#(/noreferrer)#>#[hrefshort]#</a></tr>#(/item)#
   ::
-  #(item)#::<tr class="#(col)#TableCellLight::TableCellDark#(/col)#"><td>#[name]#</td><td><a href="#[href]#" target="#[target]#">#[hrefshort]#</a></tr>#(/item)#
+  #(item)#::<tr class="#(col)#TableCellLight::TableCellDark#(/col)#"><td>#[name]#</td><td><a href="#[href]#" target="#[target]#" #(noreferrer)#::rel="noreferrer"#(/noreferrer)#>#[hrefshort]#</a></tr>#(/item)#
 #(/content)#
 
 <script type="text/javascript">
diff --git a/htroot/yacysearchitem.java b/htroot/yacysearchitem.java
index 34e258326..21038f64c 100644
--- a/htroot/yacysearchitem.java
+++ b/htroot/yacysearchitem.java
@@ -122,6 +122,7 @@ public class yacysearchitem {
         prop.put("navurlBase", QueryParams.navurlBase(RequestHeader.FileType.HTML, theSearch.query, null, false).toString());
         prop.put("localQuery", theSearch.query.isLocal() ? "1" : "0");
         final String target_special_pattern = sb.getConfig(SwitchboardConstants.SEARCH_TARGET_SPECIAL_PATTERN, "");
+        final boolean noreferrer = sb.getConfigBool(SwitchboardConstants.SEARCH_RESULT_NOREFERRER, SwitchboardConstants.SEARCH_RESULT_NOREFERRER_DEFAULT);
 
         long timeout = item == 0 ? 10000 : (theSearch.query.isLocal() ? 1000 : 3000);
         
@@ -177,6 +178,7 @@ public class yacysearchitem {
             } else {
                 prop.putXML("content_link", resultUrlstring); // putXML for rss
             }
+            prop.put("content_noreferrer", noreferrer ? 1 : 0);
             
 // END interaction
 
@@ -307,7 +309,7 @@ public class yacysearchitem {
 
         if (theSearch.query.contentdom == Classification.ContentDomain.IMAGE) {
             // image search; shows thumbnails
-            processImage(sb, prop, item, theSearch, target_special_pattern, timeout, ImageViewer.hasFullViewingRights(header, sb));
+            processImage(sb, prop, item, theSearch, target_special_pattern, timeout, ImageViewer.hasFullViewingRights(header, sb), noreferrer);
             theSearch.query.transmitcount = item + 1;
             return prop;
         }
@@ -326,6 +328,7 @@ public class yacysearchitem {
                 final String resultUrlstring = ms.url().toNormalform(true);
                 final String target = sb.getConfig(resultUrlstring.matches(target_special_pattern) ? SwitchboardConstants.SEARCH_TARGET_SPECIAL : SwitchboardConstants.SEARCH_TARGET_DEFAULT, "_self");
                 prop.putHTML("content_item_href", resultUrlstring);
+                prop.put("content_item_noreferrer", noreferrer ? 1 : 0);
                 prop.putHTML("content_item_hrefshort", nxTools.shortenURLString(resultUrlstring, MAX_URL_LENGTH));
                 prop.putHTML("content_item_target", target);
                 prop.putHTML("content_item_name", shorten(ms.title(), MAX_NAME_LENGTH));
@@ -474,9 +477,10 @@ public class yacysearchitem {
      * @param target_special_pattern
      * @param timeout result getting timeOut
      * @param fullViewingRights set to true when current user has full image viewing rights
+     * @param noreferrer set to true when the noreferrer link type should be added to the original image source links
      */
 	private static void processImage(final Switchboard sb, final serverObjects prop, final int item,
-			final SearchEvent theSearch, final String target_special_pattern, long timeout, boolean fullViewingRights) {
+			final SearchEvent theSearch, final String target_special_pattern, long timeout, boolean fullViewingRights, final boolean noreferrer) {
 		prop.put("content", theSearch.query.contentdom.getCode() + 1); // switch on specific content
 		try {
 		    SearchEvent.ImageResult image = theSearch.oneImageResult(item, timeout);
@@ -544,6 +548,7 @@ public class yacysearchitem {
 		    prop.put("content_item_attr", ""/*(ms.attr.equals("-1 x -1")) ? "" : "(" + ms.attr + ")"*/); // attributes, here: original size of image
 		    prop.put("content_item_urlhash", ASCII.String(image.imageUrl.hash()));
 		    prop.put("content_item_source", image.sourceUrl.toNormalform(true));
+		    prop.put("content_item_noreferrer", noreferrer ? 1 : 0);
 		    prop.putXML("content_item_source-xml", image.sourceUrl.toNormalform(true));
 		    prop.put("content_item_sourcedom", image.sourceUrl.getHost());
 		    prop.put("content_item_nl", (item == theSearch.query.offset) ? 0 : 1);
diff --git a/source/net/yacy/http/servlets/YaCyDefaultServlet.java b/source/net/yacy/http/servlets/YaCyDefaultServlet.java
index a9227663d..e751a782d 100644
--- a/source/net/yacy/http/servlets/YaCyDefaultServlet.java
+++ b/source/net/yacy/http/servlets/YaCyDefaultServlet.java
@@ -1020,6 +1020,7 @@ public class YaCyDefaultServlet extends HttpServlet  {
                 templatePatterns.putHTML(servletProperties.PEER_STAT_CLIENTID, sb.peers.myID());
                 templatePatterns.put(servletProperties.PEER_STAT_MYTIME, GenericFormatter.SHORT_SECOND_FORMATTER.format());
                 templatePatterns.put(servletProperties.RELATIVE_BASE, YaCyDefaultServlet.getRelativeBase(target));
+                templatePatterns.put(SwitchboardConstants.REFERRER_POLICY, sb.getConfig(SwitchboardConstants.REFERRER_POLICY, SwitchboardConstants.REFERRER_POLICY_DEFAULT));
                 Seed myPeer = sb.peers.mySeed();
                 templatePatterns.put("newpeer", myPeer.getAge() >= 1 ? 0 : 1);
                 templatePatterns.putHTML("newpeer_peerhash", myPeer.hash);
diff --git a/source/net/yacy/search/SwitchboardConstants.java b/source/net/yacy/search/SwitchboardConstants.java
index 3a506479c..0febe26ee 100644
--- a/source/net/yacy/search/SwitchboardConstants.java
+++ b/source/net/yacy/search/SwitchboardConstants.java
@@ -273,6 +273,12 @@ public final class SwitchboardConstants {
     public static final String CLUSTER_MODE_PUBLIC_PEER         = "publicpeer";
     public static final String CLUSTER_MODE_PRIVATE_PEER        = "privatepeer";
     public static final String CLUSTER_PEERS_IPPORT             = "cluster.peers.ipport";
+    
+    /** Key of the global HTTP Referrer policy delivered by meta tag */
+    public static final String REFERRER_POLICY = "referrer.meta.policy";
+    
+    /** Default value for the global HTTP Referrer policy delivered by meta tag */
+    public static final String REFERRER_POLICY_DEFAULT = "origin-when-cross-origin";
 
     
     public static final String DHT_ENABLED                      = "network.unit.dht";
@@ -532,6 +538,12 @@ public final class SwitchboardConstants {
     public static final String SEARCH_VERIFY_DELETE = "search.verify.delete";
 
     public static final String SEARCH_NAVIGATION_MAXCOUNT = "search.navigation.maxcount"; // max lines displayed in standard search navigators/facets
+    
+    /** Key of the setting controlling whether a noreferrer link type should be added to search result links */
+    public static final String SEARCH_RESULT_NOREFERRER = "search.result.noreferrer";
+    
+    /** Default setting value controlling whether a noreferrer link type should be added to search result links */
+    public static final boolean SEARCH_RESULT_NOREFERRER_DEFAULT = false;
 
     /**
      * ranking+evaluation