Added gzip compression of responses returned to user-agents accepting it

Enabled as default, but can be disabled using the "Server Access
Settings" admin page.

defaults/yacy.init

@@ -92,6 +92,11 @@ server.maxTrackingCount = 1000
 # maximum number of hosts that are tracked
 server.maxTrackingHostCount = 100
 
+# When set to true, enable gzip compression of HTTP responses when the user-agent (browser, another YaCy peer or any kind of client) 
+# accept gzip encoding (by including gzip in a 'Accept-Encoding' HTTP request header)
+# This adds some processing overhead, but reduces the amount of bytes sent over network.
+server.response.compress.gzip = true
+
 # Global HTTP Referrer policy delivered by meta tag (see https://www.w3.org/TR/referrer-policy/ for available policies)
 # Can be left empty : the browser should then fallback to the default "no-referrer-when-downgrade" policy
 # Be careful, some policies will also affect YaCy internal links : "no-referrer", "same-origin", "origin" and "strict-origin". This can be useful 

htroot/SettingsAck_p.html

@@ -208,6 +208,9 @@
           <td>Shutdown port</td><td> <span class="settingsValue">#[port.shutdown]#</span></td>
         </tr>
       </table>
+	::<!-- 37: Compression settings changed -->
+    <p>Compression settings have been saved.</p>
+    
     #(/info)#
     <p></p>
     #(needsRestart)#

htroot/SettingsAck_p.java

@@ -259,6 +259,23 @@ public class SettingsAck_p {
             return prop;
         }
         
+        /* Compression settings */
+        if (post.containsKey("serverCompression")) {
+            // set backlink
+            prop.put("needsRestart_referer", "Settings_p.html?page=ServerAccess");
+            
+			final boolean oldValue = env.getConfigBool(SwitchboardConstants.SERVER_RESPONSE_COMPRESS_GZIP,
+					SwitchboardConstants.SERVER_RESPONSE_COMPRESS_GZIP_DEFAULT);
+        	final boolean newValue = post.containsKey(SwitchboardConstants.SERVER_RESPONSE_COMPRESS_GZIP);
+            env.setConfig(SwitchboardConstants.SERVER_RESPONSE_COMPRESS_GZIP, newValue);
+            
+            prop.put("needsRestart", newValue != oldValue);
+            
+            /* server compression settings saved */
+            prop.put("info", "37");
+        	return prop;
+        }
+
         if (post.containsKey("proxysettings")) {
             // set backlink
             prop.put("needsRestart_referer", "Settings_p.html?page=proxy");

htroot/Settings_ServerAccess.inc

@@ -66,4 +66,32 @@
 </table>
 </fieldset>
 
+<fieldset>
+	<legend>Compression settings</legend>
+	
+	<div class="form-group">
+		<div class="col-sm-3">
+			<div class="checkbox">
+				<label>
+					<input name="server.response.compress.gzip" id="responseCompressGzip" 
+						type="checkbox" #(server.response.compress.gzip)#::checked#(/server.response.compress.gzip)# 
+						aria-describedby="responseCompressGzipInfo"/>
+					Compress responses with gzip
+				</label>
+			</div>
+		</div>
+		<div class="col-sm-9" id="responseCompressGzipInfo">
+ 			When checked (default), HTTP responses can be compressed using gzip. 
+ 			The requesting user-agent (a web browser, another YaCy peer or any other tool) uses the header 'Accept-Encoding' to tell whether it accepts gzip compression or not. 
+ 			This adds some processing overhead, but can significantly reduce the amount of bytes transmitted over the network. 
+        </div>
+	</div>
+	<div class="form-group">
+    	<div class="col-sm-12">
+    		<input type="submit" class="btn btn-primary" name="serverCompression" value="Submit" aria-describedby="submitInfo"/>
+			<em id="submitInfo">Changes need a server restart.</em>
+    	</div>
+    </div>
+</fieldset>
+
 </form>

htroot/Settings_p.java

@@ -135,6 +135,11 @@ public final class Settings_p {
         // server access filter
         prop.putHTML("serverfilter", env.getConfig("serverClient", "*"));
         
+        /* gzip compression of HTTP responses */
+		prop.put(SwitchboardConstants.SERVER_RESPONSE_COMPRESS_GZIP,
+				env.getConfigBool(SwitchboardConstants.SERVER_RESPONSE_COMPRESS_GZIP,
+						SwitchboardConstants.SERVER_RESPONSE_COMPRESS_GZIP_DEFAULT));
+        
         // server password
         prop.put("serveruser","server");
         

source/net/yacy/http/Jetty9HttpServerImpl.java

@@ -48,6 +48,7 @@ import org.eclipse.jetty.server.handler.ContextHandlerCollection;
 import org.eclipse.jetty.server.handler.DefaultHandler;
 import org.eclipse.jetty.server.handler.HandlerList;
 import org.eclipse.jetty.server.handler.InetAccessHandler;
+import org.eclipse.jetty.server.handler.gzip.GzipHandler;
 import org.eclipse.jetty.servlet.ServletHolder;
 import org.eclipse.jetty.util.resource.Resource;
 import org.eclipse.jetty.util.ssl.SslContextFactory;
@@ -143,6 +144,19 @@ public class Jetty9HttpServerImpl implements YaCyHttpServer {
         //sholder.setInitParameter("welcomeFile", "index.html"); // default is index.html, welcome.html
         htrootContext.addServlet(sholder, "/*");
         
+        /* Handle gzip compression of responses to user agents accepting it */
+		final GzipHandler gzipHandler;
+		if (sb.getConfigBool(SwitchboardConstants.SERVER_RESPONSE_COMPRESS_GZIP,
+				SwitchboardConstants.SERVER_RESPONSE_COMPRESS_GZIP_DEFAULT)) {
+			gzipHandler = new GzipHandler();
+			/*
+			 * Ensure decompression of requests body is disabled : it is already handled by
+			 * the GZIPRequestWrapper in the YaCyDefaultServlet
+			 */
+			gzipHandler.setInflateBufferSize(0);
+			htrootContext.setGzipHandler(gzipHandler);
+		}
+
         // -----------------------------------------------------------------------------
         // here we set and map the mandatory servlets, needed for typical YaCy operation
         // to make sure they are available even if removed in individual web.xml

source/net/yacy/search/SwitchboardConstants.java

@@ -485,6 +485,12 @@ public final class SwitchboardConstants {
     /** Setting key of the property that collects the names of all servlets that have been used so far. */
     public static final String SERVER_SERVLETS_CALLED    = "server.servlets.called";
     
+    /** Key of the setting controlling whether HTTP responses should be compressed with gzip when the user-agent accepts it (by including gzip in a 'Accept-Encoding' HTTP request header) */
+    public static final String SERVER_RESPONSE_COMPRESS_GZIP = "server.response.compress.gzip";
+    
+    /** Default setting value controlling whether HTTP responses should be compressed */
+    public static final boolean SERVER_RESPONSE_COMPRESS_GZIP_DEFAULT = true;
+
 
     /*
      * ResourceObserver