From d9adc2c255ccf09938b24b2ec23894dc4054ceab Mon Sep 17 00:00:00 2001
From: reger <reger18@arcor.de>
Date: Fri, 25 Mar 2016 05:26:48 +0100
Subject: [PATCH] load handler for Transparent Proxy on startup only if feature
is activated to save the resources and keep handler chain small if the
feature is not used. +add a warning message on settingsack_p page to restart
on first activation
---
htroot/SettingsAck_p.html | 6 +++++
htroot/SettingsAck_p.java | 6 +++--
htroot/Settings_ProxyAccess.inc | 2 +-
htroot/Settings_p.java | 3 ++-
htroot/Status.java | 2 +-
htroot/api/snapshot.java | 5 +++--
locales/de.lng | 1 +
.../net/yacy/http/AbstractRemoteHandler.java | 3 ++-
.../net/yacy/http/Jetty9HttpServerImpl.java | 10 ++++++---
source/net/yacy/http/ProxyCacheHandler.java | 8 +++----
source/net/yacy/http/ProxyHandler.java | 18 ++-------------
.../yacy/http/servlets/UrlProxyServlet.java | 3 +--
.../http/servlets/YaCyDefaultServlet.java | 22 +++++++++++++++++--
.../yacy/http/servlets/YaCyProxyServlet.java | 3 +--
source/net/yacy/search/Switchboard.java | 2 +-
.../net/yacy/search/SwitchboardConstants.java | 3 ++-
16 files changed, 58 insertions(+), 39 deletions(-)
diff --git a/htroot/SettingsAck_p.html b/htroot/SettingsAck_p.html
index 4530a40a4..7b80cde3d 100644
--- a/htroot/SettingsAck_p.html
+++ b/htroot/SettingsAck_p.html
@@ -190,7 +190,13 @@
<p>the change will take effect after restart.</p>
<p>Note: the SSL option must be switched on, see <a href="ConfigBasic.html">Basic Configuration</a></p>
#(/info)#
+ <p></p>
+ #(needsRestart)#
<p>You can now go back to the <a href="#[referer]#">Settings</a> page if you want to make more changes.</p>
+ ::
+ <p class="warning">Your need to restart YaCy to activate the changes.</p>
+ <p>or go back to the <a href="#[referer]#">Settings</a> page to make more changes.</p>
+ #(/needsRestart)#
#%env/templates/footer.template%#
</body>
</html>
diff --git a/htroot/SettingsAck_p.java b/htroot/SettingsAck_p.java
index f5ed41553..6334e74cb 100644
--- a/htroot/SettingsAck_p.java
+++ b/htroot/SettingsAck_p.java
@@ -57,7 +57,8 @@ public class SettingsAck_p {
// get referer for backlink
final MultiProtocolURL referer = header.referer();
- prop.put("referer", (referer == null) ? "Settings_p.html" : referer.toNormalform(true));
+ prop.put("needsRestart_referer", (referer == null) ? "Settings_p.html" : referer.toNormalform(true));
+ prop.put("needsRestart", false);
//if (post == null) System.out.println("POST: NULL"); else System.out.println("POST: " + post.toString());
if (post == null) {
@@ -152,8 +153,9 @@ public class SettingsAck_p {
// set transparent proxy flag
boolean isTransparentProxy = post.containsKey("isTransparentProxy");
- env.setConfig("isTransparentProxy", isTransparentProxy);
+ env.setConfig(SwitchboardConstants.PROXY_TRANSPARENT_PROXY, isTransparentProxy);
prop.put("info_isTransparentProxy", isTransparentProxy ? "on" : "off");
+ if (isTransparentProxy) prop.put("needsRestart", isTransparentProxy);
// set proxyAlwaysFresh flag
boolean proxyAlwaysFresh = post.containsKey("proxyAlwaysFresh");
diff --git a/htroot/Settings_ProxyAccess.inc b/htroot/Settings_ProxyAccess.inc
index 373e97fbf..db8c97178 100644
--- a/htroot/Settings_ProxyAccess.inc
+++ b/htroot/Settings_ProxyAccess.inc
@@ -32,7 +32,7 @@
<td>Specifies if the proxy should send the X-Forwarded-For http header.</td>
</tr>
<tr valign="top">
- <td colspan="3"><input type="submit" class="btn btn-primary btn-sm" name="httpNetworking" value="Submit" /> <em>Changes will take effect immediately.</em></td>
+ <td colspan="3"><input type="submit" class="btn btn-primary btn-sm" name="httpNetworking" value="Submit" /></td>
</tr>
</table>
</fieldset>
diff --git a/htroot/Settings_p.java b/htroot/Settings_p.java
index 0c03defbb..8453cc87d 100644
--- a/htroot/Settings_p.java
+++ b/htroot/Settings_p.java
@@ -31,6 +31,7 @@ import net.yacy.peers.Network;
import net.yacy.peers.Seed;
import net.yacy.peers.operation.yacySeedUploader;
import net.yacy.search.Switchboard;
+import net.yacy.search.SwitchboardConstants;
import net.yacy.server.serverObjects;
import net.yacy.server.serverSwitch;
@@ -83,7 +84,7 @@ public final class Settings_p {
prop.putHTML("peerLang", peerLang);
// http networking settings
- prop.put("isTransparentProxy", env.getConfigBool("isTransparentProxy", false) ? "1" : "0");
+ prop.put("isTransparentProxy", env.getConfigBool(SwitchboardConstants.PROXY_TRANSPARENT_PROXY, false) ? "1" : "0");
prop.put("proxyAlwaysFresh", env.getConfigBool("proxyAlwaysFresh", false) ? "1" : "0");
prop.put("proxy.sendViaHeader", env.getConfigBool("proxy.sendViaHeader", false) ? "1" : "0");
prop.put("proxy.sendXForwardedForHeader", env.getConfigBool("proxy.sendXForwardedForHeader", true) ? "1" : "0");
diff --git a/htroot/Status.java b/htroot/Status.java
index 0438c6ba6..9119a9ad8 100644
--- a/htroot/Status.java
+++ b/htroot/Status.java
@@ -207,7 +207,7 @@ public class Status
} else {
prop.put("remoteProxy", "0"); // not used
}
- prop.put("info_isTransparentProxy", sb.getConfigBool("isTransparentProxy", false) ? "0" : "1");
+ prop.put("info_isTransparentProxy", sb.getConfigBool(SwitchboardConstants.PROXY_TRANSPARENT_PROXY, false) ? "0" : "1");
prop.put("info_proxyURL", sb.getConfigBool("proxyURL", false) ? "0" : "1");
// peer information
diff --git a/htroot/api/snapshot.java b/htroot/api/snapshot.java
index 2a32c529d..86fc23602 100644
--- a/htroot/api/snapshot.java
+++ b/htroot/api/snapshot.java
@@ -50,6 +50,7 @@ import net.yacy.document.ImageParser;
import net.yacy.kelondro.util.FileUtils;
import net.yacy.peers.graphics.EncodedImage;
import net.yacy.search.Switchboard;
+import net.yacy.search.SwitchboardConstants;
import net.yacy.server.serverObjects;
import net.yacy.server.serverSwitch;
@@ -257,10 +258,10 @@ public class snapshot {
SolrDocument sd = sb.index.fulltext().getMetadata(durl.hash());
boolean success = false;
if (sd == null) {
- success = Transactions.store(durl, new Date(), 99, false, true, sb.getConfigBool("isTransparentProxy", false) ? "http://127.0.0.1:" + sb.getConfigInt("port", 8090) : null, sb.getConfig("crawler.http.acceptLanguage", null));
+ success = Transactions.store(durl, new Date(), 99, false, true, sb.getConfigBool(SwitchboardConstants.PROXY_TRANSPARENT_PROXY, false) ? "http://127.0.0.1:" + sb.getConfigInt("port", 8090) : null, sb.getConfig("crawler.http.acceptLanguage", null));
} else {
SolrInputDocument sid = sb.index.fulltext().getDefaultConfiguration().toSolrInputDocument(sd);
- success = Transactions.store(sid, false, true, true, sb.getConfigBool("isTransparentProxy", false) ? "http://127.0.0.1:" + sb.getConfigInt("port", 8090) : null, sb.getConfig("crawler.http.acceptLanguage", null));
+ success = Transactions.store(sid, false, true, true, sb.getConfigBool(SwitchboardConstants.PROXY_TRANSPARENT_PROXY, false) ? "http://127.0.0.1:" + sb.getConfigInt("port", 8090) : null, sb.getConfig("crawler.http.acceptLanguage", null));
}
if (success) {
pdfSnapshots = Transactions.findPaths(durl, "pdf", Transactions.State.ANY);
diff --git a/locales/de.lng b/locales/de.lng
index f1936ad9a..652e82d9c 100644
--- a/locales/de.lng
+++ b/locales/de.lng
@@ -3061,6 +3061,7 @@ After a short while you should see the effect on the==Nach kurzer Zeit können S
You can now go back to the==Sie können nun zurück zu den
Settings</a> page if you want to make more changes.==Einstellungen</a> gehen, um weitere Änderungen vorzunehmen.
You can reach your YaCy server under the new location==Dieser YaCy-Peer kann nun unter seiner neuen Adresse erreicht werden:
+Your need to restart YaCy to activate the changes.==Um die Änderungen zu aktivieren müssen Sie YaCy neu starten.
#-----------------------------
#File: Settings_MessageForwarding.inc
diff --git a/source/net/yacy/http/AbstractRemoteHandler.java b/source/net/yacy/http/AbstractRemoteHandler.java
index a9fa89b99..e9bbf9819 100644
--- a/source/net/yacy/http/AbstractRemoteHandler.java
+++ b/source/net/yacy/http/AbstractRemoteHandler.java
@@ -38,6 +38,7 @@ import net.yacy.cora.protocol.Domains;
import net.yacy.cora.protocol.HeaderFramework;
import net.yacy.repository.Blacklist.BlacklistType;
import net.yacy.search.Switchboard;
+import net.yacy.search.SwitchboardConstants;
import org.eclipse.jetty.proxy.ConnectHandler;
import org.eclipse.jetty.server.Handler;
@@ -125,7 +126,7 @@ abstract public class AbstractRemoteHandler extends ConnectHandler implements Ha
// from here we can assume it is a proxy request
// should check proxy use permission
- if (!Switchboard.getSwitchboard().getConfigBool("isTransparentProxy", false)) {
+ if (!Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.PROXY_TRANSPARENT_PROXY, false)) {
// transparent proxy not swiched on
response.sendError(HttpServletResponse.SC_FORBIDDEN,"proxy use not allowed (see System Administration -> Advanced Settings -> Proxy Access Settings -> Transparent Proxy; switched off).");
baseRequest.setHandled(true);
diff --git a/source/net/yacy/http/Jetty9HttpServerImpl.java b/source/net/yacy/http/Jetty9HttpServerImpl.java
index 431a41325..f31747951 100644
--- a/source/net/yacy/http/Jetty9HttpServerImpl.java
+++ b/source/net/yacy/http/Jetty9HttpServerImpl.java
@@ -163,9 +163,13 @@ public class Jetty9HttpServerImpl implements YaCyHttpServer {
// define list of YaCy specific general handlers
HandlerList handlers = new HandlerList();
- handlers.setHandlers(new Handler[]
- {new MonitorHandler(), domainHandler, new ProxyCacheHandler(), new ProxyHandler()});
-
+ if (sb.getConfigBool(SwitchboardConstants.PROXY_TRANSPARENT_PROXY, false)) {
+ // Proxyhandlers are only needed if feature activated (save resources if not used)
+ ConcurrentLog.info("SERVER", "load Jetty handler for transparent proxy");
+ handlers.setHandlers(new Handler[]{new MonitorHandler(), domainHandler, new ProxyCacheHandler(), new ProxyHandler()});
+ } else {
+ handlers.setHandlers(new Handler[]{new MonitorHandler(), domainHandler});
+ }
// context handler for dispatcher and security (hint: dispatcher requires a context)
ContextHandler context = new ContextHandler();
context.setServer(server);
diff --git a/source/net/yacy/http/ProxyCacheHandler.java b/source/net/yacy/http/ProxyCacheHandler.java
index 50a200025..5a369c545 100644
--- a/source/net/yacy/http/ProxyCacheHandler.java
+++ b/source/net/yacy/http/ProxyCacheHandler.java
@@ -34,13 +34,13 @@ import javax.servlet.http.HttpServletResponse;
import net.yacy.cora.document.id.DigestURL;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.cora.protocol.ResponseHeader;
+import net.yacy.crawler.data.Cache;
+import net.yacy.crawler.retrieval.Response;
+import net.yacy.http.servlets.YaCyDefaultServlet;
import org.eclipse.jetty.server.Handler;
import org.eclipse.jetty.server.Request;
-import net.yacy.crawler.data.Cache;
-import net.yacy.crawler.retrieval.Response;
-
/**
* jetty http handler serves pages from cache if available and valid
*/
@@ -65,7 +65,7 @@ public class ProxyCacheHandler extends AbstractRemoteHandler implements Handler
ResponseHeader cachedResponseHeader = Cache.getResponseHeader(url.hash());
if (cachedResponseHeader != null) {
- RequestHeader proxyHeaders = ProxyHandler.convertHeaderFromJetty(request);
+ RequestHeader proxyHeaders = YaCyDefaultServlet.convertHeaderFromJetty(request);
// TODO: this convertion is only necessary
final net.yacy.crawler.retrieval.Request yacyRequest = new net.yacy.crawler.retrieval.Request(
null,
diff --git a/source/net/yacy/http/ProxyHandler.java b/source/net/yacy/http/ProxyHandler.java
index d558aac0a..3b48f187b 100644
--- a/source/net/yacy/http/ProxyHandler.java
+++ b/source/net/yacy/http/ProxyHandler.java
@@ -30,7 +30,6 @@ import java.io.IOException;
import java.io.OutputStream;
import java.net.SocketException;
import java.util.Date;
-import java.util.Enumeration;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
@@ -47,6 +46,7 @@ import net.yacy.cora.protocol.http.HTTPClient;
import net.yacy.document.TextParser;
import net.yacy.crawler.data.Cache;
import net.yacy.crawler.retrieval.Response;
+import net.yacy.http.servlets.YaCyDefaultServlet;
import net.yacy.server.http.HTTPDProxyHandler;
import net.yacy.server.http.MultiOutputStream;
@@ -69,20 +69,6 @@ public class ProxyHandler extends AbstractRemoteHandler implements Handler {
timeout = sb.getConfigInt("proxy.clientTimeout", 10000);
}
- public static RequestHeader convertHeaderFromJetty(HttpServletRequest request) {
- RequestHeader result = new RequestHeader();
- Enumeration<String> headerNames = request.getHeaderNames();
- while(headerNames.hasMoreElements()) {
- String headerName = headerNames.nextElement();
- Enumeration<String> headers = request.getHeaders(headerName);
- while(headers.hasMoreElements()) {
- String header = headers.nextElement();
- result.add(headerName, header);
- }
- }
- return result;
- }
-
private void convertHeaderToJetty(HttpResponse in, HttpServletResponse out) {
for(Header h: in.getAllHeaders()) {
out.addHeader(h.getName(), h.getValue());
@@ -143,7 +129,7 @@ public class ProxyHandler extends AbstractRemoteHandler implements Handler {
sb.proxyLastAccess = System.currentTimeMillis();
- RequestHeader proxyHeaders = convertHeaderFromJetty(request);
+ RequestHeader proxyHeaders = YaCyDefaultServlet.convertHeaderFromJetty(request);
setProxyHeaderForClient(request, proxyHeaders);
final HTTPClient client = new HTTPClient(ClientIdentification.yacyProxyAgent);
diff --git a/source/net/yacy/http/servlets/UrlProxyServlet.java b/source/net/yacy/http/servlets/UrlProxyServlet.java
index 911e8903a..826a8cd4d 100644
--- a/source/net/yacy/http/servlets/UrlProxyServlet.java
+++ b/source/net/yacy/http/servlets/UrlProxyServlet.java
@@ -26,7 +26,6 @@ import net.yacy.cora.protocol.HeaderFramework;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.cora.protocol.ResponseHeader;
import net.yacy.cora.util.ConcurrentLog;
-import net.yacy.http.ProxyHandler;
import net.yacy.kelondro.util.FileUtils;
import net.yacy.search.Switchboard;
import net.yacy.server.http.ChunkedInputStream;
@@ -145,7 +144,7 @@ public class UrlProxyServlet extends HttpServlet implements Servlet {
hostwithport += ":" + proxyurl.getPort();
}
// 4 - get target url
- RequestHeader yacyRequestHeader = ProxyHandler.convertHeaderFromJetty(request);
+ RequestHeader yacyRequestHeader = YaCyDefaultServlet.convertHeaderFromJetty(request);
yacyRequestHeader.remove(RequestHeader.KEEP_ALIVE);
yacyRequestHeader.remove(HeaderFramework.CONTENT_LENGTH);
diff --git a/source/net/yacy/http/servlets/YaCyDefaultServlet.java b/source/net/yacy/http/servlets/YaCyDefaultServlet.java
index 8aa63bc86..1bc4b9dc7 100644
--- a/source/net/yacy/http/servlets/YaCyDefaultServlet.java
+++ b/source/net/yacy/http/servlets/YaCyDefaultServlet.java
@@ -68,7 +68,6 @@ import net.yacy.cora.util.ByteBuffer;
import net.yacy.cora.util.ConcurrentLog;
import net.yacy.data.UserDB.AccessRight;
import net.yacy.data.UserDB.Entry;
-import net.yacy.http.ProxyHandler;
import net.yacy.kelondro.util.FileUtils;
import net.yacy.kelondro.util.MemoryControl;
import net.yacy.peers.Seed;
@@ -662,8 +661,27 @@ public class YaCyDefaultServlet extends HttpServlet {
return rewriteMethod(targetClass).invoke(null, new Object[]{request, args, Switchboard.getSwitchboard()}); // add switchboard
}
+ /**
+ * Convert ServletRequest header to YaCy RequestHeader
+ * @param request ServletRequest
+ * @return RequestHeader created from ServletRequest
+ */
+ public static RequestHeader convertHeaderFromJetty(HttpServletRequest request) {
+ RequestHeader result = new RequestHeader();
+ Enumeration<String> headerNames = request.getHeaderNames();
+ while (headerNames.hasMoreElements()) {
+ String headerName = headerNames.nextElement();
+ Enumeration<String> headers = request.getHeaders(headerName);
+ while (headers.hasMoreElements()) {
+ String header = headers.nextElement();
+ result.add(headerName, header);
+ }
+ }
+ return result;
+ }
+
protected RequestHeader generateLegacyRequestHeader(HttpServletRequest request, String target, String targetExt) {
- RequestHeader legacyRequestHeader = ProxyHandler.convertHeaderFromJetty(request);
+ RequestHeader legacyRequestHeader = convertHeaderFromJetty(request);
legacyRequestHeader.put(HeaderFramework.CONNECTION_PROP_CLIENTIP, request.getRemoteAddr());
legacyRequestHeader.put(HeaderFramework.CONNECTION_PROP_PATH, target);
diff --git a/source/net/yacy/http/servlets/YaCyProxyServlet.java b/source/net/yacy/http/servlets/YaCyProxyServlet.java
index 174272bb9..2f5791783 100644
--- a/source/net/yacy/http/servlets/YaCyProxyServlet.java
+++ b/source/net/yacy/http/servlets/YaCyProxyServlet.java
@@ -31,7 +31,6 @@ import net.yacy.cora.protocol.HeaderFramework;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.cora.protocol.ResponseHeader;
import net.yacy.cora.util.ConcurrentLog;
-import net.yacy.http.ProxyHandler;
import net.yacy.kelondro.util.FileUtils;
import net.yacy.search.Switchboard;
import net.yacy.server.http.ChunkedInputStream;
@@ -117,7 +116,7 @@ public class YaCyProxyServlet extends HttpServlet implements Servlet {
if (proxyurl.getPort() != -1) {
hostwithport += ":" + proxyurl.getPort();
}
- RequestHeader yacyRequestHeader = ProxyHandler.convertHeaderFromJetty(request);
+ RequestHeader yacyRequestHeader = YaCyDefaultServlet.convertHeaderFromJetty(request);
yacyRequestHeader.remove(RequestHeader.KEEP_ALIVE);
yacyRequestHeader.remove(HeaderFramework.CONTENT_LENGTH);
diff --git a/source/net/yacy/search/Switchboard.java b/source/net/yacy/search/Switchboard.java
index 1dffc3c3c..dd05b392b 100644
--- a/source/net/yacy/search/Switchboard.java
+++ b/source/net/yacy/search/Switchboard.java
@@ -3025,7 +3025,7 @@ public final class Switchboard extends serverSwitch {
searchEvent,
sourceName,
getConfigBool(SwitchboardConstants.DHT_ENABLED, false),
- this.getConfigBool("isTransparentProxy", false) ? "http://127.0.0.1:" + sb.getConfigInt("port", 8090) : null,
+ this.getConfigBool(SwitchboardConstants.PROXY_TRANSPARENT_PROXY, false) ? "http://127.0.0.1:" + sb.getConfigInt("port", 8090) : null,
this.getConfig("crawler.http.acceptLanguage", null));
final RSSFeed feed =
EventChannel.channels(queueEntry.initiator() == null
diff --git a/source/net/yacy/search/SwitchboardConstants.java b/source/net/yacy/search/SwitchboardConstants.java
index eacb46d7a..57445a767 100644
--- a/source/net/yacy/search/SwitchboardConstants.java
+++ b/source/net/yacy/search/SwitchboardConstants.java
@@ -319,7 +319,8 @@ public final class SwitchboardConstants {
* @see Switchboard#PROXY_CACHE_LAYOUT_HASH
*/
public static final String PROXY_YACY_ONLY = "proxyYacyOnly";
-
+ public static final String PROXY_TRANSPARENT_PROXY = "isTransparentProxy";
+
public static final String AUTOCRAWL = "autocrawl";
public static final String AUTOCRAWL_INDEX_TEXT = "autocrawl.index.text";
public static final String AUTOCRAWL_INDEX_MEDIA = "autocrawl.index.media";