boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

use DigestURL in UrlProxyServlet as parameter to pass requested url to

Browse Source 
handler.
UrlProxyServlet splits url in parts to pass it on as parameter and 
HeaderFramework constructs a url from param parts. This is obsolete if
already created url is used (makes HeaderFramework.getRequestURL obsolete
= removed)
pull/93/head
reger 8 years ago
parent 3630fcc458
commit 4eeb448eb3
5 changed files with 44 additions and 116 deletions
Show Stats Download Patch File Download Diff File

36
source/net/yacy/cora/protocol/HeaderFramework.java
Unescape View File

@ -24,12 +24,10 @@ import java.io.BufferedReader;
import java.io.File; import java.io.File;
import java.io.FileOutputStream; import java.io.FileOutputStream;
import java.io.IOException; import java.io.IOException;
import java.net.MalformedURLException;
import java.text.ParseException; import java.text.ParseException;
import java.text.SimpleDateFormat; import java.text.SimpleDateFormat;
import java.util.Calendar; import java.util.Calendar;
import java.util.Date; import java.util.Date;
import java.util.HashMap;
import java.util.Iterator; import java.util.Iterator;
import java.util.Locale; import java.util.Locale;
import java.util.Map; import java.util.Map;
@ -40,10 +38,8 @@ import java.util.concurrent.ConcurrentHashMap;
import net.yacy.cora.document.encoding.ASCII; import net.yacy.cora.document.encoding.ASCII;
import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.document.encoding.UTF8;
import net.yacy.cora.document.id.DigestURL;
import net.yacy.cora.util.CommonPattern; import net.yacy.cora.util.CommonPattern;
import net.yacy.cora.util.ConcurrentLog; import net.yacy.cora.util.ConcurrentLog;
import net.yacy.cora.util.NumberTools;
/** /**
@ -200,7 +196,6 @@ public class HeaderFramework extends TreeMap<String, String> implements Map<Stri
public static final String CONNECTION_PROP_METHOD = "METHOD"; public static final String CONNECTION_PROP_METHOD = "METHOD";
public static final String CONNECTION_PROP_PATH = "PATH"; public static final String CONNECTION_PROP_PATH = "PATH";
public static final String CONNECTION_PROP_EXT = "EXT"; public static final String CONNECTION_PROP_EXT = "EXT";
public static final String CONNECTION_PROP_URL = "URL";
public static final String CONNECTION_PROP_ARGS = "ARGS"; public static final String CONNECTION_PROP_ARGS = "ARGS";
public static final String CONNECTION_PROP_CLIENTIP = "CLIENTIP"; public static final String CONNECTION_PROP_CLIENTIP = "CLIENTIP";
public static final String CONNECTION_PROP_PERSISTENT = "PERSISTENT"; public static final String CONNECTION_PROP_PERSISTENT = "PERSISTENT";
@ -208,6 +203,7 @@ public class HeaderFramework extends TreeMap<String, String> implements Map<Stri
public static final String CONNECTION_PROP_REQUEST_END = "REQUEST_END"; public static final String CONNECTION_PROP_REQUEST_END = "REQUEST_END";
/* PROPERTIES: Client -> Proxy */ /* PROPERTIES: Client -> Proxy */
public static final String CONNECTION_PROP_DIGESTURL = "URL"; // value DigestURL object
public static final String CONNECTION_PROP_CLIENT_HTTPSERVLETREQUEST = "CLIENT_HTTPSERVLETREQUEST"; public static final String CONNECTION_PROP_CLIENT_HTTPSERVLETREQUEST = "CLIENT_HTTPSERVLETREQUEST";
/* PROPERTIES: Proxy -> Client */ /* PROPERTIES: Proxy -> Client */
@ -565,36 +561,6 @@ public class HeaderFramework extends TreeMap<String, String> implements Map<Stri
theHeader.append("\r\n"); theHeader.append("\r\n");
} }
/**
* Generate a url from Header properties
* @param conProp containing host, path, query and protocol (defaults to http if missing)
* @return url
* @throws MalformedURLException
*/
public static DigestURL getRequestURL(final HashMap<String, Object> conProp) throws MalformedURLException {
String host = (String) conProp.get(HeaderFramework.CONNECTION_PROP_HOST);
final String path = (String) conProp.get(HeaderFramework.CONNECTION_PROP_PATH); // always starts with leading '/'
final String args = (String) conProp.get(HeaderFramework.CONNECTION_PROP_ARGS); // may be null if no args were given
String protocol = (String) conProp.get(HeaderFramework.CONNECTION_PROP_PROTOCOL);
if (protocol == null) protocol = "http";
//String ip = conProp.getProperty(httpHeader.CONNECTION_PROP_CLIENTIP); // the ip from the connecting peer
int port, pos;
if ((pos = host.lastIndexOf(':')) < 0) {
port = Domains.stripToPort(protocol + "://" + host); // use stripToPort to get default ports
} else {
if (pos > host.indexOf(']')) { // check for ipv6
port = NumberTools.parseIntDecSubstring(host, pos + 1);
host = host.substring(0, pos);
} else {
port = Domains.stripToPort(protocol + "://" + host); // use stripToPort to get default ports
}
}
final DigestURL url = new DigestURL(protocol, host, port, (args == null) ? path : path + "?" + args);
return url;
}
/** /**
* Reading http headers from a reader class and building up a httpHeader object * Reading http headers from a reader class and building up a httpHeader object
* @param reader the {@link BufferedReader} that is used to read the http header lines * @param reader the {@link BufferedReader} that is used to read the http header lines

12
source/net/yacy/http/servlets/UrlProxyServlet.java
Unescape View File

@ -5,7 +5,6 @@ import java.io.ByteArrayOutputStream;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.net.MalformedURLException; import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLDecoder; import java.net.URLDecoder;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.util.HashMap; import java.util.HashMap;
@ -122,7 +121,7 @@ public class UrlProxyServlet extends HttpServlet implements Servlet {
return; return;
} }
// 2 - get target url // 2 - get target url
URL proxyurl = null; DigestURL proxyurl = null;
final String strUrl = request.getParameter("url"); final String strUrl = request.getParameter("url");
if (strUrl == null) { if (strUrl == null) {
response.sendError(HttpServletResponse.SC_NOT_FOUND,"url parameter missing"); response.sendError(HttpServletResponse.SC_NOT_FOUND,"url parameter missing");
@ -130,9 +129,9 @@ public class UrlProxyServlet extends HttpServlet implements Servlet {
} }
try { try {
proxyurl = new URL(strUrl); proxyurl = new DigestURL(strUrl);
} catch (final MalformedURLException e) { } catch (final MalformedURLException e) {
proxyurl = new URL(URLDecoder.decode(strUrl, StandardCharsets.UTF_8.name())); proxyurl = new DigestURL(URLDecoder.decode(strUrl, StandardCharsets.UTF_8.name()));
} }
if (proxyurl == null) { if (proxyurl == null) {
@ -151,11 +150,8 @@ public class UrlProxyServlet extends HttpServlet implements Servlet {
final HashMap<String, Object> prop = new HashMap<String, Object>(); final HashMap<String, Object> prop = new HashMap<String, Object>();
prop.put(HeaderFramework.CONNECTION_PROP_HTTP_VER, HeaderFramework.HTTP_VERSION_1_1); prop.put(HeaderFramework.CONNECTION_PROP_HTTP_VER, HeaderFramework.HTTP_VERSION_1_1);
prop.put(HeaderFramework.CONNECTION_PROP_PROTOCOL, proxyurl.getProtocol()); prop.put(HeaderFramework.CONNECTION_PROP_DIGESTURL, proxyurl);
prop.put(HeaderFramework.CONNECTION_PROP_HOST, hostwithport);
prop.put(HeaderFramework.CONNECTION_PROP_PATH, proxyurl.getPath().replaceAll(" ", "%20"));
prop.put(HeaderFramework.CONNECTION_PROP_METHOD, request.getMethod()); // only needed for HTTPDeamon errormsg in case of blacklisted url prop.put(HeaderFramework.CONNECTION_PROP_METHOD, request.getMethod()); // only needed for HTTPDeamon errormsg in case of blacklisted url
if (proxyurl.getQuery() != null) prop.put(HeaderFramework.CONNECTION_PROP_ARGS, proxyurl.getQuery());
prop.put(HeaderFramework.CONNECTION_PROP_CLIENTIP, Domains.LOCALHOST); prop.put(HeaderFramework.CONNECTION_PROP_CLIENTIP, Domains.LOCALHOST);
prop.put(HeaderFramework.CONNECTION_PROP_CLIENT_HTTPSERVLETREQUEST, request); prop.put(HeaderFramework.CONNECTION_PROP_CLIENT_HTTPSERVLETREQUEST, request);

12
source/net/yacy/http/servlets/YaCyProxyServlet.java
Unescape View File

@ -6,7 +6,6 @@ import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.io.StringWriter; import java.io.StringWriter;
import java.net.MalformedURLException; import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLDecoder; import java.net.URLDecoder;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.util.HashMap; import java.util.HashMap;
@ -91,7 +90,7 @@ public class YaCyProxyServlet extends HttpServlet implements Servlet {
response.sendError(HttpServletResponse.SC_GATEWAY_TIMEOUT); // Need better test that isInitial response.sendError(HttpServletResponse.SC_GATEWAY_TIMEOUT); // Need better test that isInitial
return; return;
} }
URL proxyurl = null; DigestURL proxyurl = null;
String strARGS = request.getQueryString(); String strARGS = request.getQueryString();
if (strARGS == null) { if (strARGS == null) {
response.sendError(HttpServletResponse.SC_NOT_FOUND,"url parameter missing"); response.sendError(HttpServletResponse.SC_NOT_FOUND,"url parameter missing");
@ -102,10 +101,9 @@ public class YaCyProxyServlet extends HttpServlet implements Servlet {
final String strUrl = strARGS.substring(4); // strip "url=" final String strUrl = strARGS.substring(4); // strip "url="
try { try {
proxyurl = new URL(strUrl); proxyurl = new DigestURL(strUrl);
} catch (final MalformedURLException e) { } catch (final MalformedURLException e) {
proxyurl = new URL(URLDecoder.decode(strUrl, StandardCharsets.UTF_8.name())); proxyurl = new DigestURL(URLDecoder.decode(strUrl, StandardCharsets.UTF_8.name()));
} }
} }
if (proxyurl == null) { if (proxyurl == null) {
@ -123,9 +121,7 @@ public class YaCyProxyServlet extends HttpServlet implements Servlet {
final HashMap<String, Object> prop = new HashMap<String, Object>(); final HashMap<String, Object> prop = new HashMap<String, Object>();
prop.put(HeaderFramework.CONNECTION_PROP_HTTP_VER, HeaderFramework.HTTP_VERSION_1_1); prop.put(HeaderFramework.CONNECTION_PROP_HTTP_VER, HeaderFramework.HTTP_VERSION_1_1);
prop.put(HeaderFramework.CONNECTION_PROP_PROTOCOL, proxyurl.getProtocol()); prop.put(HeaderFramework.CONNECTION_PROP_DIGESTURL, proxyurl);
prop.put(HeaderFramework.CONNECTION_PROP_HOST, hostwithport);
prop.put(HeaderFramework.CONNECTION_PROP_PATH, proxyurl.getPath().replaceAll(" ", "%20"));
prop.put(HeaderFramework.CONNECTION_PROP_CLIENTIP, Domains.LOCALHOST); prop.put(HeaderFramework.CONNECTION_PROP_CLIENTIP, Domains.LOCALHOST);
prop.put(HeaderFramework.CONNECTION_PROP_CLIENT_HTTPSERVLETREQUEST, request); prop.put(HeaderFramework.CONNECTION_PROP_CLIENT_HTTPSERVLETREQUEST, request);

79
source/net/yacy/server/http/HTTPDProxyHandler.java
Unescape View File

@ -277,42 +277,32 @@ public final class HTTPDProxyHandler {
final String ip = (String) conProp.get(HeaderFramework.CONNECTION_PROP_CLIENTIP); // the ip from the connecting peer final String ip = (String) conProp.get(HeaderFramework.CONNECTION_PROP_CLIENTIP); // the ip from the connecting peer
DigestURL url = null; DigestURL url = (DigestURL) conProp.get(HeaderFramework.CONNECTION_PROP_DIGESTURL);
try { if (log.isFine()) log.fine(reqID +" GET "+ url.toString());
url = HeaderFramework.getRequestURL(conProp); if (log.isFinest()) log.finest(reqID +" header: "+ requestHeader);
if (log.isFine()) log.fine(reqID +" GET "+ url);
if (log.isFinest()) log.finest(reqID +" header: "+ requestHeader); //redirector
if (redirectorEnabled) {
//redirector synchronized (redirectorProcess) {
if (redirectorEnabled){ redirectorWriter.println(url.toNormalform(true));
synchronized(redirectorProcess){ redirectorWriter.flush();
redirectorWriter.println(url.toNormalform(true));
redirectorWriter.flush();
}
final String newUrl = redirectorReader.readLine();
if (!newUrl.equals("")) {
try {
url = new DigestURL(newUrl);
} catch(final MalformedURLException e){}//just keep the old one
}
if (log.isFinest()) log.finest(reqID +" using redirector to "+ url);
conProp.put(HeaderFramework.CONNECTION_PROP_HOST, url.getHost()+":"+url.getPort());
conProp.put(HeaderFramework.CONNECTION_PROP_PATH, url.getPath());
requestHeader.put(HeaderFramework.HOST, url.getHost()+":"+url.getPort());
requestHeader.put(HeaderFramework.CONNECTION_PROP_PATH, url.getPath());
} }
} catch (final MalformedURLException e) { final String newUrl = redirectorReader.readLine();
// get header info for error logging if (!newUrl.equals("")) {
final String host = (String) conProp.get(HeaderFramework.CONNECTION_PROP_HOST); try {
final String path = (String) conProp.get(HeaderFramework.CONNECTION_PROP_PATH); // always starts with leading '/' url = new DigestURL(newUrl);
final String args = (String) conProp.get(HeaderFramework.CONNECTION_PROP_ARGS); // may be null if no args were given } catch (final MalformedURLException e) {
final String errorMsg = "ERROR: internal error with url generation: host=" + }//just keep the old one
host + ", path=" + path + ", args=" + args; }
log.severe(errorMsg); if (log.isFinest()) {
HTTPDemon.sendRespondError(conProp,countedRespond,4,501,null,errorMsg,e); log.finest(reqID + " using redirector to " + url);
return; }
conProp.put(HeaderFramework.CONNECTION_PROP_DIGESTURL, url);
requestHeader.put(HeaderFramework.HOST, url.getHost() + ":" + url.getPort());
requestHeader.put(HeaderFramework.CONNECTION_PROP_PATH, url.getPath());
} }
// check the blacklist // check the blacklist
// blacklist idea inspired by [AS]: // blacklist idea inspired by [AS]:
// respond a 404 for all AGIS ("all you get is shit") servers // respond a 404 for all AGIS ("all you get is shit") servers
@ -1049,14 +1039,13 @@ public final class HTTPDProxyHandler {
})); }));
// getting some connection properties // getting some connection properties
String orgHostPort = "80"; DigestURL orgurl = (DigestURL) conProp.get(HeaderFramework.CONNECTION_PROP_DIGESTURL);
String orgHostName = (String) conProp.get(HeaderFramework.CONNECTION_PROP_HOST); int orgHostPort = orgurl.getPort();
String orgHostName = orgurl.getHost();
if (orgHostName == null) orgHostName = "unknown"; if (orgHostName == null) orgHostName = "unknown";
orgHostName = orgHostName.toLowerCase(); orgHostName = orgHostName.toLowerCase();
orgHostPort = Integer.toString(Domains.stripToPort(orgHostName)); String orgHostPath = orgurl.getPath(); if (orgHostPath == null) orgHostPath = "";
orgHostName = Domains.stripToHostName(orgHostName); String orgHostArgs = orgurl.getSearchpart();; if (orgHostArgs == null) orgHostArgs = "";
String orgHostPath = (String) conProp.get(HeaderFramework.CONNECTION_PROP_PATH); if (orgHostPath == null) orgHostPath = "";
String orgHostArgs = (String) conProp.get(HeaderFramework.CONNECTION_PROP_ARGS); if (orgHostArgs == null) orgHostArgs = "";
if (orgHostArgs.length() > 0) orgHostArgs = "?" + orgHostArgs; if (orgHostArgs.length() > 0) orgHostArgs = "?" + orgHostArgs;
detailedErrorMsgMap.put("hostName", orgHostName); detailedErrorMsgMap.put("hostName", orgHostName);
@ -1183,13 +1172,8 @@ public final class HTTPDProxyHandler {
logMessage.append(' '); logMessage.append(' ');
// URL // URL
final String requestURL = (String) conProp.get(HeaderFramework.CONNECTION_PROP_URL); final DigestURL requestURL = (DigestURL) conProp.get(HeaderFramework.CONNECTION_PROP_DIGESTURL);
final String requestArgs = (String) conProp.get(HeaderFramework.CONNECTION_PROP_ARGS); logMessage.append(requestURL.toString());
logMessage.append(requestURL);
if (requestArgs != null) {
logMessage.append("?")
.append(requestArgs);
}
logMessage.append(' '); logMessage.append(' ');
// Rfc931 // Rfc931
@ -1197,9 +1181,8 @@ public final class HTTPDProxyHandler {
logMessage.append(' '); logMessage.append(' ');
// Peerstatus/Peerhost // Peerstatus/Peerhost
final String host = (String) conProp.get(HeaderFramework.CONNECTION_PROP_HOST);
logMessage.append("DIRECT/"); logMessage.append("DIRECT/");
logMessage.append(host); logMessage.append(requestURL.getHost());
logMessage.append(' '); logMessage.append(' ');
// Type // Type

21
source/net/yacy/server/http/HTTPDemon.java
Unescape View File

@ -30,14 +30,12 @@ import java.io.FileInputStream;
import java.io.IOException; import java.io.IOException;
import java.io.OutputStream; import java.io.OutputStream;
import java.io.PrintStream; import java.io.PrintStream;
import java.net.MalformedURLException;
import java.util.Date; import java.util.Date;
import java.util.HashMap; import java.util.HashMap;
import java.util.Iterator; import java.util.Iterator;
import java.util.Map.Entry; import java.util.Map.Entry;
import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.document.encoding.UTF8;
import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.document.id.DigestURL;
import net.yacy.cora.protocol.Domains;
import net.yacy.cora.protocol.HeaderFramework; import net.yacy.cora.protocol.HeaderFramework;
import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.protocol.RequestHeader;
import net.yacy.cora.protocol.ResponseHeader; import net.yacy.cora.protocol.ResponseHeader;
@ -138,20 +136,9 @@ public final class HTTPDemon {
} }
// generating the desired request url // generating the desired request url
String host = (String) conProp.get(HeaderFramework.CONNECTION_PROP_HOST);
String path = (String) conProp.get(HeaderFramework.CONNECTION_PROP_PATH); if (path == null) path = "/";
final String args = (String) conProp.get(HeaderFramework.CONNECTION_PROP_ARGS);
final String method = (String) conProp.get(HeaderFramework.CONNECTION_PROP_METHOD);
final int port = Domains.stripToPort(host);
host = Domains.stripToHostName(host);
String urlString; final String method = (String) conProp.get(HeaderFramework.CONNECTION_PROP_METHOD);
try { DigestURL url = (DigestURL) conProp.get(HeaderFramework.CONNECTION_PROP_DIGESTURL);
urlString = (new DigestURL((method.equals(HeaderFramework.METHOD_CONNECT)?"https":"http"), host, port, (args == null) ? path : path + "?" + args)).toString();
} catch (final MalformedURLException e) {
urlString = "invalid URL";
}
// set rewrite values // set rewrite values
final serverObjects tp = new serverObjects(); final serverObjects tp = new serverObjects();
@ -159,8 +146,8 @@ public final class HTTPDemon {
tp.put("peerName", (switchboard.peers == null) ? "" : switchboard.peers.myName()); tp.put("peerName", (switchboard.peers == null) ? "" : switchboard.peers.myName());
tp.put("errorMessageType", Integer.toString(errorcase)); tp.put("errorMessageType", Integer.toString(errorcase));
tp.put("httpStatus", Integer.toString(httpStatusCode) + " " + httpStatusText); tp.put("httpStatus", Integer.toString(httpStatusCode) + " " + httpStatusText);
tp.put("requestMethod", (String) conProp.get(HeaderFramework.CONNECTION_PROP_METHOD)); tp.put("requestMethod", method);
tp.put("requestURL", urlString); tp.put("requestURL", url.toString());
switch (errorcase) { switch (errorcase) {
case ERRORCASE_FILE: case ERRORCASE_FILE:

Write Preview
Loading…
Cancel
Save