boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

use DigestURL in UrlProxyServlet as parameter to pass requested url to

Browse Source 
handler.
UrlProxyServlet splits url in parts to pass it on as parameter and 
HeaderFramework constructs a url from param parts. This is obsolete if
already created url is used (makes HeaderFramework.getRequestURL obsolete
= removed)
pull/93/head
reger 8 years ago
parent 3630fcc458
commit 4eeb448eb3
5 changed files with 44 additions and 116 deletions
Show Stats Download Patch File Download Diff File

36
source/net/yacy/cora/protocol/HeaderFramework.java
Unescape View File

@ -24,12 +24,10 @@ import java.io.BufferedReader;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.MalformedURLException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Locale;
import java.util.Map;
@ -40,10 +38,8 @@ import java.util.concurrent.ConcurrentHashMap;
import net.yacy.cora.document.encoding.ASCII;
import net.yacy.cora.document.encoding.UTF8;
import net.yacy.cora.document.id.DigestURL;
import net.yacy.cora.util.CommonPattern;
import net.yacy.cora.util.ConcurrentLog;
import net.yacy.cora.util.NumberTools;
/**
@ -200,7 +196,6 @@ public class HeaderFramework extends TreeMap<String, String> implements Map<Stri
public static final String CONNECTION_PROP_METHOD = "METHOD";
public static final String CONNECTION_PROP_PATH = "PATH";
public static final String CONNECTION_PROP_EXT = "EXT";
public static final String CONNECTION_PROP_URL = "URL";
public static final String CONNECTION_PROP_ARGS = "ARGS";
public static final String CONNECTION_PROP_CLIENTIP = "CLIENTIP";
public static final String CONNECTION_PROP_PERSISTENT = "PERSISTENT";
@ -208,6 +203,7 @@ public class HeaderFramework extends TreeMap<String, String> implements Map<Stri
public static final String CONNECTION_PROP_REQUEST_END = "REQUEST_END";
/* PROPERTIES: Client -> Proxy */
public static final String CONNECTION_PROP_DIGESTURL = "URL"; // value DigestURL object
public static final String CONNECTION_PROP_CLIENT_HTTPSERVLETREQUEST = "CLIENT_HTTPSERVLETREQUEST";
/* PROPERTIES: Proxy -> Client */
@ -565,36 +561,6 @@ public class HeaderFramework extends TreeMap<String, String> implements Map<Stri
theHeader.append("\r\n");
}
/**
* Generate a url from Header properties
* @param conProp containing host, path, query and protocol (defaults to http if missing)
* @return url
* @throws MalformedURLException
*/
public static DigestURL getRequestURL(final HashMap<String, Object> conProp) throws MalformedURLException {
String host = (String) conProp.get(HeaderFramework.CONNECTION_PROP_HOST);
final String path = (String) conProp.get(HeaderFramework.CONNECTION_PROP_PATH); // always starts with leading '/'
final String args = (String) conProp.get(HeaderFramework.CONNECTION_PROP_ARGS); // may be null if no args were given
String protocol = (String) conProp.get(HeaderFramework.CONNECTION_PROP_PROTOCOL);
if (protocol == null) protocol = "http";
//String ip = conProp.getProperty(httpHeader.CONNECTION_PROP_CLIENTIP); // the ip from the connecting peer
int port, pos;
if ((pos = host.lastIndexOf(':')) < 0) {
port = Domains.stripToPort(protocol + "://" + host); // use stripToPort to get default ports
} else {
if (pos > host.indexOf(']')) { // check for ipv6
port = NumberTools.parseIntDecSubstring(host, pos + 1);
host = host.substring(0, pos);
} else {
port = Domains.stripToPort(protocol + "://" + host); // use stripToPort to get default ports
}
}
final DigestURL url = new DigestURL(protocol, host, port, (args == null) ? path : path + "?" + args);
return url;
}
/**
* Reading http headers from a reader class and building up a httpHeader object
* @param reader the {@link BufferedReader} that is used to read the http header lines

12
source/net/yacy/http/servlets/UrlProxyServlet.java
Unescape View File

@ -5,7 +5,6 @@ import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLDecoder;
import java.nio.charset.StandardCharsets;
import java.util.HashMap;
@ -122,7 +121,7 @@ public class UrlProxyServlet extends HttpServlet implements Servlet {
return;
}
// 2 - get target url
URL proxyurl = null;
DigestURL proxyurl = null;
final String strUrl = request.getParameter("url");
if (strUrl == null) {
response.sendError(HttpServletResponse.SC_NOT_FOUND,"url parameter missing");
@ -130,9 +129,9 @@ public class UrlProxyServlet extends HttpServlet implements Servlet {
}
try {
proxyurl = new URL(strUrl);
proxyurl = new DigestURL(strUrl);
} catch (final MalformedURLException e) {
proxyurl = new URL(URLDecoder.decode(strUrl, StandardCharsets.UTF_8.name()));
proxyurl = new DigestURL(URLDecoder.decode(strUrl, StandardCharsets.UTF_8.name()));
}
if (proxyurl == null) {
@ -151,11 +150,8 @@ public class UrlProxyServlet extends HttpServlet implements Servlet {
final HashMap<String, Object> prop = new HashMap<String, Object>();
prop.put(HeaderFramework.CONNECTION_PROP_HTTP_VER, HeaderFramework.HTTP_VERSION_1_1);
prop.put(HeaderFramework.CONNECTION_PROP_PROTOCOL, proxyurl.getProtocol());
prop.put(HeaderFramework.CONNECTION_PROP_HOST, hostwithport);
prop.put(HeaderFramework.CONNECTION_PROP_PATH, proxyurl.getPath().replaceAll(" ", "%20"));
prop.put(HeaderFramework.CONNECTION_PROP_DIGESTURL, proxyurl);
prop.put(HeaderFramework.CONNECTION_PROP_METHOD, request.getMethod()); // only needed for HTTPDeamon errormsg in case of blacklisted url
if (proxyurl.getQuery() != null) prop.put(HeaderFramework.CONNECTION_PROP_ARGS, proxyurl.getQuery());
prop.put(HeaderFramework.CONNECTION_PROP_CLIENTIP, Domains.LOCALHOST);
prop.put(HeaderFramework.CONNECTION_PROP_CLIENT_HTTPSERVLETREQUEST, request);

12
source/net/yacy/http/servlets/YaCyProxyServlet.java
Unescape View File

@ -6,7 +6,6 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.StringWriter;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLDecoder;
import java.nio.charset.StandardCharsets;
import java.util.HashMap;
@ -91,7 +90,7 @@ public class YaCyProxyServlet extends HttpServlet implements Servlet {
response.sendError(HttpServletResponse.SC_GATEWAY_TIMEOUT); // Need better test that isInitial
return;
}
URL proxyurl = null;
DigestURL proxyurl = null;
String strARGS = request.getQueryString();
if (strARGS == null) {
response.sendError(HttpServletResponse.SC_NOT_FOUND,"url parameter missing");
@ -102,10 +101,9 @@ public class YaCyProxyServlet extends HttpServlet implements Servlet {
final String strUrl = strARGS.substring(4); // strip "url="
try {
proxyurl = new URL(strUrl);
proxyurl = new DigestURL(strUrl);
} catch (final MalformedURLException e) {
proxyurl = new URL(URLDecoder.decode(strUrl, StandardCharsets.UTF_8.name()));
proxyurl = new DigestURL(URLDecoder.decode(strUrl, StandardCharsets.UTF_8.name()));
}
}
if (proxyurl == null) {
@ -123,9 +121,7 @@ public class YaCyProxyServlet extends HttpServlet implements Servlet {
final HashMap<String, Object> prop = new HashMap<String, Object>();
prop.put(HeaderFramework.CONNECTION_PROP_HTTP_VER, HeaderFramework.HTTP_VERSION_1_1);
prop.put(HeaderFramework.CONNECTION_PROP_PROTOCOL, proxyurl.getProtocol());
prop.put(HeaderFramework.CONNECTION_PROP_HOST, hostwithport);
prop.put(HeaderFramework.CONNECTION_PROP_PATH, proxyurl.getPath().replaceAll(" ", "%20"));
prop.put(HeaderFramework.CONNECTION_PROP_DIGESTURL, proxyurl);
prop.put(HeaderFramework.CONNECTION_PROP_CLIENTIP, Domains.LOCALHOST);
prop.put(HeaderFramework.CONNECTION_PROP_CLIENT_HTTPSERVLETREQUEST, request);

51
source/net/yacy/server/http/HTTPDProxyHandler.java
Unescape View File

@ -277,10 +277,8 @@ public final class HTTPDProxyHandler {
final String ip = (String) conProp.get(HeaderFramework.CONNECTION_PROP_CLIENTIP); // the ip from the connecting peer
DigestURL url = null;
try {
url = HeaderFramework.getRequestURL(conProp);
if (log.isFine()) log.fine(reqID +" GET "+ url);
DigestURL url = (DigestURL) conProp.get(HeaderFramework.CONNECTION_PROP_DIGESTURL);
if (log.isFine()) log.fine(reqID +" GET "+ url.toString());
if (log.isFinest()) log.finest(reqID +" header: "+ requestHeader);
//redirector
@ -293,25 +291,17 @@ public final class HTTPDProxyHandler {
if (!newUrl.equals("")) {
try {
url = new DigestURL(newUrl);
} catch(final MalformedURLException e){}//just keep the old one
} catch (final MalformedURLException e) {
}//just keep the old one
}
if (log.isFinest()) {
log.finest(reqID + " using redirector to " + url);
}
if (log.isFinest()) log.finest(reqID +" using redirector to "+ url);
conProp.put(HeaderFramework.CONNECTION_PROP_HOST, url.getHost()+":"+url.getPort());
conProp.put(HeaderFramework.CONNECTION_PROP_PATH, url.getPath());
conProp.put(HeaderFramework.CONNECTION_PROP_DIGESTURL, url);
requestHeader.put(HeaderFramework.HOST, url.getHost() + ":" + url.getPort());
requestHeader.put(HeaderFramework.CONNECTION_PROP_PATH, url.getPath());
}
} catch (final MalformedURLException e) {
// get header info for error logging
final String host = (String) conProp.get(HeaderFramework.CONNECTION_PROP_HOST);
final String path = (String) conProp.get(HeaderFramework.CONNECTION_PROP_PATH); // always starts with leading '/'
final String args = (String) conProp.get(HeaderFramework.CONNECTION_PROP_ARGS); // may be null if no args were given
final String errorMsg = "ERROR: internal error with url generation: host=" +
host + ", path=" + path + ", args=" + args;
log.severe(errorMsg);
HTTPDemon.sendRespondError(conProp,countedRespond,4,501,null,errorMsg,e);
return;
}
// check the blacklist
// blacklist idea inspired by [AS]:
@ -1049,14 +1039,13 @@ public final class HTTPDProxyHandler {
}));
// getting some connection properties
String orgHostPort = "80";
String orgHostName = (String) conProp.get(HeaderFramework.CONNECTION_PROP_HOST);
DigestURL orgurl = (DigestURL) conProp.get(HeaderFramework.CONNECTION_PROP_DIGESTURL);
int orgHostPort = orgurl.getPort();
String orgHostName = orgurl.getHost();
if (orgHostName == null) orgHostName = "unknown";
orgHostName = orgHostName.toLowerCase();
orgHostPort = Integer.toString(Domains.stripToPort(orgHostName));
orgHostName = Domains.stripToHostName(orgHostName);
String orgHostPath = (String) conProp.get(HeaderFramework.CONNECTION_PROP_PATH); if (orgHostPath == null) orgHostPath = "";
String orgHostArgs = (String) conProp.get(HeaderFramework.CONNECTION_PROP_ARGS); if (orgHostArgs == null) orgHostArgs = "";
String orgHostPath = orgurl.getPath(); if (orgHostPath == null) orgHostPath = "";
String orgHostArgs = orgurl.getSearchpart();; if (orgHostArgs == null) orgHostArgs = "";
if (orgHostArgs.length() > 0) orgHostArgs = "?" + orgHostArgs;
detailedErrorMsgMap.put("hostName", orgHostName);
@ -1183,13 +1172,8 @@ public final class HTTPDProxyHandler {
logMessage.append(' ');
// URL
final String requestURL = (String) conProp.get(HeaderFramework.CONNECTION_PROP_URL);
final String requestArgs = (String) conProp.get(HeaderFramework.CONNECTION_PROP_ARGS);
logMessage.append(requestURL);
if (requestArgs != null) {
logMessage.append("?")
.append(requestArgs);
}
final DigestURL requestURL = (DigestURL) conProp.get(HeaderFramework.CONNECTION_PROP_DIGESTURL);
logMessage.append(requestURL.toString());
logMessage.append(' ');
// Rfc931
@ -1197,9 +1181,8 @@ public final class HTTPDProxyHandler {
logMessage.append(' ');
// Peerstatus/Peerhost
final String host = (String) conProp.get(HeaderFramework.CONNECTION_PROP_HOST);
logMessage.append("DIRECT/");
logMessage.append(host);
logMessage.append(requestURL.getHost());
logMessage.append(' ');
// Type

21
source/net/yacy/server/http/HTTPDemon.java
Unescape View File

@ -30,14 +30,12 @@ import java.io.FileInputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.io.PrintStream;
import java.net.MalformedURLException;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map.Entry;
import net.yacy.cora.document.encoding.UTF8;
import net.yacy.cora.document.id.DigestURL;
import net.yacy.cora.protocol.Domains;
import net.yacy.cora.protocol.HeaderFramework;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.cora.protocol.ResponseHeader;
@ -138,20 +136,9 @@ public final class HTTPDemon {
}
// generating the desired request url
String host = (String) conProp.get(HeaderFramework.CONNECTION_PROP_HOST);
String path = (String) conProp.get(HeaderFramework.CONNECTION_PROP_PATH); if (path == null) path = "/";
final String args = (String) conProp.get(HeaderFramework.CONNECTION_PROP_ARGS);
final String method = (String) conProp.get(HeaderFramework.CONNECTION_PROP_METHOD);
final int port = Domains.stripToPort(host);
host = Domains.stripToHostName(host);
String urlString;
try {
urlString = (new DigestURL((method.equals(HeaderFramework.METHOD_CONNECT)?"https":"http"), host, port, (args == null) ? path : path + "?" + args)).toString();
} catch (final MalformedURLException e) {
urlString = "invalid URL";
}
final String method = (String) conProp.get(HeaderFramework.CONNECTION_PROP_METHOD);
DigestURL url = (DigestURL) conProp.get(HeaderFramework.CONNECTION_PROP_DIGESTURL);
// set rewrite values
final serverObjects tp = new serverObjects();
@ -159,8 +146,8 @@ public final class HTTPDemon {
tp.put("peerName", (switchboard.peers == null) ? "" : switchboard.peers.myName());
tp.put("errorMessageType", Integer.toString(errorcase));
tp.put("httpStatus", Integer.toString(httpStatusCode) + " " + httpStatusText);
tp.put("requestMethod", (String) conProp.get(HeaderFramework.CONNECTION_PROP_METHOD));
tp.put("requestURL", urlString);
tp.put("requestMethod", method);
tp.put("requestURL", url.toString());
switch (errorcase) {
case ERRORCASE_FILE:

Write Preview
Loading…
Cancel
Save