boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

IPv6 host parsing bugfixes

Browse Source
pull/1/head
Michael Peter Christen 10 years ago
parent fe917deb2d
commit 247e626083
10 changed files with 89 additions and 54 deletions
Show Stats Download Patch File Download Diff File

4
htroot/IndexCreateQueues_p.java
Unescape View File

@ -14,6 +14,7 @@ import java.util.regex.PatternSyntaxException;
import net.yacy.cora.document.encoding.ASCII;
import net.yacy.cora.document.id.DigestURL;
import net.yacy.cora.protocol.Domains;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.cora.util.ConcurrentLog;
import net.yacy.crawler.CrawlSwitchboard;
@ -141,8 +142,7 @@ public class IndexCreateQueues_p {
int hc = 0;
for (Map.Entry<String, Integer[]> host: hosts.entrySet()) {
String hostnameport = host.getKey();
int p = hostnameport.lastIndexOf(':');
String hostname = p < 0 ? hostnameport : hostnameport.substring(0, p);
String hostname = Domains.stripToHostName(hostnameport);
prop.putHTML("crawler_host_" + hc + "_hostnameport", hostnameport);
prop.putHTML("crawler_host_" + hc + "_hostname", hostname);
prop.put("crawler_host_" + hc + "_embed", embed ? 1 : 0);

2
htroot/Settings_p.java
Unescape View File

@ -108,7 +108,7 @@ public final class Settings_p {
} else {
prop.put("use_proxyAccounts", "1"); //checked
/*s = env.getConfig("proxyAccount", "proxy:void");
pos = s.indexOf(":");
pos = s.indexOf(':');
if (pos < 0) {
prop.put("proxyuser","proxy");
} else {

2
source/net/yacy/cora/document/id/MultiProtocolURL.java
Unescape View File

@ -155,7 +155,7 @@ public class MultiProtocolURL implements Serializable, Comparable<MultiProtocolU
url = "file://" + url;
}
int p = url.indexOf(':');
int p = url.indexOf("://");
if (p < 0) {
url = "http://" + url;
p = 4;

66
source/net/yacy/cora/protocol/Domains.java
Unescape View File

@ -791,12 +791,78 @@ public class Domains {
final private static TimeLimiter timeLimiter = new SimpleTimeLimiter(Executors.newFixedThreadPool(20));
/**
* strip off any parts of an url, address string (containing host/ip:port) or raw IPs/Hosts,
* considering that the host may also be an (IPv4) IP or a IPv6 IP in brackets.
* @param target
* @return a host name or IP string
*/
public static String stripToHostName(String target) {
// normalize
if (target == null || target.isEmpty()) return null;
target = target.toLowerCase().trim(); // we can lowercase this because host names are case-insensitive
// extract the address (host:port) part (applies if this is an url)
int p = target.indexOf("://");
if (p > 0) target = target.substring(p + 3);
p = target.indexOf('/');
if (p > 0) target = target.substring(0, p);
// IPv4 / host heuristics
p = target.lastIndexOf(':');
if ( p < 0 ) {
// may be IPv4 or IPv6, we chop off brackets if exist
if (target.charAt(0) == '[') target = target.substring(1);
if (target.charAt(target.length() - 1) == ']') target = target.substring(0, target.length() - 1);
return target;
}
// the ':' at pos p may be either a port divider or a part of an IPv6 address
if (target.charAt(p - 1) == ']') {
target = target.substring(1, p - 1);
return target;
}
// the ':' must be a port divider
target = target.substring(0, p);
return target;
}
public static int stripToPort(String target) {
int port = 80; // default port
// normalize
if (target == null || target.isEmpty()) return port;
target = target.toLowerCase().trim(); // we can lowercase this because host names are case-insensitive
// extract the address (host:port) part (applies if this is an url)
int p = target.indexOf("://");
if (p > 0) {
String protocol = target.substring(0, p);
target = target.substring(p + 3);
if ("https".equals(protocol)) port = 443;
if ("ftp".equals(protocol)) port = 21;
if ("smb".equals(protocol)) port = 445;
}
p = target.indexOf('/');
if (p > 0) target = target.substring(0, p);
// IPv4 / host heuristics
p = target.lastIndexOf(':');
if ( p < 0 ) return port;
// the ':' must be a port divider
port = Integer.parseInt(target.substring(p + 1));
return port;
}
/**
* resolve a host address using a local DNS cache and a DNS lookup if necessary
* @param clienthost
* @return the hosts InetAddress or null if the address cannot be resolved
*/
public static InetAddress dnsResolve(final String host0) {
// consider to call stripToHostName() before calling this
if (host0 == null || host0.isEmpty()) return null;
final String host = host0.toLowerCase().trim();

2
source/net/yacy/cora/protocol/HeaderFramework.java
Unescape View File

@ -574,7 +574,7 @@ public class HeaderFramework extends TreeMap<String, String> implements Map<Stri
//String ip = conProp.getProperty(httpHeader.CONNECTION_PROP_CLIENTIP); // the ip from the connecting peer
int port, pos;
if ((pos = host.indexOf(':')) < 0) {
if ((pos = host.lastIndexOf(':')) < 0 || host.charAt(pos - 1) != ']') {
port = 80;
} else {
port = NumberTools.parseIntDecSubstring(host, pos + 1);

4
source/net/yacy/http/AbstractRemoteHandler.java
Unescape View File

@ -93,9 +93,7 @@ abstract public class AbstractRemoteHandler extends ConnectHandler implements Ha
String host = request.getHeader("Host");
if (host == null) return; // no proxy request, continue processing by handlers
int hostSplitPos = host.indexOf(':');
String hostOnly = hostSplitPos < 0 ? host : host.substring(0, hostSplitPos);
String hostOnly = Domains.stripToHostName(host);
if (localVirtualHostNames.contains(hostOnly)) return; // no proxy request (quick check), continue processing by handlers
if (Domains.isLocal(hostOnly, null)) return; // no proxy, continue processing by handlers

12
source/net/yacy/http/YacyDomainHandler.java
Unescape View File

@ -72,16 +72,8 @@ public class YacyDomainHandler extends AbstractHandler implements Handler {
path = "";
hostPort = resolved;
}
int posPort = hostPort.lastIndexOf(':');
String newHost;
int newPort;
if (posPort >= 0) {
newHost = hostPort.substring(0, posPort);
newPort = Integer.parseInt(hostPort.substring(posPort + 1));
} else {
newHost = hostPort;
newPort = 80;
}
int newPort = Domains.stripToPort(hostPort);
String newHost = Domains.stripToHostName(hostPort);
if (alternativeResolvers.myIPs().contains(newHost)) return;
if (Domains.isLocal(newHost, null)) return;
RequestDispatcher dispatcher = request.getRequestDispatcher(path + target);

17
source/net/yacy/peers/Protocol.java
Unescape View File

@ -232,12 +232,8 @@ public final class Protocol {
} else {
try {
// patch the remote peer address to avoid that remote peers spoof the network with wrong addresses
final int p = targetAddress.lastIndexOf(':');
if ( p < 0 ) return null;
String h = targetAddress.substring(0, p);
if (h.charAt(0) == '[') h = h.substring(1);
if (h.charAt(h.length() - 1) == ']') h = h.substring(0, h.length() - 1);
InetAddress ie = Domains.dnsResolve(h);
String host = Domains.stripToHostName(targetAddress);
InetAddress ie = Domains.dnsResolve(host);
otherPeer = Seed.genRemoteSeed(seed, false, ie.getHostAddress());
if ( !otherPeer.hash.equals(targetHash) ) {
Network.log.info("yacyClient.hello: consistency error: otherPeer.hash = " + otherPeer.hash + ", otherHash = " + targetHash);
@ -340,13 +336,10 @@ public final class Protocol {
} else {
try {
if ( i == 1 ) {
final int p = targetAddress.lastIndexOf(':');
if ( p < 0 ) {
return null;
}
InetAddress ia = Domains.dnsResolve(targetAddress.substring(0, p));
String host = Domains.stripToHostName(targetAddress);
InetAddress ia = Domains.dnsResolve(host);
if (ia == null) continue;
final String host = ia.getHostAddress(); // the actual address of the target as we had been successful when contacting them is patched here
host = ia.getHostAddress(); // the actual address of the target as we had been successful when contacting them is patched here
s = Seed.genRemoteSeed(seedStr, false, host);
} else {
s = Seed.genRemoteSeed(seedStr, false, null);

24
source/net/yacy/server/http/HTTPDProxyHandler.java
Unescape View File

@ -423,13 +423,8 @@ public final class HTTPDProxyHandler {
final String ip = (String) conProp.get(HeaderFramework.CONNECTION_PROP_CLIENTIP); // the ip from the connecting peer
final String httpVer = (String) conProp.get(HeaderFramework.CONNECTION_PROP_HTTP_VER); // the ip from the connecting peer
int port, pos;
if ((pos = host.indexOf(':')) < 0) {
port = 80;
} else {
port = Integer.parseInt(host.substring(pos + 1));
host = host.substring(0, pos);
}
int port = Domains.stripToPort(host);
host = Domains.stripToHostName(host);
// resolve yacy and yacyh domains
String yAddress = resolveYacyDomains(host);
@ -438,10 +433,10 @@ public final class HTTPDProxyHandler {
final String remotePath = (args == null) ? path : (path + "?" + args); // with leading '/'
// remove yacy-subdomain-path, when accessing /env
if ( (yAddress != null)
if ((yAddress != null)
&& (remotePath.startsWith("/env"))
&& ((pos = yAddress.indexOf('/')) != -1)
) yAddress = yAddress.substring(0, yAddress.indexOf('/'));
&& (yAddress.indexOf('/') != -1)
) yAddress = yAddress.substring(0, yAddress.indexOf('/'));
modifyProxyHeaders(requestHeader, httpVer);
@ -1050,11 +1045,8 @@ public final class HTTPDProxyHandler {
String orgHostName = (String) conProp.get(HeaderFramework.CONNECTION_PROP_HOST);
if (orgHostName == null) orgHostName = "unknown";
orgHostName = orgHostName.toLowerCase();
int pos = orgHostName.indexOf(':');
if (pos != -1) {
orgHostPort = orgHostName.substring(pos+1);
orgHostName = orgHostName.substring(0,pos);
}
orgHostPort = Integer.toString(Domains.stripToPort(orgHostName));
orgHostName = Domains.stripToHostName(orgHostName);
String orgHostPath = (String) conProp.get(HeaderFramework.CONNECTION_PROP_PATH); if (orgHostPath == null) orgHostPath = "";
String orgHostArgs = (String) conProp.get(HeaderFramework.CONNECTION_PROP_ARGS); if (orgHostArgs == null) orgHostArgs = "";
if (orgHostArgs.length() > 0) orgHostArgs = "?" + orgHostArgs;
@ -1078,7 +1070,7 @@ public final class HTTPDProxyHandler {
if (addr != null) if (addr != null) testHostNames.add(testHostName);
}
pos = orgHostName.lastIndexOf('.');
int pos = orgHostName.lastIndexOf('.');
if (pos != -1) {
final Iterator<String> iter = topLevelDomains.iterator();
while (iter.hasNext()) {

10
source/net/yacy/server/http/HTTPDemon.java
Unescape View File

@ -144,14 +144,8 @@ public final class HTTPDemon {
final String args = (String) conProp.get(HeaderFramework.CONNECTION_PROP_ARGS);
final String method = (String) conProp.get(HeaderFramework.CONNECTION_PROP_METHOD);
final int port;
final int pos = host.indexOf(':');
if (pos != -1) {
port = NumberTools.parseIntDecSubstring(host, pos + 1);
host = host.substring(0, pos);
} else {
port = 80;
}
final int port = Domains.stripToPort(host);
host = Domains.stripToHostName(host);
String urlString;
try {

Write Preview
Loading…
Cancel
Save