more performance hacks

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7149 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 15 years ago
parent 14c843d364
commit ac1c08924e

@ -36,10 +36,10 @@ import java.util.TreeSet;
import java.util.Map.Entry; import java.util.Map.Entry;
import java.text.SimpleDateFormat; import java.text.SimpleDateFormat;
import net.yacy.cora.protocol.Domains;
import net.yacy.cora.protocol.HeaderFramework; import net.yacy.cora.protocol.HeaderFramework;
import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.protocol.RequestHeader;
import de.anomic.net.natLib;
import de.anomic.search.QueryParams; import de.anomic.search.QueryParams;
import de.anomic.search.Switchboard; import de.anomic.search.Switchboard;
import de.anomic.server.serverAccessTracker; import de.anomic.server.serverAccessTracker;
@ -238,7 +238,7 @@ public class AccessTracker_p {
prop.put("page_list_" + entCount + "_dark", ((dark) ? 1 : 0) ); dark =! dark; prop.put("page_list_" + entCount + "_dark", ((dark) ? 1 : 0) ); dark =! dark;
prop.putHTML("page_list_" + entCount + "_host", host); prop.putHTML("page_list_" + entCount + "_host", host);
if (page == 5) { if (page == 5) {
final yacySeed remotepeer = sb.peers.lookupByIP(natLib.getInetAddress(host), true, true, true); final yacySeed remotepeer = sb.peers.lookupByIP(Domains.dnsResolve(host), true, true, true);
prop.putHTML("page_list_" + entCount + "_peername", (remotepeer == null) ? "UNKNOWN" : remotepeer.getName()); prop.putHTML("page_list_" + entCount + "_peername", (remotepeer == null) ? "UNKNOWN" : remotepeer.getName());
} }
prop.putNum("page_list_" + entCount + "_count", handles.size()); prop.putNum("page_list_" + entCount + "_count", handles.size());

@ -29,9 +29,7 @@ package www;
// if the shell's current path is HTROOT // if the shell's current path is HTROOT
import java.io.File; import java.io.File;
import java.net.InetAddress; import net.yacy.cora.protocol.Domains;
import java.net.UnknownHostException;
import net.yacy.cora.protocol.HeaderFramework; import net.yacy.cora.protocol.HeaderFramework;
import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.protocol.RequestHeader;
@ -56,11 +54,7 @@ public class welcome {
prop.putHTML("peerdomain", env.getConfig("peerName", "<nameless>").toLowerCase()); prop.putHTML("peerdomain", env.getConfig("peerName", "<nameless>").toLowerCase());
prop.putHTML("peeraddress", sb.peers.mySeed().getPublicAddress()); prop.putHTML("peeraddress", sb.peers.mySeed().getPublicAddress());
prop.put("hostname", env.myPublicIP()); prop.put("hostname", env.myPublicIP());
try{ prop.put("hostip", Domains.dnsResolve(env.myPublicIP()).getHostAddress());
prop.put("hostip", InetAddress.getByName(env.myPublicIP()).getHostAddress());
}catch(final UnknownHostException e){
prop.put("hostip", "Unknown Host Exception");
}
prop.put("port", serverCore.getPortNr(env.getConfig("port","8080"))); prop.put("port", serverCore.getPortNr(env.getConfig("port","8080")));
prop.put("clientip", header.get(HeaderFramework.CONNECTION_PROP_CLIENTIP, "")); prop.put("clientip", header.get(HeaderFramework.CONNECTION_PROP_CLIENTIP, ""));

@ -50,7 +50,6 @@ import net.yacy.kelondro.util.EventTracker;
import net.yacy.kelondro.util.ISO639; import net.yacy.kelondro.util.ISO639;
import de.anomic.crawler.CrawlProfile; import de.anomic.crawler.CrawlProfile;
import de.anomic.net.natLib;
import de.anomic.search.ContentDomain; import de.anomic.search.ContentDomain;
import de.anomic.search.Navigator; import de.anomic.search.Navigator;
import de.anomic.search.QueryParams; import de.anomic.search.QueryParams;
@ -385,7 +384,7 @@ public final class search {
prop.put("fwrec", ""); // peers that would have helped to construct this result (recommendations) prop.put("fwrec", ""); // peers that would have helped to construct this result (recommendations)
// prepare search statistics // prepare search statistics
theQuery.remotepeer = client == null ? null : sb.peers.lookupByIP(natLib.getInetAddress(client), true, false, false); theQuery.remotepeer = client == null ? null : sb.peers.lookupByIP(Domains.dnsResolve(client), true, false, false);
theQuery.resultcount = (theSearch == null) ? 0 : theSearch.getRankingResult().getLocalIndexCount() + theSearch.getRankingResult().getRemoteResourceSize(); theQuery.resultcount = (theSearch == null) ? 0 : theSearch.getRankingResult().getLocalIndexCount() + theSearch.getRankingResult().getRemoteResourceSize();
theQuery.searchtime = System.currentTimeMillis() - timestamp; theQuery.searchtime = System.currentTimeMillis() - timestamp;
theQuery.urlretrievaltime = (theSearch == null) ? 0 : theSearch.result().getURLRetrievalTime(); theQuery.urlretrievaltime = (theSearch == null) ? 0 : theSearch.result().getURLRetrievalTime();

@ -1353,7 +1353,6 @@ public final class HTTPDemon implements serverHandler, Cloneable {
boolean isThisHostIP = false; boolean isThisHostIP = false;
try { try {
// final InetAddress clientAddress = InetAddress.getByName(hostName);
final InetAddress clientAddress = Domains.dnsResolve(hostName); final InetAddress clientAddress = Domains.dnsResolve(hostName);
if (clientAddress == null) return false; if (clientAddress == null) return false;

@ -1,105 +0,0 @@
// natLib.java
// -------------------------------------
// (C) by Michael Peter Christen; mc@yacy.net
// first published on http://www.anomic.de
// Frankfurt, Germany, 2004
// last major change: 04.05.2004
//
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package de.anomic.net;
import java.net.InetAddress;
import java.net.UnknownHostException;
import de.anomic.search.Switchboard;
public class natLib {
private static boolean isNotLocal(final String ip) {
if ((ip.equals("localhost")) ||
(ip.startsWith("127")) ||
(ip.startsWith("192.168")) ||
(ip.startsWith("172.16")) ||
(ip.startsWith("10."))
) return false;
return true;
}
private static boolean isIP(final String ip) {
if (ip == null) return false;
try {
/*InetAddress dummy =*/ InetAddress.getByName(ip);
return true;
} catch (final Exception e) {
return false;
}
}
//TODO: This is not IPv6 compatible
public static boolean isProper(final String ip) {
final Switchboard sb=Switchboard.getSwitchboard();
if (sb != null) {
if (sb.isRobinsonMode()) return true;
final String yacyDebugMode = sb.getConfig("yacyDebugMode", "false");
if (yacyDebugMode.equals("true")) {
return true;
}
// support for staticIP
if (sb.getConfig("staticIP", "").equals(ip)) {
return true;
}
}
if (ip == null) return false;
if (ip.indexOf(':') >= 0) return false; // ipv6...
return (isNotLocal(ip)) && (isIP(ip));
}
public static final InetAddress getInetAddress(final String ip) {
if (ip == null) return null;
if (ip.length() < 8) return null;
final String[] ips = ip.split("\\.");
if (ips.length != 4) return null;
final byte[] ipb = new byte[4];
try {
ipb[0] = (byte) Integer.parseInt(ips[0]);
ipb[1] = (byte) Integer.parseInt(ips[1]);
ipb[2] = (byte) Integer.parseInt(ips[2]);
ipb[3] = (byte) Integer.parseInt(ips[3]);
} catch (final NumberFormatException e) {
return null;
}
try {
return InetAddress.getByAddress(ipb);
} catch (final UnknownHostException e) {
return null;
}
}
// rDNS services:
// http://www.xdr2.net/reverse_DNS_lookup.asp
// http://remote.12dt.com/rns/
// http://bl.reynolds.net.au/search/
// http://www.declude.com/Articles.asp?ID=97
// http://www.dnsstuff.com/
// listlist: http://www.aspnetimap.com/help/welcome/dnsbl.html
}

@ -57,6 +57,7 @@ import java.util.TreeMap;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import net.yacy.cora.protocol.Domains;
import net.yacy.kelondro.data.word.Word; import net.yacy.kelondro.data.word.Word;
import net.yacy.kelondro.index.HandleSet; import net.yacy.kelondro.index.HandleSet;
import net.yacy.kelondro.order.Base64Order; import net.yacy.kelondro.order.Base64Order;
@ -64,7 +65,6 @@ import net.yacy.kelondro.order.Digest;
import net.yacy.kelondro.util.DateFormatter; import net.yacy.kelondro.util.DateFormatter;
import net.yacy.kelondro.util.MapTools; import net.yacy.kelondro.util.MapTools;
import de.anomic.net.natLib;
import de.anomic.tools.bitfield; import de.anomic.tools.bitfield;
import de.anomic.tools.crypt; import de.anomic.tools.crypt;
import de.anomic.yacy.dht.FlatWordPartitionScheme; import de.anomic.yacy.dht.FlatWordPartitionScheme;
@ -468,7 +468,7 @@ public class yacySeed implements Cloneable {
* @return the IP address of the peer represented by this yacySeed object as {@link InetAddress} * @return the IP address of the peer represented by this yacySeed object as {@link InetAddress}
*/ */
public final InetAddress getInetAddress() { public final InetAddress getInetAddress() {
return natLib.getInetAddress(this.getIP()); return Domains.dnsResolve(this.getIP());
} }
/** @return the portnumber of this seed or <code>-1</code> if not present */ /** @return the portnumber of this seed or <code>-1</code> if not present */
@ -818,7 +818,8 @@ public class yacySeed implements Cloneable {
// returns null if ipString is proper, a string with the cause otherwise // returns null if ipString is proper, a string with the cause otherwise
if (ipString == null) return "IP is null"; if (ipString == null) return "IP is null";
if (ipString.length() > 0 && ipString.length() < 8) return "IP is too short: " + ipString; if (ipString.length() > 0 && ipString.length() < 8) return "IP is too short: " + ipString;
if (!natLib.isProper(ipString)) return "IP is not proper: " + ipString; //this does not work with staticIP InetAddress ip = Domains.dnsResolve(ipString);
if (ip == null) return "IP is not proper: " + ipString; //this does not work with staticIP
if (ipString.equals("localhost") || ipString.startsWith("127.") || (ipString.startsWith("0:0:0:0:0:0:0:1"))) return "IP for localhost rejected"; if (ipString.equals("localhost") || ipString.startsWith("127.") || (ipString.startsWith("0:0:0:0:0:0:0:1"))) return "IP for localhost rejected";
return null; return null;
} }

@ -31,7 +31,6 @@ import java.io.IOException;
import java.io.PrintWriter; import java.io.PrintWriter;
import java.lang.ref.SoftReference; import java.lang.ref.SoftReference;
import java.net.InetAddress; import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Hashtable; import java.util.Hashtable;
import java.util.Iterator; import java.util.Iterator;
@ -39,6 +38,7 @@ import java.util.Map;
import java.util.TreeMap; import java.util.TreeMap;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import net.yacy.cora.protocol.Domains;
import net.yacy.cora.protocol.HeaderFramework; import net.yacy.cora.protocol.HeaderFramework;
import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.protocol.RequestHeader;
import net.yacy.cora.protocol.http.HTTPClient; import net.yacy.cora.protocol.http.HTTPClient;
@ -670,7 +670,6 @@ public final class yacySeedDB implements AlternativeDomainNames {
// enumerate the cache and simultanous insert values // enumerate the cache and simultanous insert values
final Iterator<yacySeed> e = seedsConnected(true, false, null, (float) 0.0); final Iterator<yacySeed> e = seedsConnected(true, false, null, (float) 0.0);
while (e.hasNext()) { while (e.hasNext()) {
try {
seed = e.next(); seed = e.next();
if (seed != null) { if (seed != null) {
addressStr = seed.getPublicAddress(); addressStr = seed.getPublicAddress();
@ -687,11 +686,11 @@ public final class yacySeedDB implements AlternativeDomainNames {
if ((pos = addressStr.indexOf(':'))!= -1) { if ((pos = addressStr.indexOf(':'))!= -1) {
addressStr = addressStr.substring(0,pos); addressStr = addressStr.substring(0,pos);
} }
seedIPAddress = InetAddress.getByName(addressStr); seedIPAddress = Domains.dnsResolve(addressStr);
if (seedIPAddress == null) continue;
if (seed.isProper(false) == null) ipLookupCache.put(seedIPAddress, new SoftReference<yacySeed>(seed)); if (seed.isProper(false) == null) ipLookupCache.put(seedIPAddress, new SoftReference<yacySeed>(seed));
if (seedIPAddress.equals(peerIP)) return seed; if (seedIPAddress.equals(peerIP)) return seed;
} }
} catch (final UnknownHostException ex) {}
} }
// delete bad peers // delete bad peers
final Iterator<byte[]> i = badPeerHashes.iterator(); final Iterator<byte[]> i = badPeerHashes.iterator();
@ -704,7 +703,6 @@ public final class yacySeedDB implements AlternativeDomainNames {
final Iterator<yacySeed>e = seedsDisconnected(true, false, null, (float) 0.0); final Iterator<yacySeed>e = seedsDisconnected(true, false, null, (float) 0.0);
while (e.hasNext()) { while (e.hasNext()) {
try {
seed = e.next(); seed = e.next();
if (seed != null) { if (seed != null) {
addressStr = seed.getPublicAddress(); addressStr = seed.getPublicAddress();
@ -721,11 +719,11 @@ public final class yacySeedDB implements AlternativeDomainNames {
if ((pos = addressStr.indexOf(':'))!= -1) { if ((pos = addressStr.indexOf(':'))!= -1) {
addressStr = addressStr.substring(0,pos); addressStr = addressStr.substring(0,pos);
} }
seedIPAddress = InetAddress.getByName(addressStr); seedIPAddress = Domains.dnsResolve(addressStr);
if (seedIPAddress == null) continue;
if (seed.isProper(false) == null) ipLookupCache.put(seedIPAddress, new SoftReference<yacySeed>(seed)); if (seed.isProper(false) == null) ipLookupCache.put(seedIPAddress, new SoftReference<yacySeed>(seed));
if (seedIPAddress.equals(peerIP)) return seed; if (seedIPAddress.equals(peerIP)) return seed;
} }
} catch (final UnknownHostException ex) {}
} }
// delete bad peers // delete bad peers
final Iterator<byte[]> i = badPeerHashes.iterator(); final Iterator<byte[]> i = badPeerHashes.iterator();
@ -738,21 +736,19 @@ public final class yacySeedDB implements AlternativeDomainNames {
final Iterator<yacySeed> e = seedsPotential(true, false, null, (float) 0.0); final Iterator<yacySeed> e = seedsPotential(true, false, null, (float) 0.0);
while (e.hasNext()) { while (e.hasNext()) {
try {
seed = e.next(); seed = e.next();
if ((seed != null) && ((addressStr = seed.getPublicAddress()) != null)) { if ((seed != null) && ((addressStr = seed.getPublicAddress()) != null)) {
if ((pos = addressStr.indexOf(':'))!= -1) { if ((pos = addressStr.indexOf(':'))!= -1) {
addressStr = addressStr.substring(0,pos); addressStr = addressStr.substring(0,pos);
} }
seedIPAddress = InetAddress.getByName(addressStr); seedIPAddress = Domains.dnsResolve(addressStr);
if (seedIPAddress == null) continue;
if (seed.isProper(false) == null) ipLookupCache.put(seedIPAddress, new SoftReference<yacySeed>(seed)); if (seed.isProper(false) == null) ipLookupCache.put(seedIPAddress, new SoftReference<yacySeed>(seed));
if (seedIPAddress.equals(peerIP)) return seed; if (seedIPAddress.equals(peerIP)) return seed;
} }
} catch (final UnknownHostException ex) {}
} }
} }
try {
// check local seed // check local seed
if (this.mySeed == null) return null; if (this.mySeed == null) return null;
addressStr = mySeed.getPublicAddress(); addressStr = mySeed.getPublicAddress();
@ -760,14 +756,11 @@ public final class yacySeedDB implements AlternativeDomainNames {
if ((pos = addressStr.indexOf(':'))!= -1) { if ((pos = addressStr.indexOf(':'))!= -1) {
addressStr = addressStr.substring(0,pos); addressStr = addressStr.substring(0,pos);
} }
seedIPAddress = InetAddress.getByName(addressStr); seedIPAddress = Domains.dnsResolve(addressStr);
if (mySeed.isProper(false) == null) ipLookupCache.put(seedIPAddress, new SoftReference<yacySeed>(mySeed)); if (mySeed.isProper(false) == null) ipLookupCache.put(seedIPAddress, new SoftReference<yacySeed>(mySeed));
if (seedIPAddress.equals(peerIP)) return mySeed; if (seedIPAddress.equals(peerIP)) return mySeed;
// nothing found // nothing found
return null; return null;
} catch (final UnknownHostException e2) {
return null;
}
} }
private ArrayList<String> storeCache(final File seedFile, final boolean addMySeed) throws IOException { private ArrayList<String> storeCache(final File seedFile, final boolean addMySeed) throws IOException {

@ -450,12 +450,36 @@ public class Domains {
return false; return false;
} }
private static final InetAddress parseInetAddress(final String ip) {
if (ip == null) return null;
if (ip.length() < 8) return null;
final String[] ips = ip.split("\\.");
if (ips.length != 4) return null;
final byte[] ipb = new byte[4];
try {
ipb[0] = (byte) Integer.parseInt(ips[0]);
ipb[1] = (byte) Integer.parseInt(ips[1]);
ipb[2] = (byte) Integer.parseInt(ips[2]);
ipb[3] = (byte) Integer.parseInt(ips[3]);
} catch (final NumberFormatException e) {
return null;
}
try {
return InetAddress.getByAddress(ipb);
} catch (final UnknownHostException e) {
return null;
}
}
public static InetAddress dnsResolve(String host) { public static InetAddress dnsResolve(String host) {
if ((host == null) || (host.length() == 0)) return null; if ((host == null) || (host.length() == 0)) return null;
host = host.toLowerCase().trim(); host = host.toLowerCase().trim();
// try to simply parse the address
InetAddress ip = parseInetAddress(host);
if (ip != null) return ip;
// trying to resolve host by doing a name cache lookup // try to resolve host by doing a name cache lookup
InetAddress ip = nameCacheHit.get(host); ip = nameCacheHit.get(host);
if (ip != null) return ip; if (ip != null) return ip;
if (nameCacheMiss.containsKey(host)) return null; if (nameCacheMiss.containsKey(host)) return null;

@ -2118,7 +2118,7 @@ public class FTPClient {
final int b = Integer.parseInt(st.nextToken()); final int b = Integer.parseInt(st.nextToken());
final int c = Integer.parseInt(st.nextToken()); final int c = Integer.parseInt(st.nextToken());
final int d = Integer.parseInt(st.nextToken()); final int d = Integer.parseInt(st.nextToken());
final InetAddress datahost = InetAddress.getByName(a + "." + b + "." + c + "." + d); final InetAddress datahost = Domains.dnsResolve(a + "." + b + "." + c + "." + d);
final int high = Integer.parseInt(st.nextToken()); final int high = Integer.parseInt(st.nextToken());
final int low = Integer.parseInt(st.nextToken()); final int low = Integer.parseInt(st.nextToken());
if (high < 0 || high > 255 || low < 0 || low > 255) { if (high < 0 || high > 255 || low < 0 || low > 255) {

Loading…
Cancel
Save