more performance hacks

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7149 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 15 years ago
parent 14c843d364
commit ac1c08924e

@ -36,10 +36,10 @@ import java.util.TreeSet;
import java.util.Map.Entry;
import java.text.SimpleDateFormat;
import net.yacy.cora.protocol.Domains;
import net.yacy.cora.protocol.HeaderFramework;
import net.yacy.cora.protocol.RequestHeader;
import de.anomic.net.natLib;
import de.anomic.search.QueryParams;
import de.anomic.search.Switchboard;
import de.anomic.server.serverAccessTracker;
@ -238,7 +238,7 @@ public class AccessTracker_p {
prop.put("page_list_" + entCount + "_dark", ((dark) ? 1 : 0) ); dark =! dark;
prop.putHTML("page_list_" + entCount + "_host", host);
if (page == 5) {
final yacySeed remotepeer = sb.peers.lookupByIP(natLib.getInetAddress(host), true, true, true);
final yacySeed remotepeer = sb.peers.lookupByIP(Domains.dnsResolve(host), true, true, true);
prop.putHTML("page_list_" + entCount + "_peername", (remotepeer == null) ? "UNKNOWN" : remotepeer.getName());
}
prop.putNum("page_list_" + entCount + "_count", handles.size());

@ -29,9 +29,7 @@ package www;
// if the shell's current path is HTROOT
import java.io.File;
import java.net.InetAddress;
import java.net.UnknownHostException;
import net.yacy.cora.protocol.Domains;
import net.yacy.cora.protocol.HeaderFramework;
import net.yacy.cora.protocol.RequestHeader;
@ -56,11 +54,7 @@ public class welcome {
prop.putHTML("peerdomain", env.getConfig("peerName", "<nameless>").toLowerCase());
prop.putHTML("peeraddress", sb.peers.mySeed().getPublicAddress());
prop.put("hostname", env.myPublicIP());
try{
prop.put("hostip", InetAddress.getByName(env.myPublicIP()).getHostAddress());
}catch(final UnknownHostException e){
prop.put("hostip", "Unknown Host Exception");
}
prop.put("hostip", Domains.dnsResolve(env.myPublicIP()).getHostAddress());
prop.put("port", serverCore.getPortNr(env.getConfig("port","8080")));
prop.put("clientip", header.get(HeaderFramework.CONNECTION_PROP_CLIENTIP, ""));

@ -50,7 +50,6 @@ import net.yacy.kelondro.util.EventTracker;
import net.yacy.kelondro.util.ISO639;
import de.anomic.crawler.CrawlProfile;
import de.anomic.net.natLib;
import de.anomic.search.ContentDomain;
import de.anomic.search.Navigator;
import de.anomic.search.QueryParams;
@ -385,7 +384,7 @@ public final class search {
prop.put("fwrec", ""); // peers that would have helped to construct this result (recommendations)
// prepare search statistics
theQuery.remotepeer = client == null ? null : sb.peers.lookupByIP(natLib.getInetAddress(client), true, false, false);
theQuery.remotepeer = client == null ? null : sb.peers.lookupByIP(Domains.dnsResolve(client), true, false, false);
theQuery.resultcount = (theSearch == null) ? 0 : theSearch.getRankingResult().getLocalIndexCount() + theSearch.getRankingResult().getRemoteResourceSize();
theQuery.searchtime = System.currentTimeMillis() - timestamp;
theQuery.urlretrievaltime = (theSearch == null) ? 0 : theSearch.result().getURLRetrievalTime();

@ -1353,7 +1353,6 @@ public final class HTTPDemon implements serverHandler, Cloneable {
boolean isThisHostIP = false;
try {
// final InetAddress clientAddress = InetAddress.getByName(hostName);
final InetAddress clientAddress = Domains.dnsResolve(hostName);
if (clientAddress == null) return false;

@ -1,105 +0,0 @@
// natLib.java
// -------------------------------------
// (C) by Michael Peter Christen; mc@yacy.net
// first published on http://www.anomic.de
// Frankfurt, Germany, 2004
// last major change: 04.05.2004
//
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package de.anomic.net;
import java.net.InetAddress;
import java.net.UnknownHostException;
import de.anomic.search.Switchboard;
public class natLib {
private static boolean isNotLocal(final String ip) {
if ((ip.equals("localhost")) ||
(ip.startsWith("127")) ||
(ip.startsWith("192.168")) ||
(ip.startsWith("172.16")) ||
(ip.startsWith("10."))
) return false;
return true;
}
private static boolean isIP(final String ip) {
if (ip == null) return false;
try {
/*InetAddress dummy =*/ InetAddress.getByName(ip);
return true;
} catch (final Exception e) {
return false;
}
}
//TODO: This is not IPv6 compatible
public static boolean isProper(final String ip) {
final Switchboard sb=Switchboard.getSwitchboard();
if (sb != null) {
if (sb.isRobinsonMode()) return true;
final String yacyDebugMode = sb.getConfig("yacyDebugMode", "false");
if (yacyDebugMode.equals("true")) {
return true;
}
// support for staticIP
if (sb.getConfig("staticIP", "").equals(ip)) {
return true;
}
}
if (ip == null) return false;
if (ip.indexOf(':') >= 0) return false; // ipv6...
return (isNotLocal(ip)) && (isIP(ip));
}
public static final InetAddress getInetAddress(final String ip) {
if (ip == null) return null;
if (ip.length() < 8) return null;
final String[] ips = ip.split("\\.");
if (ips.length != 4) return null;
final byte[] ipb = new byte[4];
try {
ipb[0] = (byte) Integer.parseInt(ips[0]);
ipb[1] = (byte) Integer.parseInt(ips[1]);
ipb[2] = (byte) Integer.parseInt(ips[2]);
ipb[3] = (byte) Integer.parseInt(ips[3]);
} catch (final NumberFormatException e) {
return null;
}
try {
return InetAddress.getByAddress(ipb);
} catch (final UnknownHostException e) {
return null;
}
}
// rDNS services:
// http://www.xdr2.net/reverse_DNS_lookup.asp
// http://remote.12dt.com/rns/
// http://bl.reynolds.net.au/search/
// http://www.declude.com/Articles.asp?ID=97
// http://www.dnsstuff.com/
// listlist: http://www.aspnetimap.com/help/welcome/dnsbl.html
}

@ -57,6 +57,7 @@ import java.util.TreeMap;
import java.util.concurrent.ConcurrentHashMap;
import java.util.regex.Pattern;
import net.yacy.cora.protocol.Domains;
import net.yacy.kelondro.data.word.Word;
import net.yacy.kelondro.index.HandleSet;
import net.yacy.kelondro.order.Base64Order;
@ -64,7 +65,6 @@ import net.yacy.kelondro.order.Digest;
import net.yacy.kelondro.util.DateFormatter;
import net.yacy.kelondro.util.MapTools;
import de.anomic.net.natLib;
import de.anomic.tools.bitfield;
import de.anomic.tools.crypt;
import de.anomic.yacy.dht.FlatWordPartitionScheme;
@ -468,7 +468,7 @@ public class yacySeed implements Cloneable {
* @return the IP address of the peer represented by this yacySeed object as {@link InetAddress}
*/
public final InetAddress getInetAddress() {
return natLib.getInetAddress(this.getIP());
return Domains.dnsResolve(this.getIP());
}
/** @return the portnumber of this seed or <code>-1</code> if not present */
@ -818,7 +818,8 @@ public class yacySeed implements Cloneable {
// returns null if ipString is proper, a string with the cause otherwise
if (ipString == null) return "IP is null";
if (ipString.length() > 0 && ipString.length() < 8) return "IP is too short: " + ipString;
if (!natLib.isProper(ipString)) return "IP is not proper: " + ipString; //this does not work with staticIP
InetAddress ip = Domains.dnsResolve(ipString);
if (ip == null) return "IP is not proper: " + ipString; //this does not work with staticIP
if (ipString.equals("localhost") || ipString.startsWith("127.") || (ipString.startsWith("0:0:0:0:0:0:0:1"))) return "IP for localhost rejected";
return null;
}

@ -31,7 +31,6 @@ import java.io.IOException;
import java.io.PrintWriter;
import java.lang.ref.SoftReference;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.Hashtable;
import java.util.Iterator;
@ -39,6 +38,7 @@ import java.util.Map;
import java.util.TreeMap;
import java.util.concurrent.ConcurrentHashMap;
import net.yacy.cora.protocol.Domains;
import net.yacy.cora.protocol.HeaderFramework;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.cora.protocol.http.HTTPClient;
@ -670,28 +670,27 @@ public final class yacySeedDB implements AlternativeDomainNames {
// enumerate the cache and simultanous insert values
final Iterator<yacySeed> e = seedsConnected(true, false, null, (float) 0.0);
while (e.hasNext()) {
try {
seed = e.next();
if (seed != null) {
addressStr = seed.getPublicAddress();
if (addressStr == null) {
Log.logWarning("YACY","lookupByIP/Connected: address of seed " + seed.getName() + "/" + seed.hash + " is null.");
try {
badPeerHashes.put(seed.hash.getBytes());
} catch (RowSpaceExceededException e1) {
Log.logException(e1);
break;
}
continue;
seed = e.next();
if (seed != null) {
addressStr = seed.getPublicAddress();
if (addressStr == null) {
Log.logWarning("YACY","lookupByIP/Connected: address of seed " + seed.getName() + "/" + seed.hash + " is null.");
try {
badPeerHashes.put(seed.hash.getBytes());
} catch (RowSpaceExceededException e1) {
Log.logException(e1);
break;
}
if ((pos = addressStr.indexOf(':'))!= -1) {
addressStr = addressStr.substring(0,pos);
}
seedIPAddress = InetAddress.getByName(addressStr);
if (seed.isProper(false) == null) ipLookupCache.put(seedIPAddress, new SoftReference<yacySeed>(seed));
if (seedIPAddress.equals(peerIP)) return seed;
continue;
}
if ((pos = addressStr.indexOf(':'))!= -1) {
addressStr = addressStr.substring(0,pos);
}
} catch (final UnknownHostException ex) {}
seedIPAddress = Domains.dnsResolve(addressStr);
if (seedIPAddress == null) continue;
if (seed.isProper(false) == null) ipLookupCache.put(seedIPAddress, new SoftReference<yacySeed>(seed));
if (seedIPAddress.equals(peerIP)) return seed;
}
}
// delete bad peers
final Iterator<byte[]> i = badPeerHashes.iterator();
@ -704,28 +703,27 @@ public final class yacySeedDB implements AlternativeDomainNames {
final Iterator<yacySeed>e = seedsDisconnected(true, false, null, (float) 0.0);
while (e.hasNext()) {
try {
seed = e.next();
if (seed != null) {
addressStr = seed.getPublicAddress();
if (addressStr == null) {
Log.logWarning("YACY","lookupByIPDisconnected: address of seed " + seed.getName() + "/" + seed.hash + " is null.");
try {
badPeerHashes.put(seed.hash.getBytes());
} catch (RowSpaceExceededException e1) {
Log.logException(e1);
break;
}
continue;
}
if ((pos = addressStr.indexOf(':'))!= -1) {
addressStr = addressStr.substring(0,pos);
seed = e.next();
if (seed != null) {
addressStr = seed.getPublicAddress();
if (addressStr == null) {
Log.logWarning("YACY","lookupByIPDisconnected: address of seed " + seed.getName() + "/" + seed.hash + " is null.");
try {
badPeerHashes.put(seed.hash.getBytes());
} catch (RowSpaceExceededException e1) {
Log.logException(e1);
break;
}
seedIPAddress = InetAddress.getByName(addressStr);
if (seed.isProper(false) == null) ipLookupCache.put(seedIPAddress, new SoftReference<yacySeed>(seed));
if (seedIPAddress.equals(peerIP)) return seed;
continue;
}
} catch (final UnknownHostException ex) {}
if ((pos = addressStr.indexOf(':'))!= -1) {
addressStr = addressStr.substring(0,pos);
}
seedIPAddress = Domains.dnsResolve(addressStr);
if (seedIPAddress == null) continue;
if (seed.isProper(false) == null) ipLookupCache.put(seedIPAddress, new SoftReference<yacySeed>(seed));
if (seedIPAddress.equals(peerIP)) return seed;
}
}
// delete bad peers
final Iterator<byte[]> i = badPeerHashes.iterator();
@ -738,36 +736,31 @@ public final class yacySeedDB implements AlternativeDomainNames {
final Iterator<yacySeed> e = seedsPotential(true, false, null, (float) 0.0);
while (e.hasNext()) {
try {
seed = e.next();
if ((seed != null) && ((addressStr = seed.getPublicAddress()) != null)) {
if ((pos = addressStr.indexOf(':'))!= -1) {
addressStr = addressStr.substring(0,pos);
}
seedIPAddress = InetAddress.getByName(addressStr);
if (seed.isProper(false) == null) ipLookupCache.put(seedIPAddress, new SoftReference<yacySeed>(seed));
if (seedIPAddress.equals(peerIP)) return seed;
seed = e.next();
if ((seed != null) && ((addressStr = seed.getPublicAddress()) != null)) {
if ((pos = addressStr.indexOf(':'))!= -1) {
addressStr = addressStr.substring(0,pos);
}
} catch (final UnknownHostException ex) {}
seedIPAddress = Domains.dnsResolve(addressStr);
if (seedIPAddress == null) continue;
if (seed.isProper(false) == null) ipLookupCache.put(seedIPAddress, new SoftReference<yacySeed>(seed));
if (seedIPAddress.equals(peerIP)) return seed;
}
}
}
try {
// check local seed
if (this.mySeed == null) return null;
addressStr = mySeed.getPublicAddress();
if (addressStr == null) return null;
if ((pos = addressStr.indexOf(':'))!= -1) {
addressStr = addressStr.substring(0,pos);
}
seedIPAddress = InetAddress.getByName(addressStr);
if (mySeed.isProper(false) == null) ipLookupCache.put(seedIPAddress, new SoftReference<yacySeed>(mySeed));
if (seedIPAddress.equals(peerIP)) return mySeed;
// nothing found
return null;
} catch (final UnknownHostException e2) {
return null;
// check local seed
if (this.mySeed == null) return null;
addressStr = mySeed.getPublicAddress();
if (addressStr == null) return null;
if ((pos = addressStr.indexOf(':'))!= -1) {
addressStr = addressStr.substring(0,pos);
}
seedIPAddress = Domains.dnsResolve(addressStr);
if (mySeed.isProper(false) == null) ipLookupCache.put(seedIPAddress, new SoftReference<yacySeed>(mySeed));
if (seedIPAddress.equals(peerIP)) return mySeed;
// nothing found
return null;
}
private ArrayList<String> storeCache(final File seedFile, final boolean addMySeed) throws IOException {

@ -449,13 +449,37 @@ public class Domains {
}
return false;
}
private static final InetAddress parseInetAddress(final String ip) {
if (ip == null) return null;
if (ip.length() < 8) return null;
final String[] ips = ip.split("\\.");
if (ips.length != 4) return null;
final byte[] ipb = new byte[4];
try {
ipb[0] = (byte) Integer.parseInt(ips[0]);
ipb[1] = (byte) Integer.parseInt(ips[1]);
ipb[2] = (byte) Integer.parseInt(ips[2]);
ipb[3] = (byte) Integer.parseInt(ips[3]);
} catch (final NumberFormatException e) {
return null;
}
try {
return InetAddress.getByAddress(ipb);
} catch (final UnknownHostException e) {
return null;
}
}
public static InetAddress dnsResolve(String host) {
if ((host == null) || (host.length() == 0)) return null;
host = host.toLowerCase().trim();
// try to simply parse the address
InetAddress ip = parseInetAddress(host);
if (ip != null) return ip;
// trying to resolve host by doing a name cache lookup
InetAddress ip = nameCacheHit.get(host);
// try to resolve host by doing a name cache lookup
ip = nameCacheHit.get(host);
if (ip != null) return ip;
if (nameCacheMiss.containsKey(host)) return null;

@ -2118,7 +2118,7 @@ public class FTPClient {
final int b = Integer.parseInt(st.nextToken());
final int c = Integer.parseInt(st.nextToken());
final int d = Integer.parseInt(st.nextToken());
final InetAddress datahost = InetAddress.getByName(a + "." + b + "." + c + "." + d);
final InetAddress datahost = Domains.dnsResolve(a + "." + b + "." + c + "." + d);
final int high = Integer.parseInt(st.nextToken());
final int low = Integer.parseInt(st.nextToken());
if (high < 0 || high > 255 || low < 0 || low > 255) {

Loading…
Cancel
Save