boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

another performance hack: re-use of known host addresses for isLocal property; avoids look-up in local hash

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7983 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 14 years ago
parent 035ebfbf3b
commit 813f297a95
8 changed files with 104 additions and 97 deletions
Show Stats Download Patch File Download Diff File

3
htroot/Collage.java
Unescape View File

@ -28,7 +28,6 @@ import net.yacy.cora.document.MultiProtocolURI;
import net.yacy.cora.protocol.Domains; import net.yacy.cora.protocol.Domains;
import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.protocol.RequestHeader;
import net.yacy.search.Switchboard; import net.yacy.search.Switchboard;
import de.anomic.crawler.ResultImages; import de.anomic.crawler.ResultImages;
import de.anomic.server.serverObjects; import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch; import de.anomic.server.serverSwitch;
@ -95,7 +94,7 @@ public class Collage {
// check if this loads a page from localhost, which must be prevented to protect the server // check if this loads a page from localhost, which must be prevented to protect the server
// against attacks to the administration interface when localhost access is granted // against attacks to the administration interface when localhost access is granted
if ((Domains.isLocal(baseURL.getHost()) || Domains.isLocal(imageURL.getHost())) && if ((Domains.isLocal(baseURL.getHost(), null) || Domains.isLocal(imageURL.getHost(), null)) &&
sb.getConfigBool("adminAccountForLocalhost", false)) continue; sb.getConfigBool("adminAccountForLocalhost", false)) continue;
final long z = imgZIndex[i]; final long z = imgZIndex[i];

2
htroot/yacy/search.java
Unescape View File

@ -167,7 +167,7 @@ public final class search {
block = true; block = true;
} }
} }
if (block && Domains.isLocal(client)) block = false; if (block && Domains.isLocal(client, null)) block = false;
if (block) { if (block) {
prop.put("links", ""); prop.put("links", "");
prop.put("linkcount", "0"); prop.put("linkcount", "0");

56
source/de/anomic/crawler/Latency.java
Unescape View File

@ -40,8 +40,8 @@ public class Latency {
// the map is a mapping from host names to host configurations // the map is a mapping from host names to host configurations
private static final ConcurrentHashMap<String, Host> map = new ConcurrentHashMap<String, Host>(); private static final ConcurrentHashMap<String, Host> map = new ConcurrentHashMap<String, Host>();
public static void update(MultiProtocolURI url, long time) { public static void update(final MultiProtocolURI url, final long time) {
String host = url.getHost(); final String host = url.getHost();
if (host == null) return; if (host == null) return;
Host h = map.get(host); Host h = map.get(host);
if (h == null) { if (h == null) {
@ -53,8 +53,8 @@ public class Latency {
} }
} }
public static void update(MultiProtocolURI url) { public static void update(final MultiProtocolURI url) {
String host = url.getHost(); final String host = url.getHost();
if (host == null) return; if (host == null) return;
Host h = map.get(host); Host h = map.get(host);
if (h == null) { if (h == null) {
@ -66,8 +66,8 @@ public class Latency {
} }
} }
public static void slowdown(MultiProtocolURI url) { public static void slowdown(final MultiProtocolURI url) {
String host = url.getHost(); final String host = url.getHost();
if (host == null) return; if (host == null) return;
Host h = map.get(host); Host h = map.get(host);
if (h == null) { if (h == null) {
@ -79,16 +79,16 @@ public class Latency {
} }
} }
public static Host host(MultiProtocolURI url) { public static Host host(final MultiProtocolURI url) {
String host = url.getHost(); final String host = url.getHost();
if (host == null) return null; if (host == null) return null;
return map.get(host); return map.get(host);
} }
public static int average(MultiProtocolURI url) { public static int average(final MultiProtocolURI url) {
String host = url.getHost(); final String host = url.getHost();
if (host == null) return 0; if (host == null) return 0;
Host h = map.get(host); final Host h = map.get(host);
if (h == null) return 0; if (h == null) return 0;
return h.average(); return h.average();
} }
@ -103,7 +103,7 @@ public class Latency {
* @param urlhash * @param urlhash
* @return a time in milliseconds since last access of the domain or Long.MAX_VALUE if the domain was not accessed before * @return a time in milliseconds since last access of the domain or Long.MAX_VALUE if the domain was not accessed before
*/ */
public static long lastAccessDelta(MultiProtocolURI url) { public static long lastAccessDelta(final MultiProtocolURI url) {
final Latency.Host host = Latency.host(url); final Latency.Host host = Latency.host(url);
if (host == null) return Long.MAX_VALUE; // never accessed if (host == null) return Long.MAX_VALUE; // never accessed
return System.currentTimeMillis() - host.lastacc(); return System.currentTimeMillis() - host.lastacc();
@ -121,16 +121,16 @@ public class Latency {
* @return the remaining waiting time in milliseconds. The return value may be negative * @return the remaining waiting time in milliseconds. The return value may be negative
* which expresses how long the time is over the minimum waiting time. * which expresses how long the time is over the minimum waiting time.
*/ */
public static long waitingRemainingGuessed(String hostname, final long minimumLocalDelta, final long minimumGlobalDelta) { public static long waitingRemainingGuessed(final String hostname, final long minimumLocalDelta, final long minimumGlobalDelta) {
if (hostname == null) return 0; if (hostname == null) return 0;
Host host = map.get(hostname); final Host host = map.get(hostname);
if (host == null) return 0; if (host == null) return 0;
// the time since last access to the domain is the basis of the remaining calculation // the time since last access to the domain is the basis of the remaining calculation
final long timeSinceLastAccess = System.currentTimeMillis() - host.lastacc(); final long timeSinceLastAccess = System.currentTimeMillis() - host.lastacc();
// find the minimum waiting time based on the network domain (local or global) // find the minimum waiting time based on the network domain (local or global)
final boolean local = Domains.isLocal(hostname); final boolean local = Domains.isLocal(hostname, null);
long waiting = (local) ? minimumLocalDelta : minimumGlobalDelta; long waiting = (local) ? minimumLocalDelta : minimumGlobalDelta;
// if we have accessed the domain many times, get slower (the flux factor) // if we have accessed the domain many times, get slower (the flux factor)
@ -161,10 +161,10 @@ public class Latency {
* @param minimumGlobalDelta * @param minimumGlobalDelta
* @return the remaining waiting time in milliseconds * @return the remaining waiting time in milliseconds
*/ */
public static long waitingRemaining(MultiProtocolURI url, final Set<String> thisAgents, final long minimumLocalDelta, final long minimumGlobalDelta) { public static long waitingRemaining(final MultiProtocolURI url, final Set<String> thisAgents, final long minimumLocalDelta, final long minimumGlobalDelta) {
// first check if the domain was _ever_ accessed before // first check if the domain was _ever_ accessed before
Host host = host(url); final Host host = host(url);
if (host == null) return Long.MIN_VALUE; // no delay if host is new if (host == null) return Long.MIN_VALUE; // no delay if host is new
// find the minimum waiting time based on the network domain (local or global) // find the minimum waiting time based on the network domain (local or global)
@ -189,7 +189,7 @@ public class Latency {
RobotsTxtEntry robotsEntry; RobotsTxtEntry robotsEntry;
try { try {
robotsEntry = Switchboard.getSwitchboard().robots.getEntry(url, thisAgents); robotsEntry = Switchboard.getSwitchboard().robots.getEntry(url, thisAgents);
} catch (IOException e) { } catch (final IOException e) {
robotsEntry = null; robotsEntry = null;
} }
robotsDelay = (robotsEntry == null) ? 0 : robotsEntry.getCrawlDelayMillis(); robotsDelay = (robotsEntry == null) ? 0 : robotsEntry.getCrawlDelayMillis();
@ -211,17 +211,17 @@ public class Latency {
} }
public static String waitingRemainingExplain(MultiProtocolURI url, final Set<String> thisAgents, final long minimumLocalDelta, final long minimumGlobalDelta) { public static String waitingRemainingExplain(final MultiProtocolURI url, final Set<String> thisAgents, final long minimumLocalDelta, final long minimumGlobalDelta) {
// first check if the domain was _ever_ accessed before // first check if the domain was _ever_ accessed before
Host host = host(url); final Host host = host(url);
if (host == null) return "host " + host + " never accessed before -> 0"; // no delay if host is new if (host == null) return "host " + host + " never accessed before -> 0"; // no delay if host is new
StringBuilder s = new StringBuilder(50); final StringBuilder s = new StringBuilder(50);
// find the minimum waiting time based on the network domain (local or global) // find the minimum waiting time based on the network domain (local or global)
final boolean local = url.isLocal(); final boolean local = url.isLocal();
long waiting = (local) ? minimumLocalDelta : minimumGlobalDelta; final long waiting = (local) ? minimumLocalDelta : minimumGlobalDelta;
s.append("minimumDelta = ").append(waiting); s.append("minimumDelta = ").append(waiting);
// the time since last access to the domain is the basis of the remaining calculation // the time since last access to the domain is the basis of the remaining calculation
@ -242,7 +242,7 @@ public class Latency {
RobotsTxtEntry robotsEntry; RobotsTxtEntry robotsEntry;
try { try {
robotsEntry = Switchboard.getSwitchboard().robots.getEntry(url, thisAgents); robotsEntry = Switchboard.getSwitchboard().robots.getEntry(url, thisAgents);
} catch (IOException e) { } catch (final IOException e) {
robotsEntry = null; robotsEntry = null;
} }
robotsDelay = (robotsEntry == null) ? 0 : robotsEntry.getCrawlDelayMillis(); robotsDelay = (robotsEntry == null) ? 0 : robotsEntry.getCrawlDelayMillis();
@ -264,14 +264,14 @@ public class Latency {
private int count; private int count;
private final String host; private final String host;
private long robotsMinDelay; private long robotsMinDelay;
public Host(String host, long time) { public Host(final String host, final long time) {
this.host = host; this.host = host;
this.timeacc = time; this.timeacc = time;
this.count = 1; this.count = 1;
this.lastacc = System.currentTimeMillis(); this.lastacc = System.currentTimeMillis();
this.robotsMinDelay = 0; this.robotsMinDelay = 0;
} }
public void update(long time) { public void update(final long time) {
this.lastacc = System.currentTimeMillis(); this.lastacc = System.currentTimeMillis();
this.timeacc += Math.min(30000, time); this.timeacc += Math.min(30000, time);
this.count++; this.count++;
@ -296,14 +296,14 @@ public class Latency {
public String host() { public String host() {
return this.host; return this.host;
} }
public void robotsDelay(long ur) { public void robotsDelay(final long ur) {
this.robotsMinDelay = ur; this.robotsMinDelay = ur;
} }
public long robotsDelay() { public long robotsDelay() {
return this.robotsMinDelay; return this.robotsMinDelay;
} }
public long flux(long range) { public long flux(final long range) {
return count >= 1000 ? range * Math.min(5000, count) / 1000 : range / (1000 - count); return this.count >= 1000 ? range * Math.min(5000, this.count) / 1000 : range / (1000 - this.count);
} }
} }

2
source/de/anomic/server/serverCore.java
Unescape View File

@ -171,7 +171,7 @@ public final class serverCore extends AbstractBusyThread implements BusyThread {
final InetAddress uAddr = s.getInetAddress(); final InetAddress uAddr = s.getInetAddress();
if (uAddr.isAnyLocalAddress()) return "127.0.0.1"; if (uAddr.isAnyLocalAddress()) return "127.0.0.1";
String cIP = uAddr.getHostAddress(); String cIP = uAddr.getHostAddress();
if (Domains.isLocal(cIP)) cIP = "127.0.0.1"; if (Domains.isLocal(cIP, null)) cIP = "127.0.0.1";
return cIP; return cIP;
} }

4
source/net/yacy/cora/document/MultiProtocolURI.java
Unescape View File

@ -88,7 +88,7 @@ public class MultiProtocolURI implements Serializable, Comparable<MultiProtocolU
protected final String protocol, userInfo; protected final String protocol, userInfo;
protected String host, path, quest, ref; protected String host, path, quest, ref;
protected int port; protected int port;
private InetAddress hostAddress; protected InetAddress hostAddress;
/** /**
* initialization of a MultiProtocolURI to produce poison pills for concurrent blocking queues * initialization of a MultiProtocolURI to produce poison pills for concurrent blocking queues
@ -920,7 +920,7 @@ public class MultiProtocolURI implements Serializable, Comparable<MultiProtocolU
// checks for local/global IP range and local IP // checks for local/global IP range and local IP
public boolean isLocal() { public boolean isLocal() {
return this.isFile() || this.isSMB() || Domains.isLocal(this.host); return this.isFile() || this.isSMB() || Domains.isLocal(this.host, this.hostAddress);
} }
// language calculation // language calculation

24
source/net/yacy/cora/protocol/Domains.java
Unescape View File

@ -850,8 +850,8 @@ public class Domains {
return isThisHostIP; return isThisHostIP;
} }
public static int getDomainID(final String host) { public static int getDomainID(final String host, final InetAddress hostaddress) {
if (host == null || host.isEmpty() || isLocal(host)) return TLD_Local_ID; if (host == null || host.isEmpty() || isLocal(host, hostaddress)) return TLD_Local_ID;
final int p = host.lastIndexOf('.'); final int p = host.lastIndexOf('.');
final String tld = (p > 0) ? host.substring(p + 1) : ""; final String tld = (p > 0) ? host.substring(p + 1) : "";
final Integer i = TLDID.get(tld); final Integer i = TLDID.get(tld);
@ -875,11 +875,19 @@ public class Domains {
); );
} }
public static boolean isLocal(final String host) { public static boolean isLocal(final String host, final InetAddress hostaddress) {
return isLocal(host, true); return isLocal(host, hostaddress, true);
} }
private static boolean isLocal(final String host, final boolean recursive) { /**
* check if the given host is a local address.
* the hostaddress is optional and shall be given if the address is already known
* @param host
* @param hostaddress may be null if not known yet
* @param recursive
* @return true if the given host is local
*/
private static boolean isLocal(final String host, InetAddress hostaddress, final boolean recursive) {
if (noLocalCheck || // DO NOT REMOVE THIS! it is correct to return true if the check is off if (noLocalCheck || // DO NOT REMOVE THIS! it is correct to return true if the check is off
host == null || host == null ||
@ -900,8 +908,8 @@ public class Domains {
// check dns lookup: may be a local address even if the domain name looks global // check dns lookup: may be a local address even if the domain name looks global
if (!recursive) return false; if (!recursive) return false;
final InetAddress a = dnsResolve(host); if (hostaddress == null) hostaddress = dnsResolve(host);
return isLocal(a); return isLocal(hostaddress);
} }
public static boolean isLocal(final InetAddress a) { public static boolean isLocal(final InetAddress a) {
@ -912,7 +920,7 @@ public class Domains {
a.isLinkLocalAddress() | a.isLinkLocalAddress() |
a.isLoopbackAddress() || a.isLoopbackAddress() ||
a.isSiteLocalAddress() || a.isSiteLocalAddress() ||
isLocal(a.getHostAddress(), false); isLocal(a.getHostAddress(), a, false);
return localp; return localp;
} }

4
source/net/yacy/kelondro/data/meta/DigestURI.java
Unescape View File

@ -175,7 +175,7 @@ public class DigestURI extends MultiProtocolURI implements Serializable {
assert this.hash == null; // should only be called if the hash was not computed before assert this.hash == null; // should only be called if the hash was not computed before
final int id = Domains.getDomainID(this.host); // id=7: tld is local final int id = Domains.getDomainID(this.host, this.hostAddress); // id=7: tld is local
final boolean isHTTP = isHTTP(); final boolean isHTTP = isHTTP();
int p = (this.host == null) ? -1 : this.host.lastIndexOf('.'); int p = (this.host == null) ? -1 : this.host.lastIndexOf('.');
String dom = (p > 0) ? dom = this.host.substring(0, p) : ""; String dom = (p > 0) ? dom = this.host.substring(0, p) : "";
@ -278,7 +278,7 @@ public class DigestURI extends MultiProtocolURI implements Serializable {
*/ */
public static final String hosthash6(final String protocol, final String host, final int port) { public static final String hosthash6(final String protocol, final String host, final int port) {
final StringBuilder hash = new StringBuilder(12); final StringBuilder hash = new StringBuilder(12);
final int id = Domains.getDomainID(host); // id=7: tld is local final int id = Domains.getDomainID(host, null); // id=7: tld is local
int p = host.lastIndexOf('.'); int p = host.lastIndexOf('.');
String dom = (p > 0) ? dom = host.substring(0, p) : ""; String dom = (p > 0) ? dom = host.substring(0, p) : "";
p = dom.lastIndexOf('.'); p = dom.lastIndexOf('.');

8
source/net/yacy/peers/yacySeed.java
Unescape View File

@ -521,7 +521,7 @@ public class yacySeed implements Cloneable, Comparable<yacySeed>, Comparator<yac
// because java thinks it must apply the UTC offset to the current time, // because java thinks it must apply the UTC offset to the current time,
// to create a string that looks like our current time, it adds the local UTC offset to the // to create a string that looks like our current time, it adds the local UTC offset to the
// time. To create a corrected UTC Date string, we first subtract the local UTC offset. // time. To create a corrected UTC Date string, we first subtract the local UTC offset.
GenericFormatter my_SHORT_SECOND_FORMATTER = new GenericFormatter(GenericFormatter.FORMAT_SHORT_SECOND, GenericFormatter.time_second); // use our own formatter to prevent concurrency locks with other processes final GenericFormatter my_SHORT_SECOND_FORMATTER = new GenericFormatter(GenericFormatter.FORMAT_SHORT_SECOND, GenericFormatter.time_second); // use our own formatter to prevent concurrency locks with other processes
final String ls = my_SHORT_SECOND_FORMATTER.format(new Date(System.currentTimeMillis() /*- DateFormatter.UTCDiff()*/)); final String ls = my_SHORT_SECOND_FORMATTER.format(new Date(System.currentTimeMillis() /*- DateFormatter.UTCDiff()*/));
//System.out.println("SETTING LAST-SEEN of " + this.getName() + " to " + ls); //System.out.println("SETTING LAST-SEEN of " + this.getName() + " to " + ls);
this.dna.put(yacySeed.LASTSEEN, ls ); this.dna.put(yacySeed.LASTSEEN, ls );
@ -532,7 +532,7 @@ public class yacySeed implements Cloneable, Comparable<yacySeed>, Comparator<yac
*/ */
public final long getLastSeenUTC() { public final long getLastSeenUTC() {
try { try {
GenericFormatter my_SHORT_SECOND_FORMATTER = new GenericFormatter(GenericFormatter.FORMAT_SHORT_SECOND, GenericFormatter.time_second); // use our own formatter to prevent concurrency locks with other processes final GenericFormatter my_SHORT_SECOND_FORMATTER = new GenericFormatter(GenericFormatter.FORMAT_SHORT_SECOND, GenericFormatter.time_second); // use our own formatter to prevent concurrency locks with other processes
final long t = my_SHORT_SECOND_FORMATTER.parse(get(yacySeed.LASTSEEN, "20040101000000")).getTime(); final long t = my_SHORT_SECOND_FORMATTER.parse(get(yacySeed.LASTSEEN, "20040101000000")).getTime();
// getTime creates a UTC time number. But in this case java thinks, that the given // getTime creates a UTC time number. But in this case java thinks, that the given
// time string is a local time, which has a local UTC offset applied. // time string is a local time, which has a local UTC offset applied.
@ -562,7 +562,7 @@ public class yacySeed implements Cloneable, Comparable<yacySeed>, Comparator<yac
if (this.birthdate > 0) return this.birthdate; if (this.birthdate > 0) return this.birthdate;
long b; long b;
try { try {
GenericFormatter my_SHORT_SECOND_FORMATTER = new GenericFormatter(GenericFormatter.FORMAT_SHORT_SECOND, GenericFormatter.time_second); // use our own formatter to prevent concurrency locks with other processes final GenericFormatter my_SHORT_SECOND_FORMATTER = new GenericFormatter(GenericFormatter.FORMAT_SHORT_SECOND, GenericFormatter.time_second); // use our own formatter to prevent concurrency locks with other processes
b = my_SHORT_SECOND_FORMATTER.parse(get(yacySeed.BDATE, "20040101000000")).getTime(); b = my_SHORT_SECOND_FORMATTER.parse(get(yacySeed.BDATE, "20040101000000")).getTime();
} catch (final ParseException e) { } catch (final ParseException e) {
b = System.currentTimeMillis(); b = System.currentTimeMillis();
@ -864,7 +864,7 @@ public class yacySeed implements Cloneable, Comparable<yacySeed>, Comparator<yac
if (ipString == null) return ipString + " -> IP is null"; if (ipString == null) return ipString + " -> IP is null";
if (ipString.length() > 0 && ipString.length() < 8) return ipString + " -> IP is too short: "; if (ipString.length() > 0 && ipString.length() < 8) return ipString + " -> IP is too short: ";
if (Switchboard.getSwitchboard().isAllIPMode()) return null; if (Switchboard.getSwitchboard().isAllIPMode()) return null;
final boolean islocal = Domains.isLocal(ipString); final boolean islocal = Domains.isLocal(ipString, null);
//if (islocal && Switchboard.getSwitchboard().isGlobalMode()) return ipString + " - local IP for global mode rejected"; //if (islocal && Switchboard.getSwitchboard().isGlobalMode()) return ipString + " - local IP for global mode rejected";
if (!islocal && Switchboard.getSwitchboard().isIntranetMode()) return ipString + " - global IP for intranet mode rejected"; if (!islocal && Switchboard.getSwitchboard().isIntranetMode()) return ipString + " - global IP for intranet mode rejected";
return null; return null;

Write Preview
Loading…
Cancel
Save