boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

- added flag to urlhash that shows that domain is a local domain

Browse Source 
- enhanced local domain detection
- bugfixing for memory assignment in kelondroFlexSplit
- automatic memory assignment to caches according to available RAM
- bugfixes for details during search process

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@2924 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 19 years ago
parent eafb5ecd22
commit 114a76a86e
11 changed files with 148 additions and 86 deletions
Show Stats Download Patch File Download Diff File

35
htroot/yacy/hello.java
Unescape View File

@ -46,11 +46,9 @@
// javac -classpath .:../../classes hello.java // javac -classpath .:../../classes hello.java
// if the shell's current path is HTROOT // if the shell's current path is HTROOT
import java.net.InetAddress;
import java.util.Date; import java.util.Date;
import de.anomic.http.httpHeader; import de.anomic.http.httpHeader;
import de.anomic.http.httpc;
import de.anomic.server.serverCore; import de.anomic.server.serverCore;
import de.anomic.server.serverDate; import de.anomic.server.serverDate;
import de.anomic.server.serverObjects; import de.anomic.server.serverObjects;
@ -109,35 +107,14 @@ public final class hello {
// if the previous attempt (using the reported ip address) was not successful, try the ip where // if the previous attempt (using the reported ip address) was not successful, try the ip where
// the request came from // the request came from
if (urls < 0) { if ((urls < 0) && (serverCore.portForwardingEnabled || serverCore.useStaticIP) && (serverCore.isNotLocal(clientip))) {
boolean isLocalIP = false;
if (serverCore.portForwardingEnabled || serverCore.useStaticIP) {
try {
final InetAddress clientAddress = httpc.dnsResolve(clientip);
if (clientAddress != null) {
if (clientAddress.isAnyLocalAddress() || clientAddress.isLoopbackAddress()) {
isLocalIP = true;
} else {
final InetAddress[] localAddress = InetAddress.getAllByName(InetAddress.getLocalHost().getHostName());
for (i = 0; i < localAddress.length; i++) {
if (localAddress[i].equals(clientAddress)) {
isLocalIP = true;
break;
}
}
}
}
} catch (Exception e) {}
}
// we are only allowed to connect to the client IP address if it's not our own address // we are only allowed to connect to the client IP address if it's not our own address
if (!isLocalIP) {
serverCore.checkInterruption();
prop.put(yacySeed.YOURIP, clientip); serverCore.checkInterruption();
remoteSeed.put(yacySeed.IP, clientip);
urls = yacyClient.queryUrlCount(remoteSeed); prop.put(yacySeed.YOURIP, clientip);
} remoteSeed.put(yacySeed.IP, clientip);
urls = yacyClient.queryUrlCount(remoteSeed);
} }
// System.out.println("YACYHELLO: YOUR IP=" + clientip); // System.out.println("YACYHELLO: YOUR IP=" + clientip);

7
source/de/anomic/http/httpc.java
Unescape View File

@ -425,9 +425,6 @@ public final class httpc {
if ((host == null)||(host.length() == 0)) return null; if ((host == null)||(host.length() == 0)) return null;
host = host.toLowerCase().trim(); host = host.toLowerCase().trim();
// flushing old entries before accsessing
flushNameCacheHit();
// trying to resolve host by doing a name cache lookup // trying to resolve host by doing a name cache lookup
InetAddress ip = (InetAddress) nameCacheHit.get(host); InetAddress ip = (InetAddress) nameCacheHit.get(host);
if (ip != null) return ip; if (ip != null) return ip;
@ -456,6 +453,10 @@ public final class httpc {
} }
if (doCaching) { if (doCaching) {
// remove old entries
flushNameCacheHit();
// add new entries
synchronized (nameCacheHit) { synchronized (nameCacheHit) {
nameCacheHit.put(ip.getHostName(), ip); nameCacheHit.put(ip.getHostName(), ip);
nameCacheAges.setScore(ip.getHostName(), intTime(System.currentTimeMillis())); nameCacheAges.setScore(ip.getHostName(), intTime(System.currentTimeMillis()));

4
source/de/anomic/index/indexContainer.java
Unescape View File

@ -285,7 +285,7 @@ public class indexContainer extends kelondroRowSet {
private static indexContainer joinConstructiveByTest(indexContainer small, indexContainer large, long time, int maxDistance) { private static indexContainer joinConstructiveByTest(indexContainer small, indexContainer large, long time, int maxDistance) {
System.out.println("DEBUG: JOIN METHOD BY TEST"); System.out.println("DEBUG: JOIN METHOD BY TEST");
assert small.rowdef.equals(large); assert small.rowdef.equals(large.rowdef) : "small = " + small.rowdef.toString() + "; large = " + large.rowdef.toString();
indexContainer conj = new indexContainer(null, small.rowdef); // start with empty search result indexContainer conj = new indexContainer(null, small.rowdef); // start with empty search result
Iterator se = small.entries(); Iterator se = small.entries();
indexEntry ie0, ie1; indexEntry ie0, ie1;
@ -304,7 +304,7 @@ public class indexContainer extends kelondroRowSet {
private static indexContainer joinConstructiveByEnumeration(indexContainer i1, indexContainer i2, long time, int maxDistance) { private static indexContainer joinConstructiveByEnumeration(indexContainer i1, indexContainer i2, long time, int maxDistance) {
System.out.println("DEBUG: JOIN METHOD BY ENUMERATION"); System.out.println("DEBUG: JOIN METHOD BY ENUMERATION");
assert i1.rowdef.equals(i2); assert i1.rowdef.equals(i2.rowdef) : "i1 = " + i1.rowdef.toString() + "; i2 = " + i2.rowdef.toString();
indexContainer conj = new indexContainer(null, i1.rowdef); // start with empty search result indexContainer conj = new indexContainer(null, i1.rowdef); // start with empty search result
if (!((i1.order().signature().equals(i2.order().signature())) && if (!((i1.order().signature().equals(i2.order().signature())) &&
(i1.primarykey() == i2.primarykey()))) return conj; // ordering must be equal (i1.primarykey() == i2.primarykey()))) return conj; // ordering must be equal

53
source/de/anomic/index/indexURL.java
Unescape View File

@ -40,6 +40,7 @@ import de.anomic.kelondro.kelondroTree;
import de.anomic.net.URL; import de.anomic.net.URL;
import de.anomic.server.serverByteBuffer; import de.anomic.server.serverByteBuffer;
import de.anomic.server.serverCodings; import de.anomic.server.serverCodings;
import de.anomic.server.serverCore;
import de.anomic.yacy.yacySeedDB; import de.anomic.yacy.yacySeedDB;
public class indexURL { public class indexURL {
@ -133,6 +134,7 @@ public class indexURL {
// and culturally close to europe // and culturally close to europe
"AD=Andorra", "AD=Andorra",
"AL=Albania", "AL=Albania",
"AQ=Antarctica",
"AT=Austria", "AT=Austria",
"BA=Bosnia and Herzegovina", "BA=Bosnia and Herzegovina",
"BE=Belgium", "BE=Belgium",
@ -190,6 +192,7 @@ public class indexURL {
"IQ=Iraq", "IQ=Iraq",
"IR=Iran", "IR=Iran",
"PK=Pakistan", "PK=Pakistan",
"TR=Turkey",
"YE=Yemen" "YE=Yemen"
}; };
private static final String[] TLD_SouthEastAsia = { private static final String[] TLD_SouthEastAsia = {
@ -270,10 +273,7 @@ public class indexURL {
"PRO=", "PRO=",
"ARPA=", "ARPA=",
"INT=International", "INT=International",
"ARPA=Arpanet" "ARPA=Arpanet",
};
private static final String[] TLD_Unassigned = {
"AQ=Antarctica",
"NT=Neutral Zone" "NT=Neutral Zone"
}; };
@ -349,7 +349,6 @@ public class indexURL {
"TM=Turkmenistan", "TM=Turkmenistan",
"TO=Tonga", "TO=Tonga",
"TP=East Timor", "TP=East Timor",
"TR=Turkey",
"TT=Trinidad and Tobago", "TT=Trinidad and Tobago",
"TV=Tuvalu", "TV=Tuvalu",
"TW=Taiwan", "TW=Taiwan",
@ -406,7 +405,7 @@ public class indexURL {
insertTLDProps(TLD_NorthAmericaOceania, 4); insertTLDProps(TLD_NorthAmericaOceania, 4);
insertTLDProps(TLD_Africa, 5); insertTLDProps(TLD_Africa, 5);
insertTLDProps(TLD_Generic, 6); insertTLDProps(TLD_Generic, 6);
insertTLDProps(TLD_Unassigned, 7); // the id=7 is used to flag local addresses
} }
@ -479,8 +478,8 @@ public class indexURL {
tld = host.substring(p + 1); tld = host.substring(p + 1);
dom = host.substring(0, p); dom = host.substring(0, p);
} }
Integer ID = (Integer) TLDID.get(tld); Integer ID = (serverCore.isNotLocal(tld)) ? (Integer) TLDID.get(tld) : null; // identify local addresses
int id = (ID == null) ? 7 : ID.intValue(); int id = (ID == null) ? 7 : ID.intValue(); // local addresses are flagged with id=7
boolean isHTTP = url.getProtocol().equals("http"); boolean isHTTP = url.getProtocol().equals("http");
p = dom.lastIndexOf('.'); // locate subdomain p = dom.lastIndexOf('.'); // locate subdomain
String subdom = ""; String subdom = "";
@ -506,18 +505,23 @@ public class indexURL {
if (p > 0) { if (p > 0) {
rootpath = path.substring(0, p); rootpath = path.substring(0, p);
} }
// we collected enough information to compute the fragments that are basis for hashes // we collected enough information to compute the fragments that are basis for hashes
int l = dom.length(); int l = dom.length();
int domlengthKey = (l <= 8) ? 0 : (l <= 12) ? 1 : (l <= 16) ? 2 : 3; int domlengthKey = (l <= 8) ? 0 : (l <= 12) ? 1 : (l <= 16) ? 2 : 3;
byte flagbyte = (byte) (((isHTTP) ? 0 : 32) | (id << 2) | domlengthKey); byte flagbyte = (byte) (((isHTTP) ? 0 : 32) | (id << 2) | domlengthKey);
// combine the attributes
StringBuffer hash = new StringBuffer(12);
// form the 'local' part of the hash // form the 'local' part of the hash
String hash3 = kelondroBase64Order.enhancedCoder.encode(serverCodings.encodeMD5Raw(url.toNormalform())).substring(0, 5); hash.append(kelondroBase64Order.enhancedCoder.encode(serverCodings.encodeMD5Raw(url.toNormalform())).substring(0, 5)); // 5 chars
char hash2 = subdomPortPath(subdom, port, rootpath); hash.append(subdomPortPath(subdom, port, rootpath)); // 1 char
// form the 'global' part of the hash // form the 'global' part of the hash
String hash1 = protocolHostPort(url.getProtocol(), host, port); hash.append(protocolHostPort(url.getProtocol(), host, port)); // 5 chars
char hash0 = kelondroBase64Order.enhancedCoder.encodeByte(flagbyte); hash.append(kelondroBase64Order.enhancedCoder.encodeByte(flagbyte)); // 1 char
// combine the hashes
return hash3 + hash2 + hash1 + hash0; // return result hash
return new String(hash);
} }
private static char subdomPortPath(String subdom, int port, String rootpath) { private static char subdomPortPath(String subdom, int port, String rootpath) {
@ -557,6 +561,8 @@ public class indexURL {
public static final int domLengthEstimation(String urlHash) { public static final int domLengthEstimation(String urlHash) {
// generates an estimation of the original domain length // generates an estimation of the original domain length
assert (urlHash != null);
assert (urlHash.length() == 12) : "urlhash = " + urlHash;
int flagbyte = kelondroBase64Order.enhancedCoder.decodeByte(urlHash.charAt(11)); int flagbyte = kelondroBase64Order.enhancedCoder.decodeByte(urlHash.charAt(11));
int domLengthKey = flagbyte & 3; int domLengthKey = flagbyte & 3;
switch (domLengthKey) { switch (domLengthKey) {
@ -568,9 +574,21 @@ public class indexURL {
return 20; return 20;
} }
public static int domLengthNormalized(String urlHash) { public static int domLengthNormalized(String urlHash) {
return 255 * domLengthEstimation(urlHash) / 30; return 255 * domLengthEstimation(urlHash) / 30;
} }
public static final int domDomain(String urlHash) {
// returns the ID of the domain of the domain
assert (urlHash != null);
assert (urlHash.length() == 12) : "urlhash = " + urlHash;
int flagbyte = kelondroBase64Order.enhancedCoder.decodeByte(urlHash.charAt(11));
return (flagbyte & 12) >> 2;
}
public static boolean isGlobalDomain(String urlhash) {
return domDomain(urlhash) != 7;
}
public static final String oldurlHash(URL url) { public static final String oldurlHash(URL url) {
if (url == null) return null; if (url == null) return null;
@ -584,7 +602,6 @@ public class indexURL {
return hash; return hash;
} }
public static final serverByteBuffer compressIndex(indexContainer inputContainer, indexContainer excludeContainer, long maxtime) { public static final serverByteBuffer compressIndex(indexContainer inputContainer, indexContainer excludeContainer, long maxtime) {
// collect references according to domains // collect references according to domains
long timeout = (maxtime < 0) ? Long.MAX_VALUE : System.currentTimeMillis() + maxtime; long timeout = (maxtime < 0) ? Long.MAX_VALUE : System.currentTimeMillis() + maxtime;

4
source/de/anomic/kelondro/kelondroBase64Order.java
Unescape View File

@ -298,9 +298,9 @@ public class kelondroBase64Order extends kelondroAbstractOrder implements kelond
bc = b[boffset + i]; bc = b[boffset + i];
assert (bc >= 0) && (bc < 128) : "bc = " + bc + ", b = " + serverLog.arrayList(b, boffset, len); assert (bc >= 0) && (bc < 128) : "bc = " + bc + ", b = " + serverLog.arrayList(b, boffset, len);
acc = ahpla[ac]; acc = ahpla[ac];
assert (acc >= 0) : "acc = " + acc + ", a = " + serverLog.arrayList(a, aoffset, len) + ", aoffset = 0x" + Integer.toHexString(aoffset) + serverLog.table(a, 16, aoffset); assert (acc >= 0) : "acc = " + acc + ", a = " + serverLog.arrayList(a, aoffset, len) + ", aoffset = 0x" + Integer.toHexString(aoffset) + ", i = " + i + "\n" + serverLog.table(a, 16, aoffset);
bcc = ahpla[bc]; bcc = ahpla[bc];
assert (bcc >= 0) : "bcc = " + bcc + ", b = " + serverLog.arrayList(b, boffset, len) + ", boffset = 0x" + Integer.toHexString(boffset) + serverLog.table(b, 16, boffset); assert (bcc >= 0) : "bcc = " + bcc + ", b = " + serverLog.arrayList(b, boffset, len) + ", boffset = 0x" + Integer.toHexString(boffset) + ", i = " + i + "\n" + serverLog.table(b, 16, boffset);
if (acc > bcc) return 1; if (acc > bcc) return 1;
if (acc < bcc) return -1; if (acc < bcc) return -1;
// else the bytes are equal and it may go on yet undecided // else the bytes are equal and it may go on yet undecided

2
source/de/anomic/kelondro/kelondroCache.java
Unescape View File

@ -353,6 +353,7 @@ public class kelondroCache implements kelondroIndex {
// remove entry from miss- and hit-cache // remove entry from miss- and hit-cache
if (readMissCache != null) { if (readMissCache != null) {
if (readMissCache.remove(key) != null) { if (readMissCache.remove(key) != null) {
this.hasnotHit++;
// the entry does not exist before // the entry does not exist before
if (writeBufferUnique != null) { if (writeBufferUnique != null) {
// since we know that the entry does not exist, we know that new // since we know that the entry does not exist, we know that new
@ -547,6 +548,7 @@ public class kelondroCache implements kelondroIndex {
if (dummy == null) { if (dummy == null) {
this.hasnotUnique++; this.hasnotUnique++;
} else { } else {
this.hasnotHit++;
this.hasnotDouble++; this.hasnotDouble++;
return null; return null;
} }

53
source/de/anomic/kelondro/kelondroFlexSplitTable.java
Unescape View File

@ -60,31 +60,54 @@ public class kelondroFlexSplitTable implements kelondroIndex {
// first pass: find tables // first pass: find tables
HashMap t = new HashMap(); // file/Integer(size) relation HashMap t = new HashMap(); // file/Integer(size) relation
int size, sum = 0; long ram, sum = 0;
for (int i = 0; i < dir.length; i++) { for (int i = 0; i < dir.length; i++) {
if ((dir[i].startsWith(tablename)) && if ((dir[i].startsWith(tablename)) &&
(dir[i].charAt(tablename.length()) == '.') && (dir[i].charAt(tablename.length()) == '.') &&
(dir[i].length() == tablename.length() + 7)) { (dir[i].length() == tablename.length() + 7)) {
size = kelondroFlexTable.staticSize(path, dir[i]); ram = kelondroFlexTable.staticRAMIndexNeed(path, dir[i], rowdef);
if (size > 0) { if (ram > 0) {
t.put(dir[i], new Integer(size)); t.put(dir[i], new Long(ram));
sum += size; sum += ram;
} }
} }
} }
// second pass: open tables // second pass: open tables
Iterator i = t.entrySet().iterator(); Iterator i;
Map.Entry entry; Map.Entry entry;
String f; String f, maxf;
long bs; long maxram;
while (i.hasNext()) { kelondroIndex table;
entry = (Map.Entry) i.next(); while (t.size() > 0) {
f = (String) entry.getKey(); // find maximum table
size = ((Integer) entry.getValue()).intValue(); maxram = 0;
date = f.substring(tablename.length() + 1); maxf = null;
bs = buffersize * size / sum; i = t.entrySet().iterator();
tables.put(date, new kelondroCache(new kelondroFlexTable(path, f, bs / 2, preloadTime, rowdef, objectOrder), bs / 2, true, false)); while (i.hasNext()) {
entry = (Map.Entry) i.next();
f = (String) entry.getKey();
ram = ((Long) entry.getValue()).longValue();
if (ram > maxram) {
maxf = f;
maxram = ram;
}
}
// open next biggest table
t.remove(maxf);
date = maxf.substring(tablename.length() + 1);
if (maxram <= sum) {
// this will cause usage of a complete RAM index
table = new kelondroCache(new kelondroFlexTable(path, maxf, maxram, preloadTime, rowdef, objectOrder), maxram / 10, true, false);
sum -= maxram;
sum -= maxram / 10;
} else {
// this will cause a generation of a file index
table = new kelondroFlexTable(path, maxf, sum / (t.size() + 1), preloadTime, rowdef, objectOrder);
sum -= sum / (t.size() + 1);
}
tables.put(date, table);
} }
} }

14
source/de/anomic/kelondro/kelondroFlexTable.java
Unescape View File

@ -42,7 +42,7 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr
// if the ram is not sufficient, a tree file is generated // if the ram is not sufficient, a tree file is generated
// if, and only if a tree file exists, the preload time is applied // if, and only if a tree file exists, the preload time is applied
super(path, tablename, rowdef); super(path, tablename, rowdef);
long neededRAM = (super.row().column(0).cellwidth() + 4) * 12 / 10 * super.size(); long neededRAM = (super.row().column(0).cellwidth() + 4) * super.size();
File newpath = new File(path, tablename); File newpath = new File(path, tablename);
File indexfile = new File(newpath, "col.000.index"); File indexfile = new File(newpath, "col.000.index");
@ -78,10 +78,10 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr
System.out.println("*** Using File index " + indexfile); System.out.println("*** Using File index " + indexfile);
ki = new kelondroCache(kelondroTree.open(indexfile, buffersize / 3 * 2, preloadTime, treeIndexRow(rowdef.width(0)), objectOrder, 2, 80), buffersize / 3, true, false); ki = new kelondroCache(kelondroTree.open(indexfile, buffersize / 3 * 2, preloadTime, treeIndexRow(rowdef.width(0)), objectOrder, 2, 80), buffersize / 3, true, false);
RAMIndex = false; RAMIndex = false;
} else if ((preloadTime >= 0) && (stt > preloadTime)) { } else {
// generate new index file // generate new index file
System.out.print("*** Generating File index for " + size() + " entries from " + indexfile); System.out.println("*** Generating File index for " + size() + " entries from " + indexfile);
System.out.print("*** Cause: too less RAM configured. Assign at least " + neededRAM + " bytes buffersize to enable a RAM index."); System.out.println("*** Cause: too less RAM (" + (buffersize / 1024 / 1024) + " MB) configured. Assign at least " + (neededRAM / 1024 / 1024) + " MB buffersize to enable a RAM index.");
ki = initializeTreeIndex(indexfile, buffersize, preloadTime, objectOrder); ki = initializeTreeIndex(indexfile, buffersize, preloadTime, objectOrder);
System.out.println(" -done-"); System.out.println(" -done-");
@ -99,6 +99,10 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr
return kelondroFlexWidthArray.staticsize(path, tablename); return kelondroFlexWidthArray.staticsize(path, tablename);
} }
public static int staticRAMIndexNeed(File path, String tablename, kelondroRow rowdef) {
return (rowdef.column(0).cellwidth() + 4) * staticSize(path, tablename);
}
public boolean hasRAMIndex() { public boolean hasRAMIndex() {
return RAMIndex; return RAMIndex;
} }
@ -157,7 +161,7 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr
treeindex.addUnique(indexentry); treeindex.addUnique(indexentry);
c++; c++;
if (System.currentTimeMillis() - last > 30000) { if (System.currentTimeMillis() - last > 30000) {
System.out.println(".. generated " + c+ " entries, " + ((System.currentTimeMillis() - start) / c * (all - c) / 60000) + " minutes remaining"); System.out.println(".. generated " + c + "/" + all + " entries, " + ((System.currentTimeMillis() - start) / c * (all - c) / 60000) + " minutes remaining");
System.out.flush(); System.out.flush();
last = System.currentTimeMillis(); last = System.currentTimeMillis();
} }

10
source/de/anomic/plasma/plasmaSearchEvent.java
Unescape View File

@ -43,6 +43,7 @@
package de.anomic.plasma; package de.anomic.plasma;
import java.util.Collection; import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet; import java.util.HashSet;
import java.util.Iterator; import java.util.Iterator;
import java.util.Map; import java.util.Map;
@ -169,7 +170,7 @@ public final class plasmaSearchEvent extends Thread implements Runnable {
*/ */
// try to pre-fetch some LURLs if there is enough time // try to pre-fetch some LURLs if there is enough time
indexContainer rcLocal = localSearchJoin((searchContainerMap == null) ? null : searchContainerMap.values()); indexContainer rcLocal = localSearchJoin(searchContainerMap.values());
prefetchLocal(rcLocal, secondaryTimeout); prefetchLocal(rcLocal, secondaryTimeout);
// this is temporary debugging code to learn that the index abstracts are fetched correctly // this is temporary debugging code to learn that the index abstracts are fetched correctly
@ -324,9 +325,9 @@ public final class plasmaSearchEvent extends Thread implements Runnable {
true, true,
true, true,
profileLocal.getTargetTime(plasmaSearchTimingProfile.PROCESS_COLLECTION)); profileLocal.getTargetTime(plasmaSearchTimingProfile.PROCESS_COLLECTION));
if (containers.size() < query.size()) containers = null; // prevent that only a subset is returned if ((containers.size() != 0) && (containers.size() < query.size())) containers = new HashMap(); // prevent that only a subset is returned
profileLocal.setYieldTime(plasmaSearchTimingProfile.PROCESS_COLLECTION); profileLocal.setYieldTime(plasmaSearchTimingProfile.PROCESS_COLLECTION);
profileLocal.setYieldCount(plasmaSearchTimingProfile.PROCESS_COLLECTION, (containers == null) ? 0 : containers.size()); profileLocal.setYieldCount(plasmaSearchTimingProfile.PROCESS_COLLECTION, containers.size());
return containers; return containers;
} }
@ -342,8 +343,9 @@ public final class plasmaSearchEvent extends Thread implements Runnable {
indexContainer rcLocal = indexContainer.joinContainer(containers, indexContainer rcLocal = indexContainer.joinContainer(containers,
profileLocal.getTargetTime(plasmaSearchTimingProfile.PROCESS_JOIN), profileLocal.getTargetTime(plasmaSearchTimingProfile.PROCESS_JOIN),
query.maxDistance); query.maxDistance);
if (rcLocal == null) rcLocal = wordIndex.emptyContainer(null);
profileLocal.setYieldTime(plasmaSearchTimingProfile.PROCESS_JOIN); profileLocal.setYieldTime(plasmaSearchTimingProfile.PROCESS_JOIN);
profileLocal.setYieldCount(plasmaSearchTimingProfile.PROCESS_JOIN, (rcLocal == null) ? 0 : rcLocal.size()); profileLocal.setYieldCount(plasmaSearchTimingProfile.PROCESS_JOIN, rcLocal.size());
return rcLocal; return rcLocal;
} }

11
source/de/anomic/plasma/plasmaSwitchboard.java
Unescape View File

@ -150,6 +150,7 @@ import de.anomic.server.serverCodings;
import de.anomic.server.serverDate; import de.anomic.server.serverDate;
import de.anomic.server.serverFileUtils; import de.anomic.server.serverFileUtils;
import de.anomic.server.serverInstantThread; import de.anomic.server.serverInstantThread;
import de.anomic.server.serverMemory;
import de.anomic.server.serverObjects; import de.anomic.server.serverObjects;
import de.anomic.server.serverSemaphore; import de.anomic.server.serverSemaphore;
import de.anomic.server.serverSwitch; import de.anomic.server.serverSwitch;
@ -367,12 +368,20 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
// read memory amount // read memory amount
int ramLURL = (int) getConfigLong("ramCacheLURL", 1024) / 1024; int ramLURL = (int) getConfigLong("ramCacheLURL", 1024) / 1024;
long ramLURL_time = getConfigLong("ramCacheLURL_time", 1000); long ramLURL_time = getConfigLong("ramCacheLURL_time", 1000);
ramLURL = Math.max((int) (serverMemory.available() / 2 / 1024), ramLURL);
setConfig("ramCacheLURL", ramLURL * 1024);
int ramNURL = (int) getConfigLong("ramCacheNURL", 1024) / 1024; int ramNURL = (int) getConfigLong("ramCacheNURL", 1024) / 1024;
long ramNURL_time = getConfigLong("ramCacheNURL_time", 1000); long ramNURL_time = getConfigLong("ramCacheNURL_time", 1000);
ramNURL = Math.max((int) (serverMemory.available() / 10 / 1024), ramNURL);
setConfig("ramCacheNURL", ramNURL * 1024);
int ramEURL = (int) getConfigLong("ramCacheEURL", 1024) / 1024; int ramEURL = (int) getConfigLong("ramCacheEURL", 1024) / 1024;
long ramEURL_time = getConfigLong("ramCacheEURL_time", 1000); long ramEURL_time = getConfigLong("ramCacheEURL_time", 1000);
ramEURL = Math.max((int) (serverMemory.available() / 20 / 1024), ramEURL);
setConfig("ramCacheEURL", ramEURL * 1024);
int ramRWI = (int) getConfigLong("ramCacheRWI", 1024) / 1024; int ramRWI = (int) getConfigLong("ramCacheRWI", 1024) / 1024;
long ramRWI_time = getConfigLong("ramCacheRWI_time", 1000); long ramRWI_time = getConfigLong("ramCacheRWI_time", 1000);
ramRWI = Math.max((int) (serverMemory.available() / 4 / 1024), ramRWI);
setConfig("ramCacheRWI", ramRWI * 1024);
int ramHTTP = (int) getConfigLong("ramCacheHTTP", 1024) / 1024; int ramHTTP = (int) getConfigLong("ramCacheHTTP", 1024) / 1024;
long ramHTTP_time = getConfigLong("ramCacheHTTP_time", 1000); long ramHTTP_time = getConfigLong("ramCacheHTTP_time", 1000);
int ramMessage = (int) getConfigLong("ramCacheMessage", 1024) / 1024; int ramMessage = (int) getConfigLong("ramCacheMessage", 1024) / 1024;
@ -383,6 +392,8 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
long ramProfiles_time= getConfigLong("ramCacheProfiles_time", 1000); long ramProfiles_time= getConfigLong("ramCacheProfiles_time", 1000);
int ramPreNURL = (int) getConfigLong("ramCachePreNURL", 1024) / 1024; int ramPreNURL = (int) getConfigLong("ramCachePreNURL", 1024) / 1024;
long ramPreNURL_time = getConfigLong("ramCachePreNURL_time", 1000); long ramPreNURL_time = getConfigLong("ramCachePreNURL_time", 1000);
ramPreNURL = Math.max((int) (serverMemory.available() / 10 / 1024), ramPreNURL);
setConfig("ramCachePreNURL", ramPreNURL * 1024);
int ramWiki = (int) getConfigLong("ramCacheWiki", 1024) / 1024; int ramWiki = (int) getConfigLong("ramCacheWiki", 1024) / 1024;
long ramWiki_time = getConfigLong("ramCacheWiki_time", 1000); long ramWiki_time = getConfigLong("ramCacheWiki_time", 1000);
int ramBlog = (int) getConfigLong("ramCacheBlog", 1024) / 1024; int ramBlog = (int) getConfigLong("ramCacheBlog", 1024) / 1024;

39
source/de/anomic/server/serverCore.java
Unescape View File

@ -121,6 +121,15 @@ public final class serverCore extends serverAbstractThread implements serverThre
Hashtable denyHost; Hashtable denyHost;
int commandMaxLength; int commandMaxLength;
private static InetAddress[] localAddresses = null;
static {
try {
localAddresses = InetAddress.getAllByName(InetAddress.getLocalHost().getHostName());
} catch (UnknownHostException e) {
localAddresses = new InetAddress[0];
}
}
/** /**
* The session-object pool * The session-object pool
*/ */
@ -402,13 +411,29 @@ public final class serverCore extends serverAbstractThread implements serverThre
return isNotLocal(url.getHost()); return isNotLocal(url.getHost());
} }
static boolean isNotLocal(String ip) { public static boolean isNotLocal(String ip) {
if ((ip.equals("localhost")) || // generate ip address if ip is given by host
(ip.startsWith("127")) || assert (ip != null);
(ip.startsWith("192.168")) || final InetAddress clientAddress = httpc.dnsResolve(ip);
(ip.startsWith("10.")) if (clientAddress != null) {
) return false; if ((clientAddress.isAnyLocalAddress()) || (clientAddress.isLoopbackAddress())) return false;
return true; if (ip.charAt(0) > '9') ip = clientAddress.getHostAddress();
}
// check local ip addresses
if ((ip.equals("localhost")) ||
(ip.startsWith("127")) ||
(ip.startsWith("192.168")) ||
(ip.startsWith("10."))
) return false;
// finally check if there are other local IP adresses that are not in the standard IP range
for (int i = 0; i < localAddresses.length; i++) {
if (localAddresses[i].equals(clientAddress)) return false;
}
// the address must be a gloabl IP address
return true;
} }
public static String publicIP() { public static String publicIP() {

Write Preview
Loading…
Cancel
Save