boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

avoiding OutOfMemoryError routines

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@302 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 20 years ago
parent 3e8ee5a46d
commit e3c92818db
7 changed files with 109 additions and 121 deletions
Show Stats Download Patch File Download Diff File

1
source/de/anomic/kelondro/kelondroRecords.java
Unescape View File

@ -682,6 +682,7 @@ public class kelondroRecords {
cacheNode = null;
// check cache size
if (XcacheHeaders.size() > sizeBefore) checkCacheSpace();
//System.out.println("kelondroRecords cache4" + filename + ": " + XcacheHeaders.size() + " entries, " + XcacheSize + " allowed.");
}
}
}

18
source/de/anomic/plasma/plasmaSwitchboard.java
Unescape View File

@ -578,6 +578,12 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
//log.logDebug("CoreCrawl: queue is empty");
return false;
}
if (Runtime.getRuntime().freeMemory() < 2000000) {
log.logDebug("CoreCrawl: not enough memory available, dismissed (" +
"free=" + Runtime.getRuntime().freeMemory() + ")");
System.gc();
return false;
}
if (queueStack.size() >= crawlSlots) {
log.logDebug("CoreCrawl: too many processes in queue, dismissed (" +
"queueStack=" + queueStack.size() + ")");
@ -633,6 +639,12 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
//log.logDebug("LimitCrawl: queue is empty");
return false;
}
if (Runtime.getRuntime().freeMemory() < 2000000) {
log.logDebug("limitCrawlTrigger: not enough memory available, dismissed (" +
"free=" + Runtime.getRuntime().freeMemory() + ")");
System.gc();
return false;
}
// if the server is busy, we do crawling more slowly
//if (!(cacheManager.idle())) try {Thread.currentThread().sleep(2000);} catch (InterruptedException e) {}
@ -705,6 +717,12 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
//log.logDebug("GlobalCrawl: queue is empty");
return false;
}
if (Runtime.getRuntime().freeMemory() < 2000000) {
log.logDebug("remoteTriggeredCrawl: not enough memory available, dismissed (" +
"free=" + Runtime.getRuntime().freeMemory() + ")");
System.gc();
return false;
}
/*
if (queueStack.size() > 0) {
log.logDebug("GlobalCrawl: any processe is in queue, dismissed (" +

4
source/de/anomic/plasma/plasmaWordIndexAssortment.java
Unescape View File

@ -104,7 +104,7 @@ public final class plasmaWordIndexAssortment {
this.bufferSize = bufferkb * 1024;
this.log = log;
if (assortmentFile.exists()) {
// open existing singeton tree file
// open existing assortment tree file
try {
assortments = new kelondroTree(assortmentFile, bufferSize);
if (log != null) log.logSystem("Opened Assortment Database, " + assortments.size() + " entries, width " + assortmentCapacity + ", " + bufferkb + "kb buffer");
@ -113,7 +113,7 @@ public final class plasmaWordIndexAssortment {
e.printStackTrace();
}
} else {
// create new sigleton tree file
// create new assortment tree file
try {
assortments = new kelondroTree(assortmentFile, bufferSize, bufferStructure(assortmentCapacity));
if (log != null) log.logSystem("Created new Assortment Database, width " + assortmentCapacity + ", " + bufferkb + "kb buffer");

3
source/de/anomic/plasma/plasmaWordIndexAssortmentCluster.java
Unescape View File

@ -77,11 +77,12 @@ public final class plasmaWordIndexAssortmentCluster {
sizes[i] = testAssortment.size() + clusterCapacity - i;
sumSizes += sizes[i];
testAssortment.close();
testAssortment = null;
}
// initialize cluster using the cluster elements size for optimal buffer size
for (int i = 0; i < clusterCapacity; i++) {
assortments[i] = new plasmaWordIndexAssortment(assortmentsPath, i + 1, (int) completeBufferKB * sizes[i] / sumSizes, log);
assortments[i] = new plasmaWordIndexAssortment(assortmentsPath, i + 1, (int) ((long) completeBufferKB * (long) sizes[i] / (long) sumSizes), log);
}
}

34
source/de/anomic/plasma/plasmaWordIndexCache.java
Unescape View File

@ -193,6 +193,7 @@ public final class plasmaWordIndexCache implements plasmaWordIndexInterface {
long creationTime;
plasmaWordIndexEntry wordEntry;
byte[][] row;
Runtime rt = Runtime.getRuntime();
while (i.hasNext()) {
// get out one entry
node = (kelondroRecords.Node) i.next();
@ -200,11 +201,16 @@ public final class plasmaWordIndexCache implements plasmaWordIndexInterface {
wordHash = new String(row[0]);
creationTime = kelondroRecords.bytes2long(row[2]);
wordEntry = new plasmaWordIndexEntry(new String(row[3]), new String(row[4]));
// store to cache
addEntry(wordHash, wordEntry, creationTime);
urlCount++;
// protect against memory shortage
while (rt.freeMemory() < 1000000) {
//System.out.print("FLUSH+GC bevore=" + rt.freeMemory());
flushFromMem();
System.gc();
//System.out.println(", after=" + rt.freeMemory());
}
// write a log
if (System.currentTimeMillis() > messageTime) {
urlsPerSecond = 1 + urlCount * 1000 / (1 + System.currentTimeMillis() - startTime);
@ -271,12 +277,15 @@ public final class plasmaWordIndexCache implements plasmaWordIndexInterface {
public void run() {
String nextHash;
Runtime rt = Runtime.getRuntime();
while (!terminate) {
if (pause) {
try {this.sleep(300);} catch (InterruptedException e) {}
} else {
flushFromMem();
try {this.sleep(10 + java.lang.Math.min(1000, 10 * maxWords/(cache.size() + 1)));} catch (InterruptedException e) {}
if ((rt.freeMemory() > 1000000) || (cache.size() == 0)) try {
this.sleep(10 + java.lang.Math.min(1000, 10 * maxWords/(cache.size() + 1)));
} catch (InterruptedException e) {}
}
}
}
@ -388,7 +397,7 @@ public final class plasmaWordIndexCache implements plasmaWordIndexInterface {
}
private boolean flushFromAssortmentCluster(String key) {
// this should only be called if the singleton shall be deleted or returned in an index entity
// this should only be called if the assortment shall be deleted or returned in an index entity
plasmaWordIndexEntryContainer container = assortmentCluster.removeFromAll(key);
if (container == null) {
return false;
@ -543,6 +552,10 @@ public final class plasmaWordIndexCache implements plasmaWordIndexInterface {
flushThread.pause();
//serverLog.logDebug("PLASMA INDEXING", "addEntryToIndexMem: cache.size=" + cache.size() + "; hashScore.size=" + hashScore.size());
while (cache.size() >= this.maxWords) flushFromMem();
while ((cache.size() > 0) && (Runtime.getRuntime().freeMemory() < 1000000)) {
flushFromMem();
System.gc();
}
//if (flushc > 0) serverLog.logDebug("PLASMA INDEXING", "addEntryToIndexMem - flushed " + flushc + " entries");
// put new words into cache
@ -557,6 +570,7 @@ public final class plasmaWordIndexCache implements plasmaWordIndexInterface {
hashScore.addScore(wordHash, added);
hashDate.setScore(wordHash, intTime(updateTime));
}
entries = null;
}
//System.out.println("DEBUG: cache = " + cache.toString());
flushThread.proceed();
@ -564,13 +578,17 @@ public final class plasmaWordIndexCache implements plasmaWordIndexInterface {
}
private void addEntry(String wordHash, plasmaWordIndexEntry newEntry, long updateTime) {
plasmaWordIndexEntryContainer entries = (plasmaWordIndexEntryContainer) cache.get(wordHash);
if (entries == null) entries = new plasmaWordIndexEntryContainer(wordHash);
if (entries.add(new plasmaWordIndexEntry[]{newEntry}, updateTime) > 0) {
cache.put(wordHash, entries);
flushThread.pause();
plasmaWordIndexEntryContainer container = (plasmaWordIndexEntryContainer) cache.get(wordHash);
if (container == null) container = new plasmaWordIndexEntryContainer(wordHash);
plasmaWordIndexEntry[] entries = new plasmaWordIndexEntry[]{newEntry};
if (container.add(entries, updateTime) > 0) {
cache.put(wordHash, container);
hashScore.incScore(wordHash);
hashDate.setScore(wordHash, intTime(updateTime));
}
entries = null;
container = null;
flushThread.proceed();
}

157
source/de/anomic/plasma/plasmaWordIndexEntry.java
Unescape View File

@ -76,7 +76,7 @@ public class plasmaWordIndexEntry {
private int posofphrase; // position of the phrase in the text as count of sentences; 0=unknown; 1=path; 2=keywords; 3=headline; >4: in text
private int age; // calculated by using last-modified
private int quality; // result of a heuristic on the source file
private String language; // essentially the country code (the TLD as heuristic), two letters lowercase only
private byte[] language; // essentially the country code (the TLD as heuristic), two letters lowercase only
private char doctype; // type of source
private char localflag; // indicates if the index was created locally
@ -97,9 +97,6 @@ public class plasmaWordIndexEntry {
public static final char LT_LOCAL = 'L';
public static final char LT_GLOBAL = 'G';
// encoded discrete values
private String code;
// create a word hash
public static String word2hash(String word) {
return serverCodings.encodeMD5B64(word.toLowerCase(), true).substring(0, wordHashLength);
@ -191,25 +188,23 @@ public class plasmaWordIndexEntry {
this.posofphrase = posofphrase;
this.age = virtualage;
this.quality = quality;
this.language = language;
this.language = language.getBytes();
this.doctype = doctype;
this.code = null;
this.localflag = (local) ? LT_LOCAL : LT_GLOBAL;
}
public plasmaWordIndexEntry(String urlHash, String code) {
// the code is not parsed but used later on
this.urlHash = urlHash;
this.count = 0;
this.posintext = 0;
this.posinphrase = 0;
this.posofphrase = 0;
this.age = 0;
this.quality = 0;
this.language = "uk";
this.doctype = 'u';
this.code = code;
this.localflag = LT_LOCAL;
this.count = (int) serverCodings.enhancedCoder.decodeBase64Long(code.substring(6, 8));
this.posintext = (code.length() >= 14) ? (int) serverCodings.enhancedCoder.decodeBase64Long(code.substring(12, 14)) : 0;
this.posinphrase = (code.length() >= 15) ? (int) serverCodings.enhancedCoder.decodeBase64Long(code.substring(14, 16)) : 0;
this.posofphrase = (code.length() >= 16) ? (int) serverCodings.enhancedCoder.decodeBase64Long(code.substring(16, 18)) : 0;
this.age = (int) serverCodings.enhancedCoder.decodeBase64Long(code.substring(3, 6));
this.quality = (int) serverCodings.enhancedCoder.decodeBase64Long(code.substring(0, 3));
this.language = code.substring(8, 10).getBytes();
this.doctype = code.charAt(10);
this.localflag = code.charAt(11);
}
public plasmaWordIndexEntry(String external) {
@ -228,11 +223,9 @@ public class plasmaWordIndexEntry {
this.posofphrase = (int) serverCodings.enhancedCoder.decodeBase64Long(pr.getProperty("o", "__"));
this.age = (int) serverCodings.enhancedCoder.decodeBase64Long(pr.getProperty("a", "A"));
this.quality = (int) serverCodings.enhancedCoder.decodeBase64Long(pr.getProperty("q", "__"));
this.language = pr.getProperty("l", "uk");
this.language = pr.getProperty("l", "uk").getBytes();
this.doctype = pr.getProperty("d", "u").charAt(0);
this.localflag = pr.getProperty("f", ""+LT_LOCAL).charAt(0);
// clear code
this.code = null;
}
private String b64save(long x, int l) {
@ -244,120 +237,80 @@ public class plasmaWordIndexEntry {
}
}
public String toEncodedForm(boolean longAttr) {
// attention: this integrates NOT the URL into the encoding
// if you need a complete dump, use toExternalForm()
if (code == null) {
String shortAttr =
b64save(quality, plasmaCrawlLURL.urlQualityLength) +
b64save(age, 3) +
b64save(count, 2) +
language +
doctype +
localflag; // 3 + 3 + 2 + 2 + 1 + 1 = 12 bytes
if (longAttr)
return
shortAttr +
b64save(posintext, 2) +
b64save(posinphrase, 2) +
b64save(posofphrase, 2);
// 12 + 3 + 2 + 2 + 1 + 1 = 12 bytes
else
return shortAttr;
} else {
return code;
}
}
public String toEncodedForm(boolean longAttr) {
// attention: this integrates NOT the URL into the encoding
// if you need a complete dump, use toExternalForm()
String shortAttr =
b64save(quality, plasmaCrawlLURL.urlQualityLength) +
b64save(age, 3) +
b64save(count, 2) +
new String(language) +
doctype +
localflag; // 3 + 3 + 2 + 2 + 1 + 1 = 12 bytes
if (longAttr)
return
shortAttr +
b64save(posintext, 2) +
b64save(posinphrase, 2) +
b64save(posofphrase, 2);
// 12 + 3 + 2 + 2 + 1 + 1 = 12 bytes
else
return shortAttr;
}
public String toExternalForm() {
if (code == null) {
return "{" +
"h=" + urlHash +
",q=" + b64save(quality, plasmaCrawlLURL.urlQualityLength) +
",a=" + b64save(age, 3) +
",c=" + b64save(count, 2) +
",l=" + language +
",d=" + doctype +
",f=" + localflag +
",t=" + b64save(posintext, 2) +
",r=" + b64save(posinphrase, 2) +
",o=" + b64save(posofphrase, 2) +
"}";
} else {
return "{" +
"h=" + urlHash +
",q=" + code.substring(0, 3) +
",a=" + code.substring(3, 6) +
",c=" + code.substring(6, 8) +
",l=" + code.substring(8, 10) +
",d=" + code.charAt(10) +
",f=" + code.charAt(11) +
((code.length() > 12) ? (
",t=" + code.substring(12, 14) +
",r=" + code.substring(14, 16) +
",o=" + code.substring(16, 18)
) : "") +
"}";
}
}
public String toExternalForm() {
return "{" +
"h=" + urlHash +
",q=" + b64save(quality, plasmaCrawlLURL.urlQualityLength) +
",a=" + b64save(age, 3) +
",c=" + b64save(count, 2) +
",l=" + new String(language) +
",d=" + doctype +
",f=" + localflag +
",t=" + b64save(posintext, 2) +
",r=" + b64save(posinphrase, 2) +
",o=" + b64save(posofphrase, 2) +
"}";
}
public String getUrlHash() {
return urlHash;
}
public int getQuality() {
if (code == null) return quality;
else return (int) serverCodings.enhancedCoder.decodeBase64Long(code.substring(0, 3));
return quality;
}
public int getVirtualAge() {
if (code == null) return age;
else return (int) serverCodings.enhancedCoder.decodeBase64Long(code.substring(3, 6));
return age;
}
public int getCount() {
if (code == null) return count;
else return (int) serverCodings.enhancedCoder.decodeBase64Long(code.substring(6, 8));
return count;
}
public int posintext() {
if (code == null) return posintext;
if (code.length() >= 14)
return (int) serverCodings.enhancedCoder.decodeBase64Long(code.substring(12, 14));
else
return 0;
return posintext;
}
public int posinphrase() {
if (code == null) return posinphrase;
if (code.length() >= 15)
return (int) serverCodings.enhancedCoder.decodeBase64Long(code.substring(14, 16));
else
return 0;
return posinphrase;
}
public int posofphrase() {
if (code == null) return posofphrase;
if (code.length() >= 16)
return (int) serverCodings.enhancedCoder.decodeBase64Long(code.substring(16, 18));
else
return 0;
return posofphrase;
}
public String getLanguage() {
if (code == null) return language;
else return code.substring(8, 10);
return new String(language);
}
public char getType() {
if (code == null) return doctype;
else return code.charAt(10);
return doctype;
}
public boolean isLocal() {
if (code == null) return localflag == LT_LOCAL;
else return code.charAt(11) == LT_LOCAL;
return localflag == LT_LOCAL;
}
public static void main(String[] args) {

13
source/de/anomic/plasma/plasmaWordIndexEntryContainer.java
Unescape View File

@ -97,23 +97,20 @@ public class plasmaWordIndexEntryContainer implements Comparable {
this.updateTime = java.lang.Math.max(this.updateTime, c.updateTime);
return x;
}
private boolean add(plasmaWordIndexEntry entry) {
// returns true if the new entry was added, false if it already existet
String urlHash = entry.getUrlHash();
if (container.containsKey(urlHash)) return false;
container.put(urlHash, entry);
return true;
return (container.put(entry.getUrlHash(), entry) == null);
}
public boolean contains(String urlHash) {
return container.containsKey(urlHash);
}
public plasmaWordIndexEntry[] getEntryArray() {
return (plasmaWordIndexEntry[]) container.values().toArray();
}
public Iterator entries() {
// returns an iterator of plasmaWordIndexEntry objects
return container.values().iterator();

Write Preview
Loading…
Cancel
Save