boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

less OOM (works for me)

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4194 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
borg-0300 18 years ago
parent 794d296129
commit a5d28785b1
7 changed files with 2325 additions and 2288 deletions
Show Stats Download Patch File Download Diff File

11
source/de/anomic/index/indexRAMRI.java
Unescape View File

@ -4,9 +4,9 @@
//
// This is a part of YaCy, a peer-to-peer based web search engine
//
// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
// $LastChangedRevision: 1986 $
// $LastChangedBy: orbiter $
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
// LICENSE
//
@ -45,6 +45,7 @@ import de.anomic.kelondro.kelondroNaturalOrder;
import de.anomic.kelondro.kelondroRow;
import de.anomic.server.serverByteBuffer;
import de.anomic.server.serverFileUtils;
import de.anomic.server.serverMemory;
import de.anomic.server.logging.serverLog;
import de.anomic.yacy.yacySeedDB;
@ -160,7 +161,7 @@ public final class indexRAMRI implements indexRI {
// write a log
if (System.currentTimeMillis() > messageTime) {
// System.gc(); // for better statistic
serverMemory.gc(1000, "indexRAMRI, for better statistic-1"); // for better statistic - thq
wordsPerSecond = wordcount * 1000
/ (1 + System.currentTimeMillis() - startTime);
log.logInfo("dump status: " + wordcount
@ -222,7 +223,7 @@ public final class indexRAMRI implements indexRI {
//while (rt.freeMemory() < 1000000) {flushFromMem(); java.lang.System.gc();}
// write a log
if (System.currentTimeMillis() > messageTime) {
System.gc(); // for better statistic
serverMemory.gc(1000, "indexRAMRI, for better statistic-2"); // for better statistic - thq
urlsPerSecond = 1 + urlCount * 1000 / (1 + System.currentTimeMillis() - startTime);
log.logInfo("restoring status: " + urlCount + " urls done, " + ((dumpArray.size() - urlCount) / urlsPerSecond) + " seconds remaining, free mem = " + (Runtime.getRuntime().freeMemory() / 1024 / 1024) + "MB");
messageTime = System.currentTimeMillis() + 5000;

12
source/de/anomic/kelondro/kelondroCachedRecords.java
Unescape View File

@ -4,9 +4,9 @@
//
// This is a part of YaCy, a peer-to-peer based web search engine
//
// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
// $LastChangedRevision: 1986 $
// $LastChangedBy: orbiter $
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
// LICENSE
//
@ -122,7 +122,11 @@ public class kelondroCachedRecords extends kelondroAbstractRecords implements ke
// 1: cache is allowed to grow, but need not to shrink
// 2: cache is allowed to grow and must not shrink
if (available > stopGrow) return 2;
if (available > startShrink) return 1;
if (available > startShrink) {
serverMemory.gc(30000, "kelendroCacheRecords.cacheGrowStatus(...) 1"); // thq
return 1;
}
serverMemory.gc(3000, "kelendroCacheRecords.cacheGrowStatus(...) 0"); // thq
return 0;
}

14
source/de/anomic/kelondro/kelondroCollectionIndex.java
Unescape View File

@ -24,6 +24,10 @@ package de.anomic.kelondro;
// the collection arrays may be migration to another size during run-time, which means that not only the
// partitions as mentioned above are maintained, but also a set of "shadow-partitions", that represent old
// partitions and where data is read only and slowly migrated to the default partitions.
//
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
import java.io.File;
import java.io.IOException;
@ -558,7 +562,7 @@ public class kelondroCollectionIndex {
TreeMap array_add_map = new TreeMap();
ArrayList actionList;
TreeMap actionMap;
boolean madegc = false;
//boolean madegc = false;
//System.out.println("DEBUG existingContainer: " + existingContainer.toString());
while (existingContainer.size() > 0) {
oldPartitionNumber1 = ((Integer) existingContainer.lastKey()).intValue();
@ -637,12 +641,12 @@ public class kelondroCollectionIndex {
array_replace_map = new TreeMap(); // delete references
indexrows_existing.addAll(array_add_multiple(array_add_map, 0, this.payloadrow.objectsize()));
array_add_map = new TreeMap(); // delete references
if (!madegc) {
//if (!madegc) {
// prevent that this flush is made again even when there is enough memory
System.gc();
serverMemory.gc(10000, "kelendroCollectionIndex.mergeMultiple(...)"); // thq
// prevent that this gc happens more than one time
madegc = true;
}
// madegc = true;
//}
}
}
}

8
source/de/anomic/plasma/plasmaRankingCRProcess.java
Unescape View File

@ -6,9 +6,9 @@
// Frankfurt, Germany, 2005
// Created 15.11.2005
//
// $LastChangedDate: 2005-10-22 15:28:04 +0200 (Sat, 22 Oct 2005) $
// $LastChangedRevision: 968 $
// $LastChangedBy: theli $
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
@ -386,7 +386,7 @@ public class plasmaRankingCRProcess {
// finished. write to file
cr = null;
cr_in = null;
System.gc();
serverMemory.gc(1000, "plasmaRankingCRProcess.genrci(...)"); // thq
rci.toFile(rci_out);
return count;
}

37
source/de/anomic/plasma/plasmaSwitchboard.java
Unescape View File

@ -121,6 +121,8 @@ import java.util.Locale;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.Timer;
import java.util.TimerTask;
import java.util.TreeMap;
import java.util.TreeSet;
@ -159,6 +161,7 @@ import de.anomic.server.serverAbstractSwitch;
import de.anomic.server.serverDomains;
import de.anomic.server.serverFileUtils;
import de.anomic.server.serverInstantThread;
import de.anomic.server.serverMemory;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSemaphore;
import de.anomic.server.serverSwitch;
@ -253,6 +256,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
public TreeMap clusterhashes; // map of peerhash(String)/alternative-local-address as ip:port or only ip (String) or null if address in seed should be used
public boolean acceptLocalURLs, acceptGlobalURLs;
public URLLicense licensedURLs;
public Timer moreMemory;
/*
* Remote Proxy configuration
@ -1153,11 +1157,11 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
wordIndex.setWordFlushSize((int) getConfigLong("wordFlushSize", 10000));
// set a maximum amount of memory for the caches
long memprereq = Math.max(getConfigLong(INDEXER_MEMPREREQ, 0), wordIndex.minMem());
// long memprereq = Math.max(getConfigLong(INDEXER_MEMPREREQ, 0), wordIndex.minMem());
// setConfig(INDEXER_MEMPREREQ, memprereq);
// setThreadPerformance(INDEXER, getConfigLong(INDEXER_IDLESLEEP, 0), getConfigLong(INDEXER_BUSYSLEEP, 0), memprereq);
kelondroCachedRecords.setCacheGrowStati(memprereq + 4 * 1024 * 1024, memprereq + 2 * 1024 * 1024);
kelondroCache.setCacheGrowStati(memprereq + 4 * 1024 * 1024, memprereq + 2 * 1024 * 1024);
kelondroCachedRecords.setCacheGrowStati(40 * 1024 * 1024, 20 * 1024 * 1024);
kelondroCache.setCacheGrowStati(40 * 1024 * 1024, 20 * 1024 * 1024);
// make parser
log.logConfig("Starting Parser");
@ -1318,7 +1322,11 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
// deploy threads
log.logConfig("Starting Threads");
// System.gc(); // help for profiler
serverMemory.gc(1000, "plasmaSwitchboard, help for profiler"); // help for profiler - thq
moreMemory = new Timer(); // init GC Thread - thq
moreMemory.schedule(new MoreMemory(), 300000, 600000);
int indexing_cluster = Integer.parseInt(getConfig(INDEXER_CLUSTER, "1"));
if (indexing_cluster < 1) indexing_cluster = 1;
deployThread(CLEANUP, "Cleanup", "simple cleaning process for monitoring information", null,
@ -1743,6 +1751,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
public void close() {
log.logConfig("SWITCHBOARD SHUTDOWN STEP 1: sending termination signal to managed threads:");
moreMemory.cancel();
terminateAllThreads(true);
if (transferIdxThread != null) stopTransferWholeIndex(false);
log.logConfig("SWITCHBOARD SHUTDOWN STEP 2: sending termination signal to threaded indexing");
@ -1792,9 +1801,9 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
// flush some entries from the RAM cache
wordIndex.flushCacheSome();
// adopt maximum cache size to current size to prevent that further OutOfMemoryErrors occur
int newMaxCount = Math.max(1200, Math.min((int) getConfigLong(WORDCACHE_MAX_COUNT, 1200), wordIndex.dhtOutCacheSize()));
/* int newMaxCount = Math.max(1200, Math.min((int) getConfigLong(WORDCACHE_MAX_COUNT, 1200), wordIndex.dhtOutCacheSize()));
setConfig(WORDCACHE_MAX_COUNT, Integer.toString(newMaxCount));
wordIndex.setMaxWordCount(newMaxCount);
wordIndex.setMaxWordCount(newMaxCount); */
}
public boolean deQueue() {
@ -2040,14 +2049,14 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
yacyCore.newsPool.publishMyNews(yacyNewsRecord.newRecord(yacyNewsPool.CATEGORY_PROFILE_BROADCAST, news));
}
}
/*
// set a maximum amount of memory for the caches
long memprereq = Math.max(getConfigLong(INDEXER_MEMPREREQ, 0), wordIndex.minMem());
// long memprereq = Math.max(getConfigLong(INDEXER_MEMPREREQ, 0), wordIndex.minMem());
// setConfig(INDEXER_MEMPREREQ, memprereq);
// setThreadPerformance(INDEXER, getConfigLong(INDEXER_IDLESLEEP, 0), getConfigLong(INDEXER_BUSYSLEEP, 0), memprereq);
kelondroCachedRecords.setCacheGrowStati(memprereq + 4 * 1024 * 1024, memprereq + 2 * 1024 * 1024);
kelondroCache.setCacheGrowStati(memprereq + 4 * 1024 * 1024, memprereq + 2 * 1024 * 1024);
kelondroCachedRecords.setCacheGrowStati(40 * 1024 * 1024, 20 * 1024 * 1024);
kelondroCache.setCacheGrowStati(40 * 1024 * 1024, 20 * 1024 * 1024);
*/
// update the cluster set
this.clusterhashes = yacyCore.seedDB.clusterHashes(getConfig("cluster.peers.yacydomain", ""));
@ -2908,6 +2917,12 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
}
}
class MoreMemory extends TimerTask {
public final void run() {
serverMemory.gc(10000, "MoreMemory()");
}
}
class delayedShutdown extends Thread {
private plasmaSwitchboard sb;
private long delay;

4
source/de/anomic/server/serverCore.java
Unescape View File

@ -6,7 +6,7 @@
//
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy: $
// $LastChangedBy$
//
// ThreadPool
//
@ -414,7 +414,7 @@ public final class serverCore extends serverAbstractThread implements serverThre
public void freemem() {
// FIXME: can we something here to flush memory? Idea: Reduce the size of some of our various caches.
System.gc();
serverMemory.gc(2000, "serverCore.freemem()"); // thq
}
// class body

25
source/de/anomic/server/serverMemory.java
Unescape View File

@ -3,9 +3,9 @@
// (C) 2005 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
// first published 22.09.2005 on http://yacy.net
//
// $LastChangedDate: 2005-09-21 16:21:45 +0200 (Wed, 21 Sep 2005) $
// $LastChangedRevision: 763 $
// $LastChangedBy: orbiter $
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
// LICENSE
//
@ -23,7 +23,6 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package de.anomic.server;
import de.anomic.server.logging.serverLog;
@ -39,10 +38,24 @@ public class serverMemory {
private static final long[] gcs = new long[5];
private static int gcs_pos = 0;
private static long lastGC;
public final synchronized static void gc(int last, String info) { // thq
long elapsed = System.currentTimeMillis() - lastGC;
if (elapsed > last) {
long free = free();
System.gc();
lastGC = System.currentTimeMillis();
log.logInfo("[gc] before: " + bytesToString(free) + ", after: " + bytesToString(free()) + ", call: " + info);
} else if (log.isFine()) {
log.logFine("[gc] no execute, last run: " + (elapsed / 1000) + " seconds ago, call: " + info);
}
}
/** @return the amount of freed bytes by a forced GC this method performes */
private static long runGC(final boolean count) {
final long memnow = available();
System.gc();
gc(1000, "serverMemory.runGC(...)");
final long freed = available() - memnow;
if (count) {
gcs[gcs_pos] = freed;
@ -89,7 +102,7 @@ public class serverMemory {
public static boolean available(long memory, boolean gciffail) {
if (available() >= memory) return true;
if (!gciffail) return false;
System.gc();
gc(4000, "serverMemory.available(...)");
return (available() >= memory);
}

Write Preview
Loading…
Cancel
Save