boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

less OOM (works for me)

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4194 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
borg-0300 18 years ago
parent 794d296129
commit a5d28785b1
7 changed files with 2325 additions and 2288 deletions
Show Stats Download Patch File Download Diff File

11
source/de/anomic/index/indexRAMRI.java
Unescape View File

@ -4,9 +4,9 @@
// //
// This is a part of YaCy, a peer-to-peer based web search engine // This is a part of YaCy, a peer-to-peer based web search engine
// //
// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $ // $LastChangedDate$
// $LastChangedRevision: 1986 $ // $LastChangedRevision$
// $LastChangedBy: orbiter $ // $LastChangedBy$
// //
// LICENSE // LICENSE
// //
@ -45,6 +45,7 @@ import de.anomic.kelondro.kelondroNaturalOrder;
import de.anomic.kelondro.kelondroRow; import de.anomic.kelondro.kelondroRow;
import de.anomic.server.serverByteBuffer; import de.anomic.server.serverByteBuffer;
import de.anomic.server.serverFileUtils; import de.anomic.server.serverFileUtils;
import de.anomic.server.serverMemory;
import de.anomic.server.logging.serverLog; import de.anomic.server.logging.serverLog;
import de.anomic.yacy.yacySeedDB; import de.anomic.yacy.yacySeedDB;
@ -160,7 +161,7 @@ public final class indexRAMRI implements indexRI {
// write a log // write a log
if (System.currentTimeMillis() > messageTime) { if (System.currentTimeMillis() > messageTime) {
// System.gc(); // for better statistic serverMemory.gc(1000, "indexRAMRI, for better statistic-1"); // for better statistic - thq
wordsPerSecond = wordcount * 1000 wordsPerSecond = wordcount * 1000
/ (1 + System.currentTimeMillis() - startTime); / (1 + System.currentTimeMillis() - startTime);
log.logInfo("dump status: " + wordcount log.logInfo("dump status: " + wordcount
@ -222,7 +223,7 @@ public final class indexRAMRI implements indexRI {
//while (rt.freeMemory() < 1000000) {flushFromMem(); java.lang.System.gc();} //while (rt.freeMemory() < 1000000) {flushFromMem(); java.lang.System.gc();}
// write a log // write a log
if (System.currentTimeMillis() > messageTime) { if (System.currentTimeMillis() > messageTime) {
System.gc(); // for better statistic serverMemory.gc(1000, "indexRAMRI, for better statistic-2"); // for better statistic - thq
urlsPerSecond = 1 + urlCount * 1000 / (1 + System.currentTimeMillis() - startTime); urlsPerSecond = 1 + urlCount * 1000 / (1 + System.currentTimeMillis() - startTime);
log.logInfo("restoring status: " + urlCount + " urls done, " + ((dumpArray.size() - urlCount) / urlsPerSecond) + " seconds remaining, free mem = " + (Runtime.getRuntime().freeMemory() / 1024 / 1024) + "MB"); log.logInfo("restoring status: " + urlCount + " urls done, " + ((dumpArray.size() - urlCount) / urlsPerSecond) + " seconds remaining, free mem = " + (Runtime.getRuntime().freeMemory() / 1024 / 1024) + "MB");
messageTime = System.currentTimeMillis() + 5000; messageTime = System.currentTimeMillis() + 5000;

1280
source/de/anomic/kelondro/kelondroCachedRecords.java
View File

File diff suppressed because it is too large Load Diff

2122
source/de/anomic/kelondro/kelondroCollectionIndex.java
View File

File diff suppressed because it is too large Load Diff

1124
source/de/anomic/plasma/plasmaRankingCRProcess.java
View File

File diff suppressed because it is too large Load Diff

45
source/de/anomic/plasma/plasmaSwitchboard.java
Unescape View File

@ -121,6 +121,8 @@ import java.util.Locale;
import java.util.Map; import java.util.Map;
import java.util.Properties; import java.util.Properties;
import java.util.Set; import java.util.Set;
import java.util.Timer;
import java.util.TimerTask;
import java.util.TreeMap; import java.util.TreeMap;
import java.util.TreeSet; import java.util.TreeSet;
@ -159,6 +161,7 @@ import de.anomic.server.serverAbstractSwitch;
import de.anomic.server.serverDomains; import de.anomic.server.serverDomains;
import de.anomic.server.serverFileUtils; import de.anomic.server.serverFileUtils;
import de.anomic.server.serverInstantThread; import de.anomic.server.serverInstantThread;
import de.anomic.server.serverMemory;
import de.anomic.server.serverObjects; import de.anomic.server.serverObjects;
import de.anomic.server.serverSemaphore; import de.anomic.server.serverSemaphore;
import de.anomic.server.serverSwitch; import de.anomic.server.serverSwitch;
@ -253,7 +256,8 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
public TreeMap clusterhashes; // map of peerhash(String)/alternative-local-address as ip:port or only ip (String) or null if address in seed should be used public TreeMap clusterhashes; // map of peerhash(String)/alternative-local-address as ip:port or only ip (String) or null if address in seed should be used
public boolean acceptLocalURLs, acceptGlobalURLs; public boolean acceptLocalURLs, acceptGlobalURLs;
public URLLicense licensedURLs; public URLLicense licensedURLs;
public Timer moreMemory;
/* /*
* Remote Proxy configuration * Remote Proxy configuration
*/ */
@ -1153,11 +1157,11 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
wordIndex.setWordFlushSize((int) getConfigLong("wordFlushSize", 10000)); wordIndex.setWordFlushSize((int) getConfigLong("wordFlushSize", 10000));
// set a maximum amount of memory for the caches // set a maximum amount of memory for the caches
long memprereq = Math.max(getConfigLong(INDEXER_MEMPREREQ, 0), wordIndex.minMem()); // long memprereq = Math.max(getConfigLong(INDEXER_MEMPREREQ, 0), wordIndex.minMem());
// setConfig(INDEXER_MEMPREREQ, memprereq); // setConfig(INDEXER_MEMPREREQ, memprereq);
//setThreadPerformance(INDEXER, getConfigLong(INDEXER_IDLESLEEP, 0), getConfigLong(INDEXER_BUSYSLEEP, 0), memprereq); // setThreadPerformance(INDEXER, getConfigLong(INDEXER_IDLESLEEP, 0), getConfigLong(INDEXER_BUSYSLEEP, 0), memprereq);
kelondroCachedRecords.setCacheGrowStati(memprereq + 4 * 1024 * 1024, memprereq + 2 * 1024 * 1024); kelondroCachedRecords.setCacheGrowStati(40 * 1024 * 1024, 20 * 1024 * 1024);
kelondroCache.setCacheGrowStati(memprereq + 4 * 1024 * 1024, memprereq + 2 * 1024 * 1024); kelondroCache.setCacheGrowStati(40 * 1024 * 1024, 20 * 1024 * 1024);
// make parser // make parser
log.logConfig("Starting Parser"); log.logConfig("Starting Parser");
@ -1318,7 +1322,11 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
// deploy threads // deploy threads
log.logConfig("Starting Threads"); log.logConfig("Starting Threads");
// System.gc(); // help for profiler serverMemory.gc(1000, "plasmaSwitchboard, help for profiler"); // help for profiler - thq
moreMemory = new Timer(); // init GC Thread - thq
moreMemory.schedule(new MoreMemory(), 300000, 600000);
int indexing_cluster = Integer.parseInt(getConfig(INDEXER_CLUSTER, "1")); int indexing_cluster = Integer.parseInt(getConfig(INDEXER_CLUSTER, "1"));
if (indexing_cluster < 1) indexing_cluster = 1; if (indexing_cluster < 1) indexing_cluster = 1;
deployThread(CLEANUP, "Cleanup", "simple cleaning process for monitoring information", null, deployThread(CLEANUP, "Cleanup", "simple cleaning process for monitoring information", null,
@ -1743,6 +1751,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
public void close() { public void close() {
log.logConfig("SWITCHBOARD SHUTDOWN STEP 1: sending termination signal to managed threads:"); log.logConfig("SWITCHBOARD SHUTDOWN STEP 1: sending termination signal to managed threads:");
moreMemory.cancel();
terminateAllThreads(true); terminateAllThreads(true);
if (transferIdxThread != null) stopTransferWholeIndex(false); if (transferIdxThread != null) stopTransferWholeIndex(false);
log.logConfig("SWITCHBOARD SHUTDOWN STEP 2: sending termination signal to threaded indexing"); log.logConfig("SWITCHBOARD SHUTDOWN STEP 2: sending termination signal to threaded indexing");
@ -1792,9 +1801,9 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
// flush some entries from the RAM cache // flush some entries from the RAM cache
wordIndex.flushCacheSome(); wordIndex.flushCacheSome();
// adopt maximum cache size to current size to prevent that further OutOfMemoryErrors occur // adopt maximum cache size to current size to prevent that further OutOfMemoryErrors occur
int newMaxCount = Math.max(1200, Math.min((int) getConfigLong(WORDCACHE_MAX_COUNT, 1200), wordIndex.dhtOutCacheSize())); /* int newMaxCount = Math.max(1200, Math.min((int) getConfigLong(WORDCACHE_MAX_COUNT, 1200), wordIndex.dhtOutCacheSize()));
setConfig(WORDCACHE_MAX_COUNT, Integer.toString(newMaxCount)); setConfig(WORDCACHE_MAX_COUNT, Integer.toString(newMaxCount));
wordIndex.setMaxWordCount(newMaxCount); wordIndex.setMaxWordCount(newMaxCount); */
} }
public boolean deQueue() { public boolean deQueue() {
@ -2040,14 +2049,14 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
yacyCore.newsPool.publishMyNews(yacyNewsRecord.newRecord(yacyNewsPool.CATEGORY_PROFILE_BROADCAST, news)); yacyCore.newsPool.publishMyNews(yacyNewsRecord.newRecord(yacyNewsPool.CATEGORY_PROFILE_BROADCAST, news));
} }
} }
/*
// set a maximum amount of memory for the caches // set a maximum amount of memory for the caches
long memprereq = Math.max(getConfigLong(INDEXER_MEMPREREQ, 0), wordIndex.minMem()); // long memprereq = Math.max(getConfigLong(INDEXER_MEMPREREQ, 0), wordIndex.minMem());
// setConfig(INDEXER_MEMPREREQ, memprereq); // setConfig(INDEXER_MEMPREREQ, memprereq);
//setThreadPerformance(INDEXER, getConfigLong(INDEXER_IDLESLEEP, 0), getConfigLong(INDEXER_BUSYSLEEP, 0), memprereq); // setThreadPerformance(INDEXER, getConfigLong(INDEXER_IDLESLEEP, 0), getConfigLong(INDEXER_BUSYSLEEP, 0), memprereq);
kelondroCachedRecords.setCacheGrowStati(memprereq + 4 * 1024 * 1024, memprereq + 2 * 1024 * 1024); kelondroCachedRecords.setCacheGrowStati(40 * 1024 * 1024, 20 * 1024 * 1024);
kelondroCache.setCacheGrowStati(memprereq + 4 * 1024 * 1024, memprereq + 2 * 1024 * 1024); kelondroCache.setCacheGrowStati(40 * 1024 * 1024, 20 * 1024 * 1024);
*/
// update the cluster set // update the cluster set
this.clusterhashes = yacyCore.seedDB.clusterHashes(getConfig("cluster.peers.yacydomain", "")); this.clusterhashes = yacyCore.seedDB.clusterHashes(getConfig("cluster.peers.yacydomain", ""));
@ -2887,7 +2896,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
if ((curThread instanceof serverThread) && ((serverThread)curThread).shutdownInProgress()) throw new InterruptedException("Shutdown in progress ..."); if ((curThread instanceof serverThread) && ((serverThread)curThread).shutdownInProgress()) throw new InterruptedException("Shutdown in progress ...");
else if (this.terminate || curThread.isInterrupted()) throw new InterruptedException("Shutdown in progress ..."); else if (this.terminate || curThread.isInterrupted()) throw new InterruptedException("Shutdown in progress ...");
} }
public void terminate(long delay) { public void terminate(long delay) {
if (delay <= 0) throw new IllegalArgumentException("The shutdown delay must be greater than 0."); if (delay <= 0) throw new IllegalArgumentException("The shutdown delay must be greater than 0.");
(new delayedShutdown(this,delay)).start(); (new delayedShutdown(this,delay)).start();
@ -2908,6 +2917,12 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
} }
} }
class MoreMemory extends TimerTask {
public final void run() {
serverMemory.gc(10000, "MoreMemory()");
}
}
class delayedShutdown extends Thread { class delayedShutdown extends Thread {
private plasmaSwitchboard sb; private plasmaSwitchboard sb;
private long delay; private long delay;

4
source/de/anomic/server/serverCore.java
Unescape View File

@ -6,7 +6,7 @@
// //
// $LastChangedDate$ // $LastChangedDate$
// $LastChangedRevision$ // $LastChangedRevision$
// $LastChangedBy: $ // $LastChangedBy$
// //
// ThreadPool // ThreadPool
// //
@ -414,7 +414,7 @@ public final class serverCore extends serverAbstractThread implements serverThre
public void freemem() { public void freemem() {
// FIXME: can we something here to flush memory? Idea: Reduce the size of some of our various caches. // FIXME: can we something here to flush memory? Idea: Reduce the size of some of our various caches.
System.gc(); serverMemory.gc(2000, "serverCore.freemem()"); // thq
} }
// class body // class body

27
source/de/anomic/server/serverMemory.java
Unescape View File

@ -3,9 +3,9 @@
// (C) 2005 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany // (C) 2005 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
// first published 22.09.2005 on http://yacy.net // first published 22.09.2005 on http://yacy.net
// //
// $LastChangedDate: 2005-09-21 16:21:45 +0200 (Wed, 21 Sep 2005) $ // $LastChangedDate$
// $LastChangedRevision: 763 $ // $LastChangedRevision$
// $LastChangedBy: orbiter $ // $LastChangedBy$
// //
// LICENSE // LICENSE
// //
@ -23,7 +23,6 @@
// along with this program; if not, write to the Free Software // along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package de.anomic.server; package de.anomic.server;
import de.anomic.server.logging.serverLog; import de.anomic.server.logging.serverLog;
@ -38,11 +37,25 @@ public class serverMemory {
private static final long[] gcs = new long[5]; private static final long[] gcs = new long[5];
private static int gcs_pos = 0; private static int gcs_pos = 0;
private static long lastGC;
public final synchronized static void gc(int last, String info) { // thq
long elapsed = System.currentTimeMillis() - lastGC;
if (elapsed > last) {
long free = free();
System.gc();
lastGC = System.currentTimeMillis();
log.logInfo("[gc] before: " + bytesToString(free) + ", after: " + bytesToString(free()) + ", call: " + info);
} else if (log.isFine()) {
log.logFine("[gc] no execute, last run: " + (elapsed / 1000) + " seconds ago, call: " + info);
}
}
/** @return the amount of freed bytes by a forced GC this method performes */ /** @return the amount of freed bytes by a forced GC this method performes */
private static long runGC(final boolean count) { private static long runGC(final boolean count) {
final long memnow = available(); final long memnow = available();
System.gc(); gc(1000, "serverMemory.runGC(...)");
final long freed = available() - memnow; final long freed = available() - memnow;
if (count) { if (count) {
gcs[gcs_pos] = freed; gcs[gcs_pos] = freed;
@ -89,7 +102,7 @@ public class serverMemory {
public static boolean available(long memory, boolean gciffail) { public static boolean available(long memory, boolean gciffail) {
if (available() >= memory) return true; if (available() >= memory) return true;
if (!gciffail) return false; if (!gciffail) return false;
System.gc(); gc(4000, "serverMemory.available(...)");
return (available() >= memory); return (available() >= memory);
} }

Write Preview
Loading…
Cancel
Save