boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

more memory protection: auto-flush of caches in case of memory shortage

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7575 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 14 years ago
parent 7962d35425
commit 3820525464
9 changed files with 50 additions and 28 deletions
Show Stats Download Patch File Download Diff File

7
source/de/anomic/crawler/Balancer.java
Unescape View File

@ -48,6 +48,7 @@ import net.yacy.kelondro.order.Base64Order;
import net.yacy.kelondro.order.CloneableIterator;
import net.yacy.kelondro.table.Table;
import net.yacy.kelondro.util.ByteBuffer;
import net.yacy.kelondro.util.MemoryControl;
import de.anomic.crawler.retrieval.Request;
import de.anomic.http.client.Cache;
@ -268,7 +269,7 @@ public class Balancer {
if (this.ddc.has(hash)) return "double occurrence in ddc";
if (this.urlFileIndex.has(hash)) return "double occurrence in urlFileIndex";
if (this.double_push_check.size() > 10000) this.double_push_check.clear();
if (this.double_push_check.size() > 10000 || MemoryControl.shortStatus()) this.double_push_check.clear();
this.double_push_check.put(hash);
// add to index
@ -437,8 +438,10 @@ public class Balancer {
}
break;
}
if (crawlEntry != null)
if (crawlEntry != null) {
if (this.ddc.size() > 10000 || MemoryControl.shortStatus()) this.ddc.clear();
try { this.ddc.put(crawlEntry.url().hash()); } catch (RowSpaceExceededException e) {}
}
}
if (crawlEntry == null) return null;

4
source/de/anomic/crawler/Latency.java
Unescape View File

@ -29,6 +29,7 @@ import java.util.concurrent.ConcurrentHashMap;
import net.yacy.cora.document.MultiProtocolURI;
import net.yacy.cora.protocol.Domains;
import net.yacy.kelondro.util.MemoryControl;
import de.anomic.search.Switchboard;
@ -43,6 +44,7 @@ public class Latency {
Host h = map.get(host);
if (h == null) {
h = new Host(host, time);
if (map.size() > 1000 || MemoryControl.shortStatus()) map.clear();
map.put(host, h);
} else {
h.update(time);
@ -55,6 +57,7 @@ public class Latency {
Host h = map.get(host);
if (h == null) {
h = new Host(host, 3000);
if (map.size() > 1000 || MemoryControl.shortStatus()) map.clear();
map.put(host, h);
} else {
h.update();
@ -67,6 +70,7 @@ public class Latency {
Host h = map.get(host);
if (h == null) {
h = new Host(host, 3000);
if (map.size() > 1000 || MemoryControl.shortStatus()) map.clear();
map.put(host, h);
} else {
h.slowdown();

13
source/de/anomic/crawler/ResultImages.java
Unescape View File

@ -35,6 +35,7 @@ import net.yacy.cora.document.MultiProtocolURI;
import net.yacy.document.Document;
import net.yacy.document.parser.html.ImageEntry;
import net.yacy.kelondro.data.meta.DigestURI;
import net.yacy.kelondro.util.MemoryControl;
public class ResultImages {
@ -55,6 +56,9 @@ public class ResultImages {
public static void registerImages(final DigestURI source, final Document document, final boolean privateEntry) {
if (document == null) return;
if (source == null) return;
if (MemoryControl.shortStatus()) clearQueues();
limitQueues(1000);
final Map<MultiProtocolURI, ImageEntry> images = document.getImages();
for (final ImageEntry image: images.values()) {
@ -134,7 +138,7 @@ public class ResultImages {
public static int publicQueueLowSize() {
return publicImageQueueLow.size();
}
public static void clearQueues() {
privateImageQueueHigh.clear();
privateImageQueueLow.clear();
@ -143,6 +147,13 @@ public class ResultImages {
doubleCheck.clear();
}
public static void limitQueues(int limit) {
while (privateImageQueueHigh.size() > limit) privateImageQueueHigh.poll();
while (privateImageQueueLow.size() > limit) privateImageQueueLow.poll();
while (publicImageQueueHigh.size() > limit) publicImageQueueHigh.poll();
while (publicImageQueueLow.size() > limit) publicImageQueueLow.poll();
}
public static class OriginEntry {
public ImageEntry imageEntry;
public MultiProtocolURI baseURL;

5
source/de/anomic/crawler/ResultURLs.java
Unescape View File

@ -76,9 +76,8 @@ public final class ResultURLs {
}
}
private final static int initialStackCapacity = 500;
private final static Map<EventOrigin, Map<String, InitExecEntry>> resultStacks = new ConcurrentHashMap<EventOrigin, Map<String, InitExecEntry>>(initialStackCapacity); // a mapping from urlHash to Entries
private final static Map<EventOrigin, DynamicScore<String>> resultDomains = new ConcurrentHashMap<EventOrigin, DynamicScore<String>>(initialStackCapacity);
private final static Map<EventOrigin, Map<String, InitExecEntry>> resultStacks = new ConcurrentHashMap<EventOrigin, Map<String, InitExecEntry>>(); // a mapping from urlHash to Entries
private final static Map<EventOrigin, DynamicScore<String>> resultDomains = new ConcurrentHashMap<EventOrigin, DynamicScore<String>>();
static {
for (EventOrigin origin: EventOrigin.values()) {

2
source/de/anomic/http/server/HTTPDFileHandler.java
Unescape View File

@ -894,6 +894,7 @@ public final class HTTPDFileHandler {
// storing the content into the cache
ref = new SoftReference<TemplateCacheEntry>(templateCacheEntry);
if (MemoryControl.shortStatus()) templateCache.clear();
templateCache.put(targetFile, ref);
if (theLogger.isFinest()) theLogger.logFinest("Cache MISS for file " + targetFile);
} else {
@ -1207,6 +1208,7 @@ public final class HTTPDFileHandler {
if (useTemplateCache) {
// storing the method into the cache
final SoftReference<Method> ref = new SoftReference<Method>(m);
if (MemoryControl.shortStatus()) templateMethodCache.clear();
templateMethodCache.put(classFile, ref);
}

2
source/net/yacy/document/WordCache.java
Unescape View File

@ -38,6 +38,7 @@ import java.util.zip.GZIPInputStream;
import net.yacy.cora.storage.IntScore;
import net.yacy.cora.storage.ScoreMap;
import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.util.MemoryControl;
/**
* provide a completion library for the did-you-mean class
@ -74,6 +75,7 @@ public class WordCache {
public void learn(String word) {
if (word == null) return;
if (word.length() < commonWordsMinLength) return;
if (MemoryControl.shortStatus()) commonWords.clear();
commonWords.inc(word);
if (commonWords.size() > commonWordsMaxSize) {
commonWords.shrinkToMaxSize(commonWordsMaxSize / 2);

4
source/net/yacy/kelondro/index/Cache.java
Unescape View File

@ -180,7 +180,7 @@ public final class Cache implements Index, Iterable<Row.Entry> {
// check memory
long available = MemoryControl.available();
if (available - 2 * 1024 * 1024 < readMissCache.memoryNeededForGrow()) {
if (MemoryControl.shortStatus() || available - 2 * 1024 * 1024 < readMissCache.memoryNeededForGrow()) {
readMissCache.clear();
}
available = MemoryControl.available();
@ -200,7 +200,7 @@ public final class Cache implements Index, Iterable<Row.Entry> {
// check memory
long available = MemoryControl.available();
if (available - 2 * 1024 * 1024 < readHitCache.memoryNeededForGrow()) {
if (MemoryControl.shortStatus() || available - 2 * 1024 * 1024 < readHitCache.memoryNeededForGrow()) {
readHitCache.clear();
}
available = MemoryControl.available();

39
source/net/yacy/kelondro/rwi/IndexCell.java
Unescape View File

@ -73,7 +73,7 @@ public final class IndexCell<ReferenceType extends Reference> extends AbstractBu
private final int writeBufferSize;
private Semaphore dumperSemaphore = new Semaphore(1);
private Semaphore cleanerSemaphore = new Semaphore(1);
private final Map<byte[], HandleSet> failedURLs; // mapping from word hashes to a list of url hashes
private final Map<byte[], HandleSet> removeDelayedURLs; // mapping from word hashes to a list of url hashes
public IndexCell(
@ -100,7 +100,7 @@ public final class IndexCell<ReferenceType extends Reference> extends AbstractBu
this.targetFileSize = targetFileSize;
this.maxFileSize = maxFileSize;
this.writeBufferSize = writeBufferSize;
this.failedURLs = new TreeMap<byte[], HandleSet>(URIMetadataRow.rowdef.objectOrder);
this.removeDelayedURLs = new TreeMap<byte[], HandleSet>(URIMetadataRow.rowdef.objectOrder);
}
@ -174,12 +174,13 @@ public final class IndexCell<ReferenceType extends Reference> extends AbstractBu
assert countRam == null || countRam.size() >= 0;
int c = countRam == null ? countFile : countFile + countRam.size();
// exclude entries from delayed remove
synchronized (this.failedURLs) {
HandleSet s = this.failedURLs.get(termHash);
synchronized (this.removeDelayedURLs) {
HandleSet s = this.removeDelayedURLs.get(termHash);
if (s != null) c -= s.size();
if (c < 0) c = 0;
}
// put count result into cache
if (MemoryControl.shortStatus()) this.countCache.clear();
this.countCache.put(termHash, c);
return c;
}
@ -219,8 +220,8 @@ public final class IndexCell<ReferenceType extends Reference> extends AbstractBu
}
if (result == null) return null;
// remove the failed urls
synchronized (this.failedURLs) {
HandleSet s = this.failedURLs.get(termHash);
synchronized (this.removeDelayedURLs) {
HandleSet s = this.removeDelayedURLs.get(termHash);
if (s != null) result.removeEntries(s);
}
return result;
@ -261,8 +262,8 @@ public final class IndexCell<ReferenceType extends Reference> extends AbstractBu
public void removeDelayed(byte[] termHash, HandleSet urlHashes) {
HandleSet r;
synchronized (failedURLs) {
r = this.failedURLs.get(termHash);
synchronized (removeDelayedURLs) {
r = this.removeDelayedURLs.get(termHash);
}
if (r == null) {
r = new HandleSet(URIMetadataRow.rowdef.primaryKeyLength, URIMetadataRow.rowdef.objectOrder, 0);
@ -273,15 +274,15 @@ public final class IndexCell<ReferenceType extends Reference> extends AbstractBu
try {remove(termHash, urlHashes);} catch (IOException e1) {}
return;
}
synchronized (failedURLs) {
this.failedURLs.put(termHash, r);
synchronized (removeDelayedURLs) {
this.removeDelayedURLs.put(termHash, r);
}
}
public void removeDelayed(byte[] termHash, byte[] urlHashBytes) {
HandleSet r;
synchronized (failedURLs) {
r = this.failedURLs.get(termHash);
synchronized (removeDelayedURLs) {
r = this.removeDelayedURLs.get(termHash);
}
if (r == null) {
r = new HandleSet(URIMetadataRow.rowdef.primaryKeyLength, URIMetadataRow.rowdef.objectOrder, 0);
@ -292,20 +293,20 @@ public final class IndexCell<ReferenceType extends Reference> extends AbstractBu
try {remove(termHash, urlHashBytes);} catch (IOException e1) {}
return;
}
synchronized (failedURLs) {
this.failedURLs.put(termHash, r);
synchronized (removeDelayedURLs) {
this.removeDelayedURLs.put(termHash, r);
}
}
public void removeDelayed() throws IOException {
HandleSet words = new HandleSet(URIMetadataRow.rowdef.primaryKeyLength, URIMetadataRow.rowdef.objectOrder, 0); // a set of url hashes where a worker thread tried to work on, but failed.
synchronized (failedURLs) {
for (byte[] b: failedURLs.keySet()) try {words.put(b);} catch (RowSpaceExceededException e) {}
synchronized (removeDelayedURLs) {
for (byte[] b: removeDelayedURLs.keySet()) try {words.put(b);} catch (RowSpaceExceededException e) {}
}
synchronized (failedURLs) {
synchronized (removeDelayedURLs) {
for (byte[] b: words) {
HandleSet urls = failedURLs.remove(b);
HandleSet urls = removeDelayedURLs.remove(b);
if (urls != null) remove(b, urls);
}
}
@ -410,7 +411,7 @@ public final class IndexCell<ReferenceType extends Reference> extends AbstractBu
*/
public synchronized void clear() throws IOException {
this.countCache.clear();
this.failedURLs.clear();
this.removeDelayedURLs.clear();
this.ram.clear();
this.array.clear();
}

2
source/net/yacy/kelondro/table/Table.java
Unescape View File

@ -253,7 +253,7 @@ public class Table implements Index, Iterable<Row.Entry> {
private boolean abandonTable() {
// check if not enough memory is there to maintain a memory copy of the table
return MemoryControl.available() < minmemremaining;
return MemoryControl.shortStatus() || MemoryControl.available() < minmemremaining;
}
public byte[] smallestKey() {

Write Preview
Loading…
Cancel
Save