boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

more patches for a better out-of-memory management

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7555 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 14 years ago
parent c2c5b12882
commit 8d14916c74
15 changed files with 96 additions and 23 deletions
Show Stats Download Patch File Download Diff File

7
source/de/anomic/http/server/HTTPDFileHandler.java
Unescape View File

@ -1220,7 +1220,12 @@ public final class HTTPDFileHandler {
}
private static final Object invokeServlet(final File targetClass, final RequestHeader request, final serverObjects args) throws IllegalArgumentException, IllegalAccessException, InvocationTargetException {
return rewriteMethod(targetClass).invoke(null, new Object[] {request, args, switchboard});
try {
return rewriteMethod(targetClass).invoke(null, new Object[] {request, args, switchboard});
} catch (OutOfMemoryError e) {
Log.logException(e);
return null;
}
}
/**

2
source/de/anomic/search/Switchboard.java
Unescape View File

@ -1173,7 +1173,7 @@ public final class Switchboard extends serverSwitch {
blogDB.close();
blogCommentDB.close();
userDB.close();
bookmarksDB.close();
if (bookmarksDB != null) bookmarksDB.close(); // may null if concurrent initialization was not finished
messageDB.close();
webStructure.close();
crawlQueues.close();

7
source/de/anomic/yacy/graphics/WebStructureGraph.java
Unescape View File

@ -66,7 +66,12 @@ public class WebStructureGraph {
this.structureFile = structureFile;
// load web structure
final Map<String, String> loadedStructure = (this.structureFile.exists()) ? FileUtils.loadMap(this.structureFile) : new TreeMap<String, String>();
Map<String, String> loadedStructure;
try {
loadedStructure = (this.structureFile.exists()) ? FileUtils.loadMap(this.structureFile) : new TreeMap<String, String>();
} catch (OutOfMemoryError e) {
loadedStructure = new TreeMap<String, String>();
}
if (loadedStructure != null) this.structure_old.putAll(loadedStructure);
// delete out-dated entries in case the structure is too big

7
source/de/anomic/yacy/yacySeedDB.java
Unescape View File

@ -1017,7 +1017,12 @@ public final class yacySeedDB implements AlternativeDomainNames {
Map<String, String> dna0;
ConcurrentHashMap<String, String> dna;
while (it.hasNext()) {
dna0 = it.next();
try {
dna0 = it.next();
} catch (OutOfMemoryError e) {
Log.logException(e);
dna0 = null;
}
assert dna0 != null;
if (dna0 == null) continue;
if (dna0 instanceof ConcurrentHashMap) {

6
source/net/yacy/cora/protocol/Domains.java
Unescape View File

@ -39,6 +39,7 @@ import java.util.regex.Pattern;
import net.yacy.cora.storage.ARC;
import net.yacy.cora.storage.ConcurrentARC;
import net.yacy.kelondro.util.MemoryControl;
public class Domains {
@ -484,6 +485,11 @@ public class Domains {
InetAddress ip = parseInetAddress(host);
if (ip != null) return ip;
if (MemoryControl.shortStatus()) {
NAME_CACHE_HIT.clear();
NAME_CACHE_MISS.clear();
}
// try to resolve host by doing a name cache lookup
ip = NAME_CACHE_HIT.get(host);
if (ip != null) return ip;

1
source/net/yacy/document/parser/sitemapParser.java
Unescape View File

@ -157,6 +157,7 @@ public class sitemapParser extends AbstractParser implements Parser {
catch (ParserConfigurationException e) { throw new IOException (e); }
catch (SAXParseException e) { throw new IOException (e); }
catch (SAXException e) { throw new IOException (e); }
catch (OutOfMemoryError e) { throw new IOException (e); }
NodeList sitemapNodes = doc.getElementsByTagName("sitemap");
for (int i = 0; i < sitemapNodes.getLength(); i++) {
String url = new SitemapEntry((Element) sitemapNodes.item(i)).url();

7
source/net/yacy/kelondro/blob/Gap.java
Unescape View File

@ -61,7 +61,12 @@ public class Gap extends TreeMap<Long, Integer> {
public Gap(final File file) throws IOException {
super();
// read the index dump and fill the index
DataInputStream is = new DataInputStream(new BufferedInputStream(new FileInputStream(file), 1024 * 1024));
DataInputStream is;
try {
is = new DataInputStream(new BufferedInputStream(new FileInputStream(file), 1024 * 1024));
} catch (OutOfMemoryError e) {
is = new DataInputStream(new FileInputStream(file));
}
long p;
int l;
while (true) {

3
source/net/yacy/kelondro/blob/Heap.java
Unescape View File

@ -40,6 +40,7 @@ import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.order.ByteOrder;
import net.yacy.kelondro.order.CloneableIterator;
import net.yacy.kelondro.order.NaturalOrder;
import net.yacy.kelondro.util.MemoryControl;
public final class Heap extends HeapModifier implements BLOB {
@ -349,7 +350,7 @@ public final class Heap extends HeapModifier implements BLOB {
assert this.buffer != null;
// if there is not enough space in the buffer, flush all
if (this.buffersize + b.length > buffermax) {
if (this.buffersize + b.length > buffermax || MemoryControl.shortStatus()) {
// this is too big. Flush everything
super.shrinkWithGapsAtEnd();
flushBuffer();

15
source/net/yacy/kelondro/blob/MapHeap.java
Unescape View File

@ -50,6 +50,7 @@ import net.yacy.kelondro.order.CloneableIterator;
import net.yacy.kelondro.order.NaturalOrder;
import net.yacy.kelondro.order.RotateIterator;
import net.yacy.kelondro.util.FileUtils;
import net.yacy.kelondro.util.MemoryControl;
import net.yacy.kelondro.util.kelondroException;
public class MapHeap implements Map<byte[], Map<String, String>> {
@ -155,7 +156,11 @@ public class MapHeap implements Map<byte[], Map<String, String>> {
if (blob != null) blob.insert(key, sb);
// write map to cache
cache.put(key, newMap);
if (MemoryControl.shortStatus()) {
cache.clear();
} else {
cache.put(key, newMap);
}
}
}
}
@ -283,8 +288,12 @@ public class MapHeap implements Map<byte[], Map<String, String>> {
throw new IOException(e.getMessage());
}
// write map to cache
cache.put(key, map);
if (MemoryControl.shortStatus()) {
cache.clear();
} else {
// write map to cache
cache.put(key, map);
}
}
// return value

15
source/net/yacy/kelondro/data/word/Word.java
Unescape View File

@ -56,7 +56,14 @@ public class Word {
public static final int commonHashLength = 12;
private static final int hashCacheSize = Math.max(100000, Math.min(10000000, (int) (MemoryControl.available() / 20000L)));
private static final ARC<String, byte[]> hashCache = new ConcurrentARC<String, byte[]>(hashCacheSize, 2 * Runtime.getRuntime().availableProcessors());
private static ARC<String, byte[]> hashCache = null;
static {
try {
hashCache = new ConcurrentARC<String, byte[]>(hashCacheSize, 2 * Runtime.getRuntime().availableProcessors());
} catch (OutOfMemoryError e) {
hashCache = new ConcurrentARC<String, byte[]>(1000, Runtime.getRuntime().availableProcessors());
}
}
// object carries statistics for words and sentences
public int count; // number of occurrences
@ -112,7 +119,11 @@ public class Word {
// calculate the hash
h = Base64Order.enhancedCoder.encodeSubstring(Digest.encodeMD5Raw(wordlc), commonHashLength);
assert h[2] != '@';
hashCache.put(wordlc, h); // prevent expensive MD5 computation and encoding
if (MemoryControl.shortStatus()) {
hashCache.clear();
} else {
hashCache.put(wordlc, h); // prevent expensive MD5 computation and encoding
}
return h;
}

7
source/net/yacy/kelondro/index/HandleMap.java
Unescape View File

@ -79,7 +79,12 @@ public final class HandleMap implements Iterable<Row.Entry> {
public HandleMap(final int keylength, final ByteOrder objectOrder, final int idxbytes, final File file) throws IOException, RowSpaceExceededException {
this(keylength, objectOrder, idxbytes, (int) (file.length() / (keylength + idxbytes)), file.getAbsolutePath());
// read the index dump and fill the index
InputStream is = new BufferedInputStream(new FileInputStream(file), 1024 * 1024);
InputStream is;
try {
is = new BufferedInputStream(new FileInputStream(file), 1024 * 1024);
} catch (OutOfMemoryError e) {
is = new FileInputStream(file);
}
if (file.getName().endsWith(".gz")) is = new GZIPInputStream(is);
final byte[] a = new byte[keylength + idxbytes];
int c;

8
source/net/yacy/kelondro/io/ByteCountInputStream.java
Unescape View File

@ -31,6 +31,8 @@ import java.io.IOException;
import java.io.InputStream;
//import java.util.HashMap;
import net.yacy.kelondro.logging.Log;
public final class ByteCountInputStream extends FilterInputStream {
// private final static Object syncObject = new Object();
@ -115,7 +117,11 @@ public final class ByteCountInputStream extends FilterInputStream {
// }
public final void close() throws IOException {
super.close();
try {
super.close();
} catch (OutOfMemoryError e) {
Log.logException(e);
}
this.finish();
}

8
source/net/yacy/kelondro/logging/Log.java
Unescape View File

@ -22,10 +22,12 @@
package net.yacy.kelondro.logging;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.PrintStream;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.logging.FileHandler;
@ -381,7 +383,11 @@ public final class Log {
Thread.setDefaultUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler(){
public void uncaughtException(final Thread t, final Throwable e) {
String msg = String.format("Thread %s: %s",t.getName(), e.getMessage());
exceptionLog.logSevere(msg, e);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
PrintStream ps = new PrintStream(baos);
e.printStackTrace(ps);
ps.close();
exceptionLog.logSevere(msg + "\n" + baos.toString(), e);
//System.err.print("Exception in thread \"" + t.getName() + "\" ");
//e.printStackTrace(System.err);
}

13
source/net/yacy/kelondro/util/MemoryControl.java
Unescape View File

@ -35,18 +35,16 @@ public class MemoryControl {
private static final Runtime runtime = Runtime.getRuntime();
public static long maxMemory = runtime.maxMemory(); // this value does never change during runtime
private static final Log log = new Log("MEMORY");
private static final long[] gcs = new long[5];
private static int gcs_pos = 0;
private static long lastGC = 0l;
private static long DHTMbyte = 0L;
private static long prevDHTtreshold = 0L;
private static int DHTtresholdCount = 0;
private static boolean allowDHT = true;
private static boolean shortStatus = false;
/**
* Runs the garbage collector if last garbage collection is more than last millis ago
@ -139,6 +137,11 @@ public class MemoryControl {
* @return whether enough memory could be freed (or is free) or not
*/
public static boolean request(final long size, final boolean force) {
boolean r = request0(size, force);
shortStatus = !r;
return r;
}
private static boolean request0(final long size, final boolean force) {
final long avg = getAverageGCFree();
if (avg >= size) return true;
long avail = available();
@ -168,6 +171,10 @@ public class MemoryControl {
}
}
public static boolean shortStatus() {
return shortStatus;
}
/**
* memory that is currently bound in objects
* @return used bytes

13
source/net/yacy/visualization/RasterPlotter.java
Unescape View File

@ -68,7 +68,7 @@ public class RasterPlotter {
protected final int width, height;
private final int[] cc;
private final BufferedImage image;
private BufferedImage image;
private final WritableRaster grid;
private int defaultColR, defaultColG, defaultColB;
private final long backgroundCol;
@ -79,9 +79,6 @@ public class RasterPlotter {
}
public RasterPlotter(final int width, final int height, final DrawMode drawMode, final long backgroundColor) {
if (!(MemoryControl.request(1024 * 1024 + 3 * width * height, false))) {
throw new RuntimeException(RasterPlotter.class.getSimpleName() + ": not enough memory (" + MemoryControl.available() + ") available");
}
this.cc = new int[3];
this.width = width;
this.height = height;
@ -90,8 +87,12 @@ public class RasterPlotter {
this.defaultColG = 0xFF;
this.defaultColB = 0xFF;
this.defaultMode = drawMode;
this.image = new BufferedImage(width, height, BufferedImage.TYPE_INT_RGB);
try {
this.image = new BufferedImage(width, height, BufferedImage.TYPE_INT_RGB);
} catch (OutOfMemoryError e) {
this.image = new BufferedImage(1, 1, BufferedImage.TYPE_INT_RGB);
//throw new RuntimeException(RasterPlotter.class.getSimpleName() + ": not enough memory (" + MemoryControl.available() + ") available");
}
this.clear();
this.grid = image.getRaster();
}

Write Preview
Loading…
Cancel
Save