boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

added more control over memory allocation

Browse Source 
should avoid some of the OOMs

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6436 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 16 years ago
parent 52470d0de4
commit 77c99e500f
4 changed files with 35 additions and 22 deletions
Show Stats Download Patch File Download Diff File

2
source/de/anomic/http/server/servlets/transferURL.java
Unescape View File

@ -141,7 +141,7 @@ public final class transferURL {
// write entry to database // write entry to database
yacyCore.log.logInfo("Accepting URL " + i + "/" + urlc + " from peer " + otherPeerName + ": " + lEntry.metadata().url().toNormalform(true, false)); yacyCore.log.logInfo("Accepting URL " + i + "/" + urlc + " from peer " + otherPeerName + ": " + lEntry.metadata().url().toNormalform(true, false));
try { try {
sb.indexSegments.urlMetadata(Segments.Process.DHTIN).store(lEntry); sb.indexSegments.urlMetadata(Segments.Process.DHTIN).store(lEntry); // OOM here!
sb.crawlResults.stack(lEntry, iam, iam, EventOrigin.DHT_TRANSFER); sb.crawlResults.stack(lEntry, iam, iam, EventOrigin.DHT_TRANSFER);
if (yacyCore.log.isFine()) yacyCore.log.logFine("transferURL: received URL '" + metadata.url().toNormalform(false, true) + "' from peer " + otherPeerName); if (yacyCore.log.isFine()) yacyCore.log.logFine("transferURL: received URL '" + metadata.url().toNormalform(false, true) + "' from peer " + otherPeerName);
received++; received++;

16
source/de/anomic/server/serverCore.java
Unescape View File

@ -540,7 +540,7 @@ public final class serverCore extends AbstractBusyThread implements BusyThread {
// closing the socket to the client // closing the socket to the client
if (this.controlSocket != null) try { if (this.controlSocket != null) try {
this.controlSocket.close(); this.controlSocket.close();
serverCore.this.log.logInfo("Closing main socket of thread '" + this.getName() + "'"); log.logInfo("Closing main socket of thread '" + this.getName() + "'");
//this.controlSocket = null; //this.controlSocket = null;
} catch (final Exception e) {} } catch (final Exception e) {}
} }
@ -558,7 +558,7 @@ public final class serverCore extends AbstractBusyThread implements BusyThread {
} }
public void log(final boolean outgoing, final String request) { public void log(final boolean outgoing, final String request) {
if (serverCore.this.log.isFine()) serverCore.this.log.logFine(this.userAddress.getHostAddress() + "/" + this.identity + " " + if (log.isFine()) log.logFine(this.userAddress.getHostAddress() + "/" + this.identity + " " +
"[" + ((busySessions == null)? -1 : busySessions.size()) + ", " + this.commandCounter + "[" + ((busySessions == null)? -1 : busySessions.size()) + ", " + this.commandCounter +
((outgoing) ? "] > " : "] < ") + ((outgoing) ? "] > " : "] < ") +
request); request);
@ -570,7 +570,7 @@ public final class serverCore extends AbstractBusyThread implements BusyThread {
} }
public byte[] readLine() { public byte[] readLine() {
return receive(this.in, serverCore.this.commandMaxLength, true); return receive(this.in, commandMaxLength, true);
} }
/** /**
@ -653,7 +653,7 @@ public final class serverCore extends AbstractBusyThread implements BusyThread {
} finally { } finally {
if (busySessions != null) { if (busySessions != null) {
busySessions.remove(this); busySessions.remove(this);
if (serverCore.this.log.isFinest()) serverCore.this.log.logFinest("* removed session "+ this.controlSocket.getRemoteSocketAddress() + " " + this.request); if (log.isFinest()) log.logFinest("* removed session "+ this.controlSocket.getRemoteSocketAddress() + " " + this.request);
} }
this.controlSocket = null; this.controlSocket = null;
} }
@ -664,7 +664,7 @@ public final class serverCore extends AbstractBusyThread implements BusyThread {
protected void finalize() { protected void finalize() {
if (busySessions != null && busySessions.contains(this)) { if (busySessions != null && busySessions.contains(this)) {
busySessions.remove(this); busySessions.remove(this);
if(serverCore.this.log.isFinest()) serverCore.this.log.logFinest("* removed session "+ this.controlSocket.getRemoteSocketAddress() + this.request); if(log.isFinest()) log.logFinest("* removed session "+ this.controlSocket.getRemoteSocketAddress() + this.request);
} }
this.close(); this.close();
} }
@ -708,7 +708,7 @@ public final class serverCore extends AbstractBusyThread implements BusyThread {
if (this.request == null) break; if (this.request == null) break;
if (reqProtocol.equals("HTTP")) { if (reqProtocol.equals("HTTP")) {
this.commandObj = serverCore.this.handlerPrototype.clone(); this.commandObj = handlerPrototype.clone();
} }
// initializing the session // initializing the session
@ -772,8 +772,8 @@ public final class serverCore extends AbstractBusyThread implements BusyThread {
} catch (final NoSuchMethodException e) { } catch (final NoSuchMethodException e) {
log.logSevere("command execution, method exception " + e.getMessage() + " for client " + this.userAddress.getHostAddress(), e); log.logSevere("command execution, method exception " + e.getMessage() + " for client " + this.userAddress.getHostAddress(), e);
if (!this.userAddress.isSiteLocalAddress()) { if (!this.userAddress.isSiteLocalAddress()) {
if (serverCore.this.denyHost != null) { if (denyHost != null) {
serverCore.this.denyHost.put((""+this.userAddress.getHostAddress()), "deny"); // block client: hacker attempt denyHost.put((""+this.userAddress.getHostAddress()), "deny"); // block client: hacker attempt
} }
} }
break; break;

31
source/net/yacy/kelondro/index/RowCollection.java
Unescape View File

@ -49,7 +49,8 @@ import net.yacy.kelondro.util.kelondroException;
public class RowCollection implements Iterable<Row.Entry> { public class RowCollection implements Iterable<Row.Entry> {
public static final long growfactor100 = 140L; public static final long growfactorLarge100 = 140L;
public static final long growfactorSmall100 = 120L;
private static final int isortlimit = 20; private static final int isortlimit = 20;
private static int availableCPU = Runtime.getRuntime().availableProcessors(); private static int availableCPU = Runtime.getRuntime().availableProcessors();
@ -202,13 +203,25 @@ public class RowCollection implements Iterable<Row.Entry> {
return this.rowdef; return this.rowdef;
} }
protected final void ensureSize(final int elements) { protected final long neededSpaceForEnsuredSize(final int elements, boolean forcegc) {
assert elements > 0 : "elements = " + elements; assert elements > 0 : "elements = " + elements;
final long needed = elements * rowdef.objectsize; final long needed = elements * rowdef.objectsize;
if (chunkcache.length >= needed) return; if (chunkcache.length >= needed) return 0;
assert needed > 0 : "needed = " + needed; assert needed > 0 : "needed = " + needed;
assert needed * growfactor100 / 100L > 0 : "elements = " + elements + ", new = " + (needed * growfactor100 / 100L); long allocram = needed * growfactorLarge100 / 100L;
byte[] newChunkcache = new byte[(int) (needed * growfactor100 / 100L)]; // increase space assert allocram > 0 : "elements = " + elements + ", new = " + allocram;
if (MemoryControl.request(allocram, false)) return allocram;
allocram = needed * growfactorSmall100 / 100L;
assert allocram > 0 : "elements = " + elements + ", new = " + allocram;
if (MemoryControl.request(allocram, forcegc)) return allocram;
return needed;
}
protected final void ensureSize(final int elements) {
long allocram = neededSpaceForEnsuredSize(elements, true);
if (allocram == 0) return;
assert allocram > chunkcache.length : "wrong alloc computation: allocram = " + allocram + ", chunkcache.length = " + chunkcache.length;
byte[] newChunkcache = new byte[(int) allocram]; // increase space
System.arraycopy(chunkcache, 0, newChunkcache, 0, chunkcache.length); System.arraycopy(chunkcache, 0, newChunkcache, 0, chunkcache.length);
chunkcache = newChunkcache; chunkcache = newChunkcache;
} }
@ -220,13 +233,13 @@ public class RowCollection implements Iterable<Row.Entry> {
* @return * @return
*/ */
public final long memoryNeededForGrow() { public final long memoryNeededForGrow() {
return (((long) (chunkcount + 1)) * ((long) rowdef.objectsize)) * growfactor100 / 100L; return neededSpaceForEnsuredSize(chunkcount + 1, false);
} }
public synchronized void trim(final boolean plusGrowFactor) { public synchronized void trim(final boolean plusGrowFactor) {
if (chunkcache.length == 0) return; if (chunkcache.length == 0) return;
int needed = chunkcount * rowdef.objectsize; long needed = chunkcount * rowdef.objectsize;
if (plusGrowFactor) needed = (int) (((long) needed) * growfactor100 / 100L); if (plusGrowFactor) needed = neededSpaceForEnsuredSize(chunkcount, false);
if (needed >= chunkcache.length) if (needed >= chunkcache.length)
return; // in case that the growfactor causes that the cache would return; // in case that the growfactor causes that the cache would
// grow instead of shrink, simply ignore the growfactor // grow instead of shrink, simply ignore the growfactor
@ -234,7 +247,7 @@ public class RowCollection implements Iterable<Row.Entry> {
return; // if the swap buffer is not available, we must give up. return; // if the swap buffer is not available, we must give up.
// This is not critical. Otherwise we provoke a serious // This is not critical. Otherwise we provoke a serious
// problem with OOM // problem with OOM
byte[] newChunkcache = new byte[needed]; byte[] newChunkcache = new byte[(int) needed];
System.arraycopy(chunkcache, 0, newChunkcache, 0, Math.min( System.arraycopy(chunkcache, 0, newChunkcache, 0, Math.min(
chunkcache.length, newChunkcache.length)); chunkcache.length, newChunkcache.length));
chunkcache = newChunkcache; chunkcache = newChunkcache;

6
source/net/yacy/kelondro/table/Table.java
Unescape View File

@ -268,9 +268,9 @@ public class Table implements ObjectIndex, Iterable<Row.Entry> {
final HashMap<String, String> map = new HashMap<String, String>(); final HashMap<String, String> map = new HashMap<String, String>();
map.put("tableSize", Integer.toString(index.size())); map.put("tableSize", Integer.toString(index.size()));
map.put("tableKeyChunkSize", Integer.toString(index.row().objectsize)); map.put("tableKeyChunkSize", Integer.toString(index.row().objectsize));
map.put("tableKeyMem", Integer.toString((int) (((long) index.row().objectsize) * ((long) index.size()) * RowCollection.growfactor100 / 100L))); map.put("tableKeyMem", Integer.toString(index.row().objectsize * index.size()));
map.put("tableValueChunkSize", (table == null) ? "0" : Integer.toString(table.row().objectsize)); map.put("tableValueChunkSize", (table == null) ? "0" : Integer.toString(table.row().objectsize));
map.put("tableValueMem", (table == null) ? "0" : Integer.toString((int) (((long) table.row().objectsize) * ((long) table.size()) * RowCollection.growfactor100 / 100L))); map.put("tableValueMem", (table == null) ? "0" : Integer.toString(table.row().objectsize * table.size()));
return map; return map;
} }
@ -279,7 +279,7 @@ public class Table implements ObjectIndex, Iterable<Row.Entry> {
} }
public static int staticRAMIndexNeed(final File f, final Row rowdef) throws IOException { public static int staticRAMIndexNeed(final File f, final Row rowdef) throws IOException {
return (int) (((long)(rowdef.primaryKeyLength + 4)) * tableSize(f, rowdef.objectsize) * RowCollection.growfactor100 / 100L); return (int) (((long)(rowdef.primaryKeyLength + 4)) * tableSize(f, rowdef.objectsize) * RowCollection.growfactorLarge100 / 100L);
} }
public synchronized void addUnique(final Entry row) throws IOException { public synchronized void addUnique(final Entry row) throws IOException {

Write Preview
Loading…
Cancel
Save