boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

fixed handlemap spread factor and null iterator handling

Browse Source
pull/1/head
Michael Peter Christen 13 years ago
parent 00f2df1120
commit 10c9c17d51
6 changed files with 88 additions and 56 deletions
Show Stats Download Patch File Download Diff File

4
htroot/PerformanceMemory_p.java
Unescape View File

@ -109,14 +109,14 @@ public class PerformanceMemory_p {
prop.put("EcoList_" + c + "_tableIndexPath", ((p = filename.indexOf("DATA",0)) < 0) ? filename : filename.substring(p));
prop.putNum("EcoList_" + c + "_tableSize", mapx.get(Table.StatKeys.tableSize));
assert mapx.get(Table.StatKeys.tableKeyMem) != null : mapx;
String v = mapx.get(Table.StatKeys.tableKeyMem);
mem = v == null ? 0 : Long.parseLong(v);
totalmem += mem;
prop.put("EcoList_" + c + "_tableKeyMem", Formatter.bytesToString(mem));
prop.put("EcoList_" + c + "_tableKeyChunkSize", mapx.get(Table.StatKeys.tableKeyChunkSize));
mem = Long.parseLong(mapx.get(Table.StatKeys.tableValueMem));
v = mapx.get(Table.StatKeys.tableValueMem);
mem = v == null ? 0 : Long.parseLong(v);
totalmem += mem;
prop.put("EcoList_" + c + "_tableValueMem", Formatter.bytesToString(mem));
prop.put("EcoList_" + c + "_tableValueChunkSize", mapx.get(Table.StatKeys.tableValueChunkSize));

6
source/net/yacy/kelondro/blob/MapHeap.java
Unescape View File

@ -392,17 +392,19 @@ public class MapHeap implements Map<byte[], Map<String, String>> {
this.firstKey = firstKey;
this.secondKey = secondKey;
final CloneableIterator<byte[]> i = MapHeap.this.blob.keys(up, firstKey);
this.iterator = (rotating) ? new RotateIterator<byte[]>(i, secondKey, MapHeap.this.blob.size()) : i;
this.iterator = rotating ? new RotateIterator<byte[]>(i, secondKey, MapHeap.this.blob.size()) : i;
}
@Override
public byte[] next() {
assert this.iterator != null;
if (this.iterator == null) return null;
return removeFillchar(this.iterator.next());
}
@Override
public boolean hasNext() {
return this.iterator.hasNext();
return this.iterator != null && this.iterator.hasNext();
}
@Override

2
source/net/yacy/kelondro/index/HandleMap.java
Unescape View File

@ -110,7 +110,7 @@ public final class HandleMap implements Iterable<Row.Entry> {
}
private static final int spread(final int expectedspace) {
return Math.min(Runtime.getRuntime().availableProcessors(), Math.max(1, expectedspace / 3000));
return Math.min(Runtime.getRuntime().availableProcessors(), Math.max(Runtime.getRuntime().availableProcessors(), expectedspace / 8000));
}
public final int[] saturation() {

23
source/net/yacy/kelondro/index/RAMIndexCluster.java
Unescape View File

@ -51,7 +51,7 @@ public final class RAMIndexCluster implements Index, Iterable<Row.Entry>, Clonea
this.cluster = new RAMIndex[clusterSize];
this.rowdef = rowdef;
for (int i = 0; i < clusterSize; i++) {
this.cluster[i] = new RAMIndex(name + "." + i, rowdef, 0);
this.cluster[i] = null; // lazy initialization, the actual initialization is at accessArray()
}
}
@ -69,7 +69,7 @@ public final class RAMIndexCluster implements Index, Iterable<Row.Entry>, Clonea
public RAMIndexCluster clone() {
final RAMIndex[] a = new RAMIndex[this.cluster.length];
for (int i = 0; i < this.cluster.length; i++) {
a[i] = this.cluster[i].clone();
a[i] = this.cluster[i] == null ? null : this.cluster[i].clone();
}
return new RAMIndexCluster(this.name + ".clone", this.rowdef, a);
}
@ -111,8 +111,11 @@ public final class RAMIndexCluster implements Index, Iterable<Row.Entry>, Clonea
private final RAMIndex accessArray(final int i) {
RAMIndex r = this.cluster[i];
if (r == null) synchronized (this.cluster) {
r = new RAMIndex(this.name + "." + i, this.rowdef, 0);
this.cluster[i] = r;
r = this.cluster[i];
if (r == null) {
r = new RAMIndex(this.name + "." + i, this.rowdef, 0);
this.cluster[i] = r;
}
}
return r;
}
@ -295,15 +298,13 @@ public final class RAMIndexCluster implements Index, Iterable<Row.Entry>, Clonea
@SuppressWarnings("unchecked")
public final CloneableIterator<Entry> rows(final boolean up, final byte[] firstKey) {
synchronized (this.cluster) {
final CloneableIterator<Entry>[] col = new CloneableIterator[this.cluster.length];
for (int i = 0; i < this.cluster.length; i++) {
if (this.cluster[i] == null) {
col[i] = null;
} else {
col[i] = this.cluster[i].rows(up, firstKey);
final List<CloneableIterator<Entry>> col = new ArrayList<CloneableIterator<Entry>>(this.cluster.length);
for (RAMIndex element : this.cluster) {
if (element != null) {
col.add(element.rows(up, firstKey));
}
}
return StackIterator.stack(col);
return StackIterator.stack(col.toArray(new CloneableIterator[col.size()]));
}
}

59
source/net/yacy/kelondro/order/StackIterator.java
Unescape View File

@ -28,10 +28,10 @@ import net.yacy.cora.order.CloneableIterator;
public class StackIterator<E> implements CloneableIterator<E> {
private final CloneableIterator<E> a, b;
private E na, nb;
public StackIterator(
final CloneableIterator<E> a,
final CloneableIterator<E> b) {
@ -42,57 +42,76 @@ public class StackIterator<E> implements CloneableIterator<E> {
nextb();
}
@Override
public StackIterator<E> clone(final Object modifier) {
return new StackIterator<E>(a.clone(modifier), b.clone(modifier));
return new StackIterator<E>(this.a.clone(modifier), this.b.clone(modifier));
}
private void nexta() {
try {
if ((a != null) && (a.hasNext())) na = a.next(); else na = null;
if ((this.a != null) && (this.a.hasNext())) this.na = this.a.next(); else this.na = null;
} catch (final ConcurrentModificationException e) {
na = null;
this.na = null;
}
}
private void nextb() {
try {
if ((b != null) && (b.hasNext())) nb = b.next(); else nb = null;
if ((this.b != null) && (this.b.hasNext())) this.nb = this.b.next(); else this.nb = null;
} catch (final ConcurrentModificationException e) {
nb = null;
this.nb = null;
}
}
@Override
public boolean hasNext() {
return (na != null) || (nb != null);
return (this.na != null) || (this.nb != null);
}
@Override
public E next() {
E s;
if (na == null) {
s = nb;
if (this.na == null) {
s = this.nb;
nextb();
return s;
}
if (nb == null) {
s = na;
if (this.nb == null) {
s = this.na;
nexta();
return s;
}
// just stack the Objects
s = na;
s = this.na;
nexta();
return s;
}
@Override
public void remove() {
throw new java.lang.UnsupportedOperationException("merge does not support remove");
}
@SuppressWarnings("unchecked")
public static <A> CloneableIterator<A> stack(final CloneableIterator<A>[] iterators) {
// this extends the ability to combine two iterators
// to the ability of combining a set of iterators
if (iterators == null) return null;
if (iterators.length == 0) return null;
if (iterators == null || iterators.length == 0) return new CloneableIterator<A>() {
@Override
public boolean hasNext() {
return false;
}
@Override
public A next() {
return null;
}
@Override
public void remove() {
}
@Override
public CloneableIterator<A> clone(Object modifier) {
return null;
}
};
if (iterators.length == 1) {
return iterators[0];
}

50
source/net/yacy/kelondro/util/StandardMemoryStrategy.java
Unescape View File

@ -38,7 +38,7 @@ public class StandardMemoryStrategy extends MemoryStrategy {
private long prevTreshold = 0L;
private int tresholdCount = 0;
private boolean proper = true;
public StandardMemoryStrategy() {
name = "Standard Memory Strategy";
error= false; //since this is the standard implementation we assume always false here
@ -49,6 +49,7 @@ public class StandardMemoryStrategy extends MemoryStrategy {
* @param last time which must be passed since lased gc
* @param info additional info for log
*/
@Override
protected final synchronized boolean gc(final int last, final String info) { // thq
assert last >= 10000; // too many forced GCs will cause bad execution performance
final long elapsed = System.currentTimeMillis() - lastGC;
@ -58,8 +59,8 @@ public class StandardMemoryStrategy extends MemoryStrategy {
System.gc();
lastGC = System.currentTimeMillis();
final long after = free();
gcs[gcs_pos++] = after - before;
if (gcs_pos >= gcs.length) gcs_pos = 0;
this.gcs[this.gcs_pos++] = after - before;
if (this.gcs_pos >= this.gcs.length) this.gcs_pos = 0;
if (log.isFine()) log.logInfo("[gc] before: " + Formatter.bytesToString(before) +
", after: " + Formatter.bytesToString(after) +
@ -80,7 +81,7 @@ public class StandardMemoryStrategy extends MemoryStrategy {
protected final long getAverageGCFree() {
long x = 0;
int y = 0;
for (final long gc : gcs)
for (final long gc : this.gcs)
if (gc != 0) {
x += gc;
y++;
@ -92,14 +93,16 @@ public class StandardMemoryStrategy extends MemoryStrategy {
* memory that is free without increasing of total memory taken from os
* @return bytes
*/
@Override
protected final long free() {
return runtime.freeMemory();
return this.runtime.freeMemory();
}
/**
* memory that is available including increasing total memory up to maximum
* @return bytes
*/
@Override
protected final long available() {
return maxMemory() - total() + free();
}
@ -108,18 +111,20 @@ public class StandardMemoryStrategy extends MemoryStrategy {
* maximum memory the Java virtual will allocate machine; may vary over time in some cases
* @return bytes
*/
@Override
protected final long maxMemory()
{
return runtime.maxMemory();
return this.runtime.maxMemory();
}
/**
* currently allocated memory in the Java virtual machine; may vary over time
* @return bytes
*/
@Override
protected final long total()
{
return runtime.totalMemory();
return this.runtime.totalMemory();
}
/**
@ -141,6 +146,7 @@ public class StandardMemoryStrategy extends MemoryStrategy {
* @param force specifies whether a GC should be run even in case former GCs didn't provide enough memory
* @return whether enough memory could be freed (or is free) or not
*/
@Override
protected boolean request(final long size, final boolean force, boolean shortStatus) {
if (size <= 0) return true;
final boolean r = request0(size, force);
@ -181,42 +187,46 @@ public class StandardMemoryStrategy extends MemoryStrategy {
* memory that is currently bound in objects
* @return used bytes
*/
@Override
protected final long used() {
return total() - free();
}
@Override
protected boolean properState() {
return proper;
return this.proper;
}
@Override
protected void resetProperState() {
proper = true;
tresholdCount = 0;
this.proper = true;
this.tresholdCount = 0;
}
/**
* set the memory to be available
*/
@Override
protected void setProperMbyte(final long mbyte) {
properMbyte = mbyte;
tresholdCount = 0;
this.properMbyte = mbyte;
this.tresholdCount = 0;
}
private void checkProper(final long available) {
// disable proper state if memory is less than treshold - 4 times, maximum 11 minutes between each detection
if ((available >> 20) < properMbyte) {
if ((available >> 20) < this.properMbyte) {
final long t = System.currentTimeMillis();
if(prevTreshold + 11L /* minutes */ * 60000L > t) {
tresholdCount++;
if(tresholdCount > 3 /* occurencies - 1 */) proper = false;
if(this.prevTreshold + 11L /* minutes */ * 60000L > t) {
this.tresholdCount++;
if(this.tresholdCount > 3 /* occurencies - 1 */) this.proper = false;
}
else tresholdCount = 1;
else this.tresholdCount = 1;
prevTreshold = t;
this.prevTreshold = t;
log.logInfo("checkProper: below treshold; tresholdCount: " + tresholdCount + "; proper: " + proper);
log.logInfo("checkProper: below treshold; tresholdCount: " + this.tresholdCount + "; proper: " + this.proper);
}
else if (!proper && (available >> 20) > (properMbyte * 2L)) // we were wrong!
else if (!this.proper && (available >> 20) > (this.properMbyte * 2L)) // we were wrong!
resetProperState();
}

Write Preview
Loading…
Cancel
Save