diff --git a/htroot/PerformanceMemory_p.html b/htroot/PerformanceMemory_p.html index b3e533548..e0fac875b 100644 --- a/htroot/PerformanceMemory_p.html +++ b/htroot/PerformanceMemory_p.html @@ -99,6 +99,30 @@ +

Object Index Caches:

+ + + + + + + + + + #{indexcache}# + + + + + + + + #{/indexcache}# + + + +
TableSizeChunk SizeNeeded MemoryUsed Memory
#[Name]##[Count]##[ChunkSize]##[NeededMem]##[UsedMem]#
Total Mem: #[indexcacheTotalMem]# MB
+

Object Read Caches:

diff --git a/htroot/PerformanceMemory_p.java b/htroot/PerformanceMemory_p.java index bab97d110..9d528ec39 100644 --- a/htroot/PerformanceMemory_p.java +++ b/htroot/PerformanceMemory_p.java @@ -30,6 +30,7 @@ import java.util.Iterator; import java.util.Map; import net.yacy.kelondro.index.Cache; +import net.yacy.kelondro.index.ObjectIndexCache; import net.yacy.kelondro.table.Table; import net.yacy.kelondro.util.Domains; import net.yacy.kelondro.util.FileUtils; @@ -116,10 +117,36 @@ public class PerformanceMemory_p { prop.put("EcoList", c); prop.putNum("EcoIndexTotalMem", totalmem / (1024 * 1024d)); + // write object cache table + Iterator> oi = ObjectIndexCache.objects(); + c = 0; + mem = 0; + Map.Entry oie; + ObjectIndexCache cache; + long hitmem, totalhitmem = 0; + while (oi.hasNext()) { + oie = oi.next(); + filename = oie.getKey(); + cache = oie.getValue(); + prop.put("indexcache_" + c + "_Name", ((p = filename.indexOf("DATA")) < 0) ? filename : filename.substring(p)); + + hitmem = cache.mem(); + totalhitmem += hitmem; + prop.put("indexcache_" + c + "_ChunkSize", cache.row().objectsize); + prop.putNum("indexcache_" + c + "_Count", cache.size()); + prop.put("indexcache_" + c + "_NeededMem", cache.size() * cache.row().objectsize); + prop.put("indexcache_" + c + "_UsedMem", hitmem); + + c++; + } + prop.put("indexcache", c); + prop.putNum("indexcacheTotalMem", totalhitmem / (1024 * 1024d)); + // write object cache table i = Cache.filenames(); c = 0; - long hitmem, missmem, totalhitmem = 0, totalmissmem = 0; + long missmem, totalmissmem = 0; + totalhitmem = 0; while (i.hasNext()) { filename = i.next(); map = Cache.memoryStats(filename); diff --git a/source/de/anomic/crawler/Balancer.java b/source/de/anomic/crawler/Balancer.java index 1a134886a..07eaee23f 100644 --- a/source/de/anomic/crawler/Balancer.java +++ b/source/de/anomic/crawler/Balancer.java @@ -498,7 +498,9 @@ public class Balancer { Request request; while (i.hasNext()) { handle = i.next(); - request = new Request(this.urlFileIndex.get(handle)); + Row.Entry entry = this.urlFileIndex.get(handle); + if (entry == null) continue; + request = new Request(entry); host = request.url().getHost(); try { pushHashToDomainStacks(host, handle); diff --git a/source/de/anomic/data/URLAnalysis.java b/source/de/anomic/data/URLAnalysis.java index 86b1a8856..0ab280ac5 100644 --- a/source/de/anomic/data/URLAnalysis.java +++ b/source/de/anomic/data/URLAnalysis.java @@ -408,6 +408,7 @@ public class URLAnalysis { System.out.println("INDEX REFERENCE COLLECTION starting dump of statistics"); idx.dump(new File(statisticPath)); System.out.println("INDEX REFERENCE COLLECTION finished dump, wrote " + idx.size() + " entries to " + statisticPath); + idx.close(); } catch (Exception e) { Log.logException(e); } @@ -433,6 +434,7 @@ public class URLAnalysis { update = System.currentTimeMillis(); } } + idx.close(); mr.close(); System.out.println("INDEX DIFF URL-COL finished diff, starting dump to " + diffFile); count = hs.dump(new File(diffFile)); diff --git a/source/de/anomic/search/blockrank/CRProcess.java b/source/de/anomic/search/blockrank/CRProcess.java index 8ae828307..836c79d6b 100644 --- a/source/de/anomic/search/blockrank/CRProcess.java +++ b/source/de/anomic/search/blockrank/CRProcess.java @@ -217,7 +217,7 @@ public class CRProcess { } else { // initialize counters and dates acc_entry = acc.row().newEntry(); - acc_entry.setCol("Referee", key, null); + acc_entry.setCol(0, key, null); for (int i = 1; i < acc.row().columns(); i++) { acc_entry.setCol(i, new_entry.getAttr(acc.row().column(i).nickname, 0)); } diff --git a/source/de/anomic/yacy/dht/FlatWordPartitionScheme.java b/source/de/anomic/yacy/dht/FlatWordPartitionScheme.java index 3b4720bf1..318602292 100755 --- a/source/de/anomic/yacy/dht/FlatWordPartitionScheme.java +++ b/source/de/anomic/yacy/dht/FlatWordPartitionScheme.java @@ -111,7 +111,7 @@ public class FlatWordPartitionScheme implements PartitionScheme { System.gc(); // for resource measurement long a = MemoryControl.available(); - HandleMap idx = new HandleMap(12, Base64Order.enhancedCoder, 4, 150000); + HandleMap idx = new HandleMap(12, Base64Order.enhancedCoder, 4, 150000, "test"); for (int i = 0; i < count; i++) { try { idx.inc(FlatWordPartitionScheme.positionToHash(r.nextInt(count))); @@ -126,7 +126,7 @@ public class FlatWordPartitionScheme implements PartitionScheme { long memk = a - MemoryControl.available(); System.out.println("Used Memory: " + memk + " bytes"); System.out.println("x " + idx.get(FlatWordPartitionScheme.positionToHash(0))); - idx = null; + idx.close(); r = new Random(0); start = System.currentTimeMillis(); diff --git a/source/net/yacy/document/parser/html/TransformerWriter.java b/source/net/yacy/document/parser/html/TransformerWriter.java index 81d4e49db..e106e8ba6 100644 --- a/source/net/yacy/document/parser/html/TransformerWriter.java +++ b/source/net/yacy/document/parser/html/TransformerWriter.java @@ -245,7 +245,7 @@ public final class TransformerWriter extends Writer { } catch (IOException e) { Log.logException(e); } - filterCont = new CharBuffer(); + if (filterCont == null) filterCont = new CharBuffer(); else filterCont.reset(); return new char[0]; } else { // we ignore that thing and return it again diff --git a/source/net/yacy/kelondro/blob/HeapReader.java b/source/net/yacy/kelondro/blob/HeapReader.java index 6747241c2..a621df8c4 100644 --- a/source/net/yacy/kelondro/blob/HeapReader.java +++ b/source/net/yacy/kelondro/blob/HeapReader.java @@ -189,7 +189,7 @@ public class HeapReader { Log.logInfo("HeapReader", "generating index for " + heapFile.toString() + ", " + (file.length() / 1024 / 1024) + " MB. Please wait."); this.free = new Gap(); - HandleMap.initDataConsumer indexready = HandleMap.asynchronusInitializer(keylength, this.ordering, 8, Math.max(10, (int) (Runtime.getRuntime().freeMemory() / (10 * 1024 * 1024)))); + HandleMap.initDataConsumer indexready = HandleMap.asynchronusInitializer(this.name() + ".initializer", keylength, this.ordering, 8, Math.max(10, (int) (Runtime.getRuntime().freeMemory() / (10 * 1024 * 1024)))); byte[] key = new byte[keylength]; int reclen; long seek = 0; @@ -248,7 +248,6 @@ public class HeapReader { Log.logException(e); } Log.logInfo("HeapReader", "finished index generation for " + heapFile.toString() + ", " + index.size() + " entries, " + free.size() + " gaps."); - } private void mergeFreeEntries() throws IOException { diff --git a/source/net/yacy/kelondro/blob/HeapWriter.java b/source/net/yacy/kelondro/blob/HeapWriter.java index 8555fb3ae..9d1ad632c 100644 --- a/source/net/yacy/kelondro/blob/HeapWriter.java +++ b/source/net/yacy/kelondro/blob/HeapWriter.java @@ -80,7 +80,7 @@ public final class HeapWriter { this.heapFileTMP = temporaryHeapFile; this.heapFileREADY = readyHeapFile; this.keylength = keylength; - this.index = new HandleMap(keylength, ordering, 8, 100000); + this.index = new HandleMap(keylength, ordering, 8, 100000, readyHeapFile.getAbsolutePath()); this.os = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(temporaryHeapFile), outBuffer)); this.seek = 0; } diff --git a/source/net/yacy/kelondro/index/BufferedObjectIndex.java b/source/net/yacy/kelondro/index/BufferedObjectIndex.java index 4bf728efd..7bd285799 100644 --- a/source/net/yacy/kelondro/index/BufferedObjectIndex.java +++ b/source/net/yacy/kelondro/index/BufferedObjectIndex.java @@ -72,6 +72,10 @@ public class BufferedObjectIndex implements ObjectIndex, Iterable { } } + public long mem() { + return this.backend.mem() + this.buffer.mem(); + } + /** * check size of buffer in such a way that a put into the buffer is possible * afterwards without exceeding the given maximal buffersize diff --git a/source/net/yacy/kelondro/index/Cache.java b/source/net/yacy/kelondro/index/Cache.java index ac6d1bf0a..851d9e481 100644 --- a/source/net/yacy/kelondro/index/Cache.java +++ b/source/net/yacy/kelondro/index/Cache.java @@ -98,6 +98,10 @@ public final class Cache implements ObjectIndex, Iterable { this.hasnotDelete = 0; } + public long mem() { + return this.index.mem() + this.readHitCache.mem() + this.readMissCache.mem(); + } + public final int writeBufferSize() { return 0; } diff --git a/source/net/yacy/kelondro/index/HandleMap.java b/source/net/yacy/kelondro/index/HandleMap.java index 1c2ec1fac..2f6405186 100644 --- a/source/net/yacy/kelondro/index/HandleMap.java +++ b/source/net/yacy/kelondro/index/HandleMap.java @@ -63,9 +63,9 @@ public final class HandleMap implements Iterable { * @param objectOrder * @param space */ - public HandleMap(final int keylength, final ByteOrder objectOrder, final int idxbytes, final int expectedspace) { + public HandleMap(final int keylength, final ByteOrder objectOrder, final int idxbytes, final int expectedspace, String name) { this.rowdef = new Row(new Column[]{new Column("key", Column.celltype_binary, Column.encoder_bytes, keylength, "key"), new Column("long c-" + idxbytes + " {b256}")}, objectOrder); - this.index = new RowSetArray(rowdef, spread(expectedspace)); + this.index = new RowSetArray(name, rowdef, spread(expectedspace)); } /** @@ -77,7 +77,7 @@ public final class HandleMap implements Iterable { * @throws RowSpaceExceededException */ public HandleMap(final int keylength, final ByteOrder objectOrder, final int idxbytes, final File file) throws IOException, RowSpaceExceededException { - this(keylength, objectOrder, idxbytes, (int) (file.length() / (keylength + idxbytes))); + this(keylength, objectOrder, idxbytes, (int) (file.length() / (keylength + idxbytes)), file.getAbsolutePath()); // read the index dump and fill the index InputStream is = new BufferedInputStream(new FileInputStream(file), 1024 * 1024); if (file.getName().endsWith(".gz")) is = new GZIPInputStream(is); @@ -95,8 +95,12 @@ public final class HandleMap implements Iterable { assert this.index.size() == file.length() / (keylength + idxbytes); } + public long mem() { + return index.mem(); + } + private static final int spread(int expectedspace) { - return Math.min(Runtime.getRuntime().availableProcessors() * 2, Math.max(1, expectedspace / 3000)); + return Math.min(Runtime.getRuntime().availableProcessors(), Math.max(1, expectedspace / 3000)); } public final int[] saturation() { @@ -304,8 +308,8 @@ public final class HandleMap implements Iterable { * @param bufferSize * @return */ - public final static initDataConsumer asynchronusInitializer(final int keylength, final ByteOrder objectOrder, final int idxbytes, final int expectedspace) { - final initDataConsumer initializer = new initDataConsumer(new HandleMap(keylength, objectOrder, idxbytes, expectedspace)); + public final static initDataConsumer asynchronusInitializer(String name, final int keylength, final ByteOrder objectOrder, final int idxbytes, final int expectedspace) { + final initDataConsumer initializer = new initDataConsumer(new HandleMap(keylength, objectOrder, idxbytes, expectedspace, name)); final ExecutorService service = Executors.newSingleThreadExecutor(); initializer.setResult(service.submit(initializer)); service.shutdown(); @@ -389,6 +393,9 @@ public final class HandleMap implements Iterable { return map; } + public void close() { + this.map.close(); + } } public Iterator iterator() { diff --git a/source/net/yacy/kelondro/index/IndexTest.java b/source/net/yacy/kelondro/index/IndexTest.java index b79e5e1a7..a613804e1 100644 --- a/source/net/yacy/kelondro/index/IndexTest.java +++ b/source/net/yacy/kelondro/index/IndexTest.java @@ -113,7 +113,7 @@ public class IndexTest { Runtime.getRuntime().gc(); final long freeStartKelondro = MemoryControl.available(); HandleMap ii = null; - ii = new HandleMap(12, Base64Order.enhancedCoder, 4, count); + ii = new HandleMap(12, Base64Order.enhancedCoder, 4, count, "test"); for (int i = 0; i < count; i++) try { ii.putUnique(tests[i], 1); @@ -128,7 +128,7 @@ public class IndexTest { for (int i = 0; i < count; i++) if (ii.get(tests[i]) != 1) bugs++; Runtime.getRuntime().gc(); final long freeEndKelondro = MemoryControl.available(); - ii.clear(); ii = null; + ii.clear(); ii.close(); ii = null; final long t7 = System.currentTimeMillis(); System.out.println("time for HandleMap test: " + (t7 - t6) + ", " + bugs + " bugs"); System.out.println("memory for HandleMap: " + (freeStartKelondro - freeEndKelondro) / mb + " MB\n"); diff --git a/source/net/yacy/kelondro/index/ObjectIndex.java b/source/net/yacy/kelondro/index/ObjectIndex.java index 62741422e..41ef7ad7e 100644 --- a/source/net/yacy/kelondro/index/ObjectIndex.java +++ b/source/net/yacy/kelondro/index/ObjectIndex.java @@ -43,6 +43,7 @@ public interface ObjectIndex extends Iterable { public String filename(); // returns a unique identified for this index; can be a real or artificial file name public int size(); + public long mem(); public boolean isEmpty(); public Row row(); public byte[] smallestKey(); diff --git a/source/net/yacy/kelondro/index/ObjectIndexCache.java b/source/net/yacy/kelondro/index/ObjectIndexCache.java index 156b43133..db5982c0e 100644 --- a/source/net/yacy/kelondro/index/ObjectIndexCache.java +++ b/source/net/yacy/kelondro/index/ObjectIndexCache.java @@ -28,6 +28,8 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Iterator; import java.util.List; +import java.util.Map; +import java.util.TreeMap; import net.yacy.kelondro.index.Row.Entry; import net.yacy.kelondro.order.CloneableIterator; @@ -37,35 +39,39 @@ import net.yacy.kelondro.order.StackIterator; public final class ObjectIndexCache implements ObjectIndex, Iterable { + private static final TreeMap objectTracker = new TreeMap(); + + private final String name; private final Row rowdef; private RowSet index0; private RowSet index1; private final Row.EntryComparator entryComparator; //private final int spread; - public ObjectIndexCache(final Row rowdef, final int expectedspace) { + public ObjectIndexCache(String name, final Row rowdef, final int expectedspace) { + this.name = name; this.rowdef = rowdef; this.entryComparator = new Row.EntryComparator(rowdef.objectOrder); //this.spread = Math.max(10, expectedspace / 3000); reset(); + objectTracker.put(name, this); } - public ObjectIndexCache(final Row rowdef, final int expectedspace, final int initialspace) throws RowSpaceExceededException { - this.rowdef = rowdef; - this.entryComparator = new Row.EntryComparator(rowdef.objectOrder); - //this.spread = Math.max(10, expectedspace / 3000); - reset(initialspace); - } - - private ObjectIndexCache(final Row rowdef, RowSet index0, RowSet index1, Row.EntryComparator entryComparator) { + private ObjectIndexCache(String name, final Row rowdef, RowSet index0, RowSet index1, Row.EntryComparator entryComparator) { + this.name = name; this.rowdef = rowdef; this.index0 = index0; this.index1 = index1; this.entryComparator = entryComparator; + objectTracker.put(name, this); + } + + public static final Iterator> objects() { + return objectTracker.entrySet().iterator(); } public ObjectIndexCache clone() { - return new ObjectIndexCache(this.rowdef, index0.clone(), index1.clone(), entryComparator); + return new ObjectIndexCache(this.name + ".clone", this.rowdef, index0.clone(), index1.clone(), entryComparator); } public void clear() { @@ -247,6 +253,17 @@ public final class ObjectIndexCache implements ObjectIndex, Iterable return list; } + public long mem() { + if (index0 != null && index1 == null) { + return index0.mem(); + } + if (index0 == null && index1 != null) { + return index1.mem(); + } + assert (index0 != null && index1 != null); + return index0.mem() + index1.mem(); + } + public final synchronized int size() { if (index0 != null && index1 == null) { return index0.size(); @@ -359,6 +376,7 @@ public final class ObjectIndexCache implements ObjectIndex, Iterable public final synchronized void close() { if (index0 != null) index0.close(); if (index1 != null) index1.close(); + objectTracker.remove(this.name); } public final String filename() { diff --git a/source/net/yacy/kelondro/index/Row.java b/source/net/yacy/kelondro/index/Row.java index c4416b196..d8ed5fa23 100644 --- a/source/net/yacy/kelondro/index/Row.java +++ b/source/net/yacy/kelondro/index/Row.java @@ -388,14 +388,6 @@ public final class Row { return rowinstance[offset + colstart[column]] == 0; } - @Deprecated - public final void setCol(final String nickname, final char c) { - if (nickref == null) genNickRef(); - final Object[] ref = nickref.get(nickname); - if (ref == null) return; - rowinstance[offset + ((Integer) ref[1]).intValue()] = (byte) c; - } - @Deprecated public final void setCol(final String nickname, final byte[] cell) { if (nickref == null) genNickRef(); @@ -448,17 +440,6 @@ public final class Row { } } - public final void setCol(final String nick, final String cell, final String encoding) { - if (encoding == null) - setCol(nick, cell.getBytes()); - else - try { - setCol(nick, cell.getBytes(encoding)); - } catch (final UnsupportedEncodingException e) { - Log.logSevere("Row", "", e); - } - } - @Deprecated public final void setCol(final String nickname, final long cell) { if (nickref == null) genNickRef(); @@ -506,26 +487,6 @@ public final class Row { throw new kelondroException("ROW", "addCol did not find appropriate encoding"); } - @Deprecated - public final byte[] getCol(final String nickname, final byte[] dflt) { - if (nickref == null) genNickRef(); - final Object[] ref = nickref.get(nickname); - if (ref == null) return dflt; - final Column col = (Column) ref[0]; - final byte[] cell = new byte[col.cellwidth]; - System.arraycopy(rowinstance, offset + ((Integer) ref[1]).intValue(), cell, 0, cell.length); - return cell; - } - - @Deprecated - public final String getColString(final String nickname, final String dflt, final String encoding) { - if (nickref == null) genNickRef(); - final Object[] ref = nickref.get(nickname); - if (ref == null) return dflt; - final Column col = (Column) ref[0]; - return getColString(((Integer) ref[1]).intValue(), col.cellwidth, encoding); - } - public final String getColString(final int column, final String encoding) { return getColString(colstart[column], row[column].cellwidth, encoding); } @@ -545,16 +506,6 @@ public final class Row { } } - @Deprecated - public final long getColLong(final String nickname, final long dflt) { - if (nickref == null) genNickRef(); - final Object[] ref = nickref.get(nickname); - if (ref == null) return dflt; - final Column col = (Column) ref[0]; - final int clstrt = ((Integer) ref[1]).intValue(); - return getColLong(col.encoder, clstrt, col.cellwidth); - } - public final long getColLong(final int column) { // uses the column definition to choose the right encoding return getColLong(row[column].encoder, colstart[column], row[column].cellwidth); @@ -582,14 +533,6 @@ public final class Row { throw new kelondroException("ROW", "getColLong did not find appropriate encoding"); } - @Deprecated - public final byte getColByte(final String nickname, final byte dflt) { - if (nickref == null) genNickRef(); - final Object[] ref = nickref.get(nickname); - if (ref == null) return dflt; - return rowinstance[offset + ((Integer) ref[1]).intValue()]; - } - public final byte getColByte(final int column) { return rowinstance[offset + colstart[column]]; } @@ -619,13 +562,6 @@ public final class Row { return c; } - public final char[] getColChars(final int column) { - final int w = row[column].cellwidth; - final char[] c = new char[w]; - System.arraycopy(rowinstance, offset + colstart[column], c, 0, w); - return c; - } - public final void writeToArray(final int column, final byte[] target, final int targetOffset) { // this method shall replace the getColBytes where possible, bacause it may reduce the number of new byte[] allocations assert (targetOffset + row[column].cellwidth <= target.length) : "targetOffset = " + targetOffset + ", target.length = " + target.length + ", row[column].cellwidth() = " + row[column].cellwidth; diff --git a/source/net/yacy/kelondro/index/RowCollection.java b/source/net/yacy/kelondro/index/RowCollection.java index 344c091b7..56fea2db1 100644 --- a/source/net/yacy/kelondro/index/RowCollection.java +++ b/source/net/yacy/kelondro/index/RowCollection.java @@ -167,6 +167,14 @@ public class RowCollection implements Iterable, Cloneable { this.chunkcount = 0; this.sortBound = 0; } + + /** + * calculate the memory that the structure occupies in ram + * @return number of bytes in use + */ + public long mem() { + return this.chunkcache.length; + } private static final Row exportMeasureRow = exportRow(0 /* no relevance */); diff --git a/source/net/yacy/kelondro/index/RowSetArray.java b/source/net/yacy/kelondro/index/RowSetArray.java index 1b6b6cd68..89e76b6d4 100644 --- a/source/net/yacy/kelondro/index/RowSetArray.java +++ b/source/net/yacy/kelondro/index/RowSetArray.java @@ -36,19 +36,22 @@ import net.yacy.kelondro.order.StackIterator; public final class RowSetArray implements ObjectIndex, Iterable, Cloneable { - private final Row rowdef; + private final String name; + private final Row rowdef; private final ObjectIndexCache[] array; - public RowSetArray(final Row rowdef, final int arraySize) { + public RowSetArray(String name, final Row rowdef, final int arraySize) { //assert arraySize < 100 : arraySize; + this.name = name; this.array = new ObjectIndexCache[arraySize]; this.rowdef = rowdef; for (int i = 0; i < arraySize; i++) { - this.array[i] = new ObjectIndexCache(rowdef, 0); + this.array[i] = new ObjectIndexCache(name + "." + i, rowdef, 0); } } - private RowSetArray(final Row rowdef, final ObjectIndexCache[] array) { + private RowSetArray(String name, final Row rowdef, final ObjectIndexCache[] array) { + this.name = name; this.array = array; this.rowdef = rowdef; } @@ -58,15 +61,15 @@ public final class RowSetArray implements ObjectIndex, Iterable, Clon for (int i = 0; i < this.array.length; i++) { a[i] = this.array[i].clone(); } - return new RowSetArray(this.rowdef, a); + return new RowSetArray(this.name + ".clone", this.rowdef, a); } private final int indexFor(final byte[] key) { - return (int) (this.rowdef.objectOrder.cardinal(key) % ((long) array.length)); + return (int) ((this.rowdef.objectOrder.cardinal(key) / 17) % ((long) array.length)); } private final int indexFor(final Entry row) { - return (int) (this.rowdef.objectOrder.cardinal(row.bytes(), 0, row.getPrimaryKeyLength()) % ((long) array.length)); + return (int) ((this.rowdef.objectOrder.cardinal(row.bytes(), 0, row.getPrimaryKeyLength()) / 17) % ((long) array.length)); } public final byte[] smallestKey() { @@ -96,7 +99,7 @@ public final class RowSetArray implements ObjectIndex, Iterable, Clon private final ObjectIndexCache accessArray(final int i) { ObjectIndexCache r = this.array[i]; if (r == null) synchronized (this.array) { - r = new ObjectIndexCache(this.rowdef, 0); + r = new ObjectIndexCache(name + "." + i, this.rowdef, 0); this.array[i] = r; } return r; @@ -114,15 +117,15 @@ public final class RowSetArray implements ObjectIndex, Iterable, Clon public final void clear() { synchronized (this.array) { - for (int i = 0; i < this.array.length; i++) { - if (this.array[i] != null) this.array[i].clear(); - this.array[i] = null; - } + for (ObjectIndexCache c: this.array) if (c != null) c.clear(); } } public final void close() { clear(); + synchronized (this.array) { + for (ObjectIndexCache c: this.array) if (c != null) c.close(); + } } public final void deleteOnExit() { @@ -265,6 +268,19 @@ public final class RowSetArray implements ObjectIndex, Iterable, Clon return c; } + + public long mem() { + long m = 0; + synchronized (this.array) { + for (int i = 0; i < this.array.length; i++) { + if (this.array[i] != null) { + m += this.array[i].mem(); + } + } + } + return m; + } + public final boolean isEmpty() { synchronized (this.array) { for (int i = 0; i < this.array.length; i++) { diff --git a/source/net/yacy/kelondro/io/CharBuffer.java b/source/net/yacy/kelondro/io/CharBuffer.java index fa833bf94..5c9b0477a 100644 --- a/source/net/yacy/kelondro/io/CharBuffer.java +++ b/source/net/yacy/kelondro/io/CharBuffer.java @@ -123,7 +123,7 @@ public final class CharBuffer extends Writer { private void grow() { int newsize = buffer.length * 2 + 1; - if (newsize < 256) newsize = 256; + if (newsize < 32) newsize = 32; char[] tmp = new char[newsize]; System.arraycopy(buffer, offset, tmp, 0, length); buffer = tmp; @@ -173,7 +173,9 @@ public final class CharBuffer extends Writer { } public CharBuffer append(final String s) { - return append(s,0,s.length()); + final char[] temp = new char[s.length()]; + s.getChars(0, temp.length, temp, 0); + return append(temp); } public CharBuffer append(final String s, final int off, final int len) { diff --git a/source/net/yacy/kelondro/rwi/ReferenceContainerArray.java b/source/net/yacy/kelondro/rwi/ReferenceContainerArray.java index 79e8e4749..38191f58d 100644 --- a/source/net/yacy/kelondro/rwi/ReferenceContainerArray.java +++ b/source/net/yacy/kelondro/rwi/ReferenceContainerArray.java @@ -331,7 +331,7 @@ public final class ReferenceContainerArray { final Row payloadrow) throws IOException, RowSpaceExceededException { System.out.println("CELL REFERENCE COLLECTION startup"); - HandleMap references = new HandleMap(payloadrow.primaryKeyLength, termOrder, 4, 1000000); + HandleMap references = new HandleMap(payloadrow.primaryKeyLength, termOrder, 4, 1000000, heapLocation.getAbsolutePath()); String[] files = heapLocation.list(); for (String f: files) { if (f.length() < 22 || !f.startsWith("text.index") || !f.endsWith(".blob")) continue; diff --git a/source/net/yacy/kelondro/table/SQLTable.java b/source/net/yacy/kelondro/table/SQLTable.java index d78b9ad8f..c3ce5c164 100644 --- a/source/net/yacy/kelondro/table/SQLTable.java +++ b/source/net/yacy/kelondro/table/SQLTable.java @@ -102,7 +102,11 @@ public class SQLTable implements ObjectIndex, Iterable { } } - + + public long mem() { + return 0; + } + public byte[] smallestKey() { return null; } diff --git a/source/net/yacy/kelondro/table/SplitTable.java b/source/net/yacy/kelondro/table/SplitTable.java index e786d4f1f..420cc34b4 100644 --- a/source/net/yacy/kelondro/table/SplitTable.java +++ b/source/net/yacy/kelondro/table/SplitTable.java @@ -80,7 +80,7 @@ public class SplitTable implements ObjectIndex, Iterable { private long fileSizeLimit; private boolean useTailCache; private boolean exceed134217727; - + public SplitTable( final File path, final String tablename, @@ -109,6 +109,12 @@ public class SplitTable implements ObjectIndex, Iterable { init(); } + public long mem() { + long m = 0; + for (ObjectIndex i: tables.values()) m += i.mem(); + return m; + } + public final byte[] smallestKey() { HandleSet keysort = new HandleSet(this.rowdef.primaryKeyLength, this.rowdef.objectOrder, this.tables.size()); for (ObjectIndex oi: this.tables.values()) try { diff --git a/source/net/yacy/kelondro/table/Table.java b/source/net/yacy/kelondro/table/Table.java index 05df29923..20fa238be 100644 --- a/source/net/yacy/kelondro/table/Table.java +++ b/source/net/yacy/kelondro/table/Table.java @@ -124,7 +124,7 @@ public class Table implements ObjectIndex, Iterable { // initialize index and copy table final int records = Math.max(fileSize, initialSpace); final long neededRAM4table = (records) * ((rowdef.objectsize) + 4L) * 3L; - table = ((exceed134217727 || neededRAM4table < maxarraylength) && + this.table = ((exceed134217727 || neededRAM4table < maxarraylength) && (useTailCache && MemoryControl.available() > neededRAM4table + 200 * 1024 * 1024)) ? new RowSet(taildef, records) : null; Log.logInfo("TABLE", "initialization of " + tablefile.getName() + ". table copy: " + ((table == null) ? "no" : "yes") + ", available RAM: " + (MemoryControl.available() / 1024 / 1024) + "MB, needed: " + (neededRAM4table/1024/1024 + 200) + "MB, allocating space for " + records + " entries"); @@ -134,18 +134,18 @@ public class Table implements ObjectIndex, Iterable { // there is now not enough memory left for the index. So delete the table again to free the memory // for the index Log.logSevere("TABLE", tablefile.getName() + ": not enough RAM (" + (MemoryControl.available() / 1024 / 1024) + "MB) left for index, deleting allocated table space to enable index space allocation (needed: " + (neededRAM4index / 1024 / 1024) + "MB)"); - table = null; System.gc(); + this.table = null; System.gc(); Log.logSevere("TABLE", tablefile.getName() + ": RAM after releasing the table: " + (MemoryControl.available() / 1024 / 1024) + "MB"); } - index = new HandleMap(rowdef.primaryKeyLength, rowdef.objectOrder, 4, records); - HandleMap errors = new HandleMap(rowdef.primaryKeyLength, NaturalOrder.naturalOrder, 4, records); + this.index = new HandleMap(rowdef.primaryKeyLength, rowdef.objectOrder, 4, records, tablefile.getAbsolutePath()); + HandleMap errors = new HandleMap(rowdef.primaryKeyLength, NaturalOrder.naturalOrder, 4, records, tablefile.getAbsolutePath() + ".errors"); Log.logInfo("TABLE", tablefile + ": TABLE " + tablefile.toString() + " has table copy " + ((table == null) ? "DISABLED" : "ENABLED")); // read all elements from the file into the copy table Log.logInfo("TABLE", "initializing RAM index for TABLE " + tablefile.getName() + ", please wait."); int i = 0; byte[] key; - if (table == null) { + if (this.table == null) { final Iterator ki = new ChunkIterator(tablefile, rowdef.objectsize, rowdef.primaryKeyLength); while (ki.hasNext()) { key = ki.next(); @@ -153,7 +153,7 @@ public class Table implements ObjectIndex, Iterable { assert key != null; if (key == null) {i++; continue;} if (rowdef.objectOrder.wellformed(key)) { - index.putUnique(key, i++); + this.index.putUnique(key, i++); } else { errors.putUnique(key, i++); } @@ -173,13 +173,13 @@ public class Table implements ObjectIndex, Iterable { index.putUnique(key, i++); // write the tail into the table try { - table.addUnique(taildef.newEntry(record, rowdef.primaryKeyLength, true)); + this.table.addUnique(taildef.newEntry(record, rowdef.primaryKeyLength, true)); } catch (RowSpaceExceededException e) { - table = null; + this.table = null; break; } if (abandonTable()) { - table = null; + this.table = null; break; } } else { @@ -201,7 +201,8 @@ public class Table implements ObjectIndex, Iterable { Log.logWarning("Table", "removing not well-formed entry " + idx + " with key: " + NaturalOrder.arrayList(key, 0, key.length) + ", " + errorcc++ + "/" + errorc); removeInFile(idx); } - assert index.size() == this.file.size() : "index.size() = " + index.size() + ", this.file.size() = " + this.file.size(); + errors.close(); + assert this.index.size() == this.file.size() : "index.size() = " + index.size() + ", this.file.size() = " + this.file.size(); // remove doubles if (!freshFile) { @@ -215,9 +216,9 @@ public class Table implements ObjectIndex, Iterable { final byte[] record = new byte[rowdef.objectsize]; key = new byte[rowdef.primaryKeyLength]; for (final Long[] ds: doubles) { - file.get(ds[0].intValue(), record, 0); + this.file.get(ds[0].intValue(), record, 0); System.arraycopy(record, 0, key, 0, rowdef.primaryKeyLength); - index.putUnique(key, ds[0].intValue()); + this.index.putUnique(key, ds[0].intValue()); } // then remove the other doubles by removing them from the table, but do a re-indexing while doing that // first aggregate all the delete positions because the elements from the top positions must be removed first @@ -247,6 +248,10 @@ public class Table implements ObjectIndex, Iterable { tableTracker.put(tablefile.toString(), this); } + public long mem() { + return index.mem() + ((table == null) ? 0 : table.mem()); + } + private boolean abandonTable() { // check if not enough memory is there to maintain a memory copy of the table return MemoryControl.available() < minmemremaining; @@ -427,7 +432,9 @@ public class Table implements ObjectIndex, Iterable { public void close() { this.file.close(); this.file = null; + if (this.table != null) this.table.close(); this.table = null; + if (this.index != null) this.index.close(); this.index = null; } @@ -783,7 +790,7 @@ public class Table implements ObjectIndex, Iterable { // initialize index and copy table table = (table == null) ? null : new RowSet(taildef); - index = new HandleMap(rowdef.primaryKeyLength, rowdef.objectOrder, 4, 100000); + index.clear(); } public Row row() {