boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

fix for http://forum.yacy-websuche.de/viewtopic.php?p=18500#p18500

Browse Source 
- catch uncatched OOM
- less wasting of memory

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6555 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 15 years ago
parent 32972139af
commit eeca2ded92
23 changed files with 142 additions and 79 deletions
Show Stats Download Patch File Download Diff File

2
source/de/anomic/crawler/ZURL.java
Unescape View File

@ -95,7 +95,7 @@ public class ZURL implements Iterable<ZURL.Entry> {
public ZURL() {
// creates a new ZUR in RAM
this.urlIndex = new RowSet(rowdef, 0);
this.urlIndex = new RowSet(rowdef);
this.stack = new ConcurrentLinkedQueue<String>();
}

2
source/de/anomic/data/URLAnalysis.java
Unescape View File

@ -416,7 +416,7 @@ public class URLAnalysis {
System.out.println("INDEX DIFF URL-COL startup");
HandleMap idx = new HandleMap(URIMetadataRow.rowdef.primaryKeyLength, URIMetadataRow.rowdef.objectOrder, 4, new File(statisticFile), 0);
MetadataRepository mr = new MetadataRepository(new File(metadataPath), "text.urlmd", false, false);
HandleSet hs = new HandleSet(URIMetadataRow.rowdef.primaryKeyLength, URIMetadataRow.rowdef.objectOrder, 0, 1000000);
HandleSet hs = new HandleSet(URIMetadataRow.rowdef.primaryKeyLength, URIMetadataRow.rowdef.objectOrder, 1000000, 0);
System.out.println("INDEX DIFF URL-COL loaded dump, starting diff");
long start = System.currentTimeMillis();
long update = start - 7000;

2
source/de/anomic/yacy/dht/FlatWordPartitionScheme.java
Unescape View File

@ -111,7 +111,7 @@ public class FlatWordPartitionScheme implements PartitionScheme {
System.gc(); // for resource measurement
long a = MemoryControl.available();
HandleMap idx = new HandleMap(12, Base64Order.enhancedCoder, 4, 0, 150000);
HandleMap idx = new HandleMap(12, Base64Order.enhancedCoder, 4, 150000);
for (int i = 0; i < count; i++) {
try {
idx.inc(FlatWordPartitionScheme.positionToHash(r.nextInt(count)));

7
source/de/anomic/yacy/yacyClient.java
Unescape View File

@ -538,7 +538,12 @@ public final class yacyClient {
assert words > 0 : "wordhashes = " + wordhashes;
final ReferenceContainer<WordReference>[] container = new ReferenceContainer[words];
for (int i = 0; i < words; i++) {
container[i] = ReferenceContainer.emptyContainer(Segment.wordReferenceFactory, wordhashes.substring(i * Word.commonHashLength, (i + 1) * Word.commonHashLength).getBytes(), count);
try {
container[i] = ReferenceContainer.emptyContainer(Segment.wordReferenceFactory, wordhashes.substring(i * Word.commonHashLength, (i + 1) * Word.commonHashLength).getBytes(), count);
} catch (RowSpaceExceededException e) {
Log.logException(e);
return null;
}
}
// insert results to containers

2
source/net/yacy/kelondro/blob/HeapReader.java
Unescape View File

@ -166,7 +166,7 @@ public class HeapReader {
Log.logInfo("HeapReader", "generating index for " + heapFile.toString() + ", " + (file.length() / 1024 / 1024) + " MB. Please wait.");
this.free = new Gap();
HandleMap.initDataConsumer indexready = HandleMap.asynchronusInitializer(keylength, this.ordering, 8, 0, Math.max(10, (int) (Runtime.getRuntime().freeMemory() / (10 * 1024 * 1024))));
HandleMap.initDataConsumer indexready = HandleMap.asynchronusInitializer(keylength, this.ordering, 8, Math.max(10, (int) (Runtime.getRuntime().freeMemory() / (10 * 1024 * 1024))));
byte[] key = new byte[keylength];
int reclen;
long seek = 0;

2
source/net/yacy/kelondro/blob/HeapWriter.java
Unescape View File

@ -78,7 +78,7 @@ public final class HeapWriter {
this.heapFileTMP = temporaryHeapFile;
this.heapFileREADY = readyHeapFile;
this.keylength = keylength;
this.index = new HandleMap(keylength, ordering, 8, 10, 100000);
this.index = new HandleMap(keylength, ordering, 8, 100000);
this.os = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(temporaryHeapFile), outBuffer));
//this.doublecheck = new HashSet<String>();
this.seek = 0;

4
source/net/yacy/kelondro/index/Cache.java
Unescape View File

@ -82,8 +82,8 @@ public final class Cache implements ObjectIndex, Iterable<Row.Entry> {
private void init() {
this.keyrow = new Row(new Column[]{index.row().column(0)}, index.row().objectOrder);
this.readHitCache = new RowSet(index.row(), 0);
this.readMissCache = new RowSet(this.keyrow, 0);
this.readHitCache = new RowSet(index.row());
this.readMissCache = new RowSet(this.keyrow);
this.readHit = 0;
this.readMiss = 0;
this.writeUnique = 0;

15
source/net/yacy/kelondro/index/HandleMap.java
Unescape View File

@ -62,9 +62,14 @@ public final class HandleMap implements Iterable<Row.Entry> {
* @param objectOrder
* @param space
*/
public HandleMap(final int keylength, final ByteOrder objectOrder, int idxbytes, final int initialspace, final int expectedspace) {
public HandleMap(final int keylength, final ByteOrder objectOrder, int idxbytes, final int expectedspace) {
this.rowdef = new Row(new Column[]{new Column("key", Column.celltype_binary, Column.encoder_bytes, keylength, "key"), new Column("long c-" + idxbytes + " {b256}")}, objectOrder);
this.index = new ObjectIndexCache(rowdef, initialspace, expectedspace);
this.index = new ObjectIndexCache(rowdef, expectedspace);
}
public HandleMap(final int keylength, final ByteOrder objectOrder, int idxbytes, final int expectedspace, final int initialspace) throws RowSpaceExceededException {
this.rowdef = new Row(new Column[]{new Column("key", Column.celltype_binary, Column.encoder_bytes, keylength, "key"), new Column("long c-" + idxbytes + " {b256}")}, objectOrder);
this.index = new ObjectIndexCache(rowdef, expectedspace, initialspace);
}
/**
@ -76,7 +81,7 @@ public final class HandleMap implements Iterable<Row.Entry> {
* @throws RowSpaceExceededException
*/
public HandleMap(final int keylength, final ByteOrder objectOrder, int idxbytes, final File file, final int expectedspace) throws IOException, RowSpaceExceededException {
this(keylength, objectOrder, idxbytes, (int) (file.length() / (keylength + idxbytes)), expectedspace);
this(keylength, objectOrder, idxbytes, expectedspace, (int) (file.length() / (keylength + idxbytes)));
// read the index dump and fill the index
InputStream is = new BufferedInputStream(new FileInputStream(file), 1024 * 1024);
if (file.getName().endsWith(".gz")) is = new GZIPInputStream(is);
@ -290,8 +295,8 @@ public final class HandleMap implements Iterable<Row.Entry> {
* @param bufferSize
* @return
*/
public final static initDataConsumer asynchronusInitializer(final int keylength, final ByteOrder objectOrder, int idxbytes, final int space, final int expectedspace) {
initDataConsumer initializer = new initDataConsumer(new HandleMap(keylength, objectOrder, idxbytes, space, expectedspace));
public final static initDataConsumer asynchronusInitializer(final int keylength, final ByteOrder objectOrder, int idxbytes, final int expectedspace) {
initDataConsumer initializer = new initDataConsumer(new HandleMap(keylength, objectOrder, idxbytes, expectedspace));
ExecutorService service = Executors.newSingleThreadExecutor();
initializer.setResult(service.submit(initializer));
service.shutdown();

11
source/net/yacy/kelondro/index/HandleSet.java
Unescape View File

@ -44,9 +44,14 @@ public final class HandleSet implements Iterable<byte[]> {
private final Row rowdef;
private ObjectIndex index;
public HandleSet(final int keylength, final ByteOrder objectOrder, final int initialspace, final int expectedspace) {
public HandleSet(final int keylength, final ByteOrder objectOrder, final int expectedspace) {
this.rowdef = new Row(new Column[]{new Column("key", Column.celltype_binary, Column.encoder_bytes, keylength, "key")}, objectOrder);
this.index = new ObjectIndexCache(rowdef, initialspace, expectedspace);
this.index = new ObjectIndexCache(rowdef, expectedspace);
}
public HandleSet(final int keylength, final ByteOrder objectOrder, final int expectedspace, final int initialspace) throws RowSpaceExceededException {
this.rowdef = new Row(new Column[]{new Column("key", Column.celltype_binary, Column.encoder_bytes, keylength, "key")}, objectOrder);
this.index = new ObjectIndexCache(rowdef, expectedspace, initialspace);
}
/**
@ -58,7 +63,7 @@ public final class HandleSet implements Iterable<byte[]> {
* @throws RowSpaceExceededException
*/
public HandleSet(final int keylength, final ByteOrder objectOrder, final File file, final int expectedspace) throws IOException, RowSpaceExceededException {
this(keylength, objectOrder, (int) (file.length() / (keylength + 8)), expectedspace);
this(keylength, objectOrder, expectedspace, (int) (file.length() / (keylength + 8)));
// read the index dump and fill the index
InputStream is = new BufferedInputStream(new FileInputStream(file), 1024 * 1024);
byte[] a = new byte[keylength];

7
source/net/yacy/kelondro/index/IndexTest.java
Unescape View File

@ -112,7 +112,12 @@ public class IndexTest {
System.out.println("sorted map");
Runtime.getRuntime().gc();
long freeStartKelondro = MemoryControl.available();
HandleMap ii = new HandleMap(12, Base64Order.enhancedCoder, 4, count, count);
HandleMap ii = null;
try {
ii = new HandleMap(12, Base64Order.enhancedCoder, 4, count, count);
} catch (RowSpaceExceededException e1) {
e1.printStackTrace();
}
for (int i = 0; i < count; i++)
try {
ii.putUnique(tests[i], 1);

23
source/net/yacy/kelondro/index/ObjectArrayCache.java
Unescape View File

@ -46,11 +46,16 @@ public final class ObjectArrayCache {
private RowSet index1;
//private final kelondroOrder<kelondroRow.Entry> entryOrder;
public ObjectArrayCache(final int payloadSize, final int initSize) {
this.rowdef = new Row("Cardinal key-4 {b256}, byte[] payload-" + payloadSize, NaturalOrder.naturalOrder);
this.index0 = new RowSet(rowdef, initSize);
this.index1 = null;
//this.entryOrder = new kelondroRow.EntryComparator(rowdef.objectOrder);
public ObjectArrayCache(final int payloadSize) {
this.rowdef = new Row("Cardinal key-4 {b256}, byte[] payload-" + payloadSize, NaturalOrder.naturalOrder);
this.index0 = new RowSet(rowdef);
this.index1 = null;
}
public ObjectArrayCache(final int payloadSize, final int initSize) throws RowSpaceExceededException {
this.rowdef = new Row("Cardinal key-4 {b256}, byte[] payload-" + payloadSize, NaturalOrder.naturalOrder);
this.index0 = new RowSet(rowdef, initSize);
this.index1 = null;
}
public final long memoryNeededForGrow() {
@ -71,7 +76,7 @@ public final class ObjectArrayCache {
// finish initialization phase
index0.sort();
index0.uniq();
index1 = new RowSet(rowdef, 0);
index1 = new RowSet(rowdef);
}
final Row.Entry indexentry = index0.get(key);
if (indexentry != null) return indexentry.getColBytes(1);
@ -133,7 +138,7 @@ public final class ObjectArrayCache {
// finish initialization phase
index0.sort();
index0.uniq();
index1 = new RowSet(rowdef, 0);
index1 = new RowSet(rowdef);
}
final Row.Entry indexentry = index0.remove(key);
if (indexentry != null) {
@ -187,7 +192,7 @@ public final class ObjectArrayCache {
// finish initialization phase
index0.sort();
index0.uniq();
index1 = new RowSet(rowdef, 0);
index1 = new RowSet(rowdef);
}
return index0.rows();
} else {
@ -219,7 +224,7 @@ public final class ObjectArrayCache {
int p, rc = 0;
final ArrayList<Long> ra = new ArrayList<Long>();
final HashSet<Long> jcontrol = new HashSet<Long>();
final ObjectArrayCache kcontrol = new ObjectArrayCache(1, 0);
final ObjectArrayCache kcontrol = new ObjectArrayCache(1);
for (int i = 0; i < 1000000; i++) {
r = Math.abs(random.nextLong() % 10000);
//System.out.println("add " + r);

41
source/net/yacy/kelondro/index/ObjectIndexCache.java
Unescape View File

@ -42,22 +42,35 @@ public final class ObjectIndexCache implements ObjectIndex, Iterable<Row.Entry>
private final Row.EntryComparator entryComparator;
private final int spread;
public ObjectIndexCache(final Row rowdef, final int initialspace, final int expectedspace) {
this.rowdef = rowdef;
this.entryComparator = new Row.EntryComparator(rowdef.objectOrder);
this.spread = Math.max(10, expectedspace / 3000);
reset(initialspace);
public ObjectIndexCache(final Row rowdef, final int expectedspace) {
this.rowdef = rowdef;
this.entryComparator = new Row.EntryComparator(rowdef.objectOrder);
this.spread = Math.max(10, expectedspace / 3000);
reset();
}
public ObjectIndexCache(final Row rowdef, final int expectedspace, final int initialspace) throws RowSpaceExceededException {
this.rowdef = rowdef;
this.entryComparator = new Row.EntryComparator(rowdef.objectOrder);
this.spread = Math.max(10, expectedspace / 3000);
reset(initialspace);
}
public void clear() {
reset(0);
reset();
}
public final synchronized void reset(final int initialspace) {
this.index0 = null; // first flush RAM to make room
this.index0 = new RowSet(rowdef, initialspace);
public final synchronized void reset() {
this.index0 = null; // first flush RAM to make room
this.index0 = new RowSet(rowdef);
this.index1 = null; // to show that this is the initialization phase
}
}
public final synchronized void reset(final int initialspace) throws RowSpaceExceededException {
this.index0 = null; // first flush RAM to make room
this.index0 = new RowSet(rowdef, initialspace);
this.index1 = null; // to show that this is the initialization phase
}
public final Row row() {
return index0.row();
@ -69,7 +82,7 @@ public final class ObjectIndexCache implements ObjectIndex, Iterable<Row.Entry>
index0.sort();
index0.uniq();
index0.trim(false);
index1 = new RowSetArray(rowdef, 0, spread);
index1 = new RowSetArray(rowdef, spread);
}
}
@ -230,7 +243,7 @@ public final class ObjectIndexCache implements ObjectIndex, Iterable<Row.Entry>
// finish initialization phase
index0.sort();
index0.uniq();
index1 = new RowSetArray(rowdef, 0, spread);
index1 = new RowSetArray(rowdef, spread);
return index0.keys(up, firstKey);
}
assert (index1 != null);
@ -259,7 +272,7 @@ public final class ObjectIndexCache implements ObjectIndex, Iterable<Row.Entry>
// finish initialization phase
index0.sort();
index0.uniq();
index1 = new RowSetArray(rowdef, 0, spread);
index1 = new RowSetArray(rowdef, spread);
return index0.rows(up, firstKey);
}
assert (index1 != null);
@ -293,7 +306,7 @@ public final class ObjectIndexCache implements ObjectIndex, Iterable<Row.Entry>
// finish initialization phase
index0.sort();
index0.uniq();
index1 = new RowSetArray(rowdef, 0, spread);
index1 = new RowSetArray(rowdef, spread);
return index0.rows();
}
assert (index1 != null);

12
source/net/yacy/kelondro/index/RowCollection.java
Unescape View File

@ -79,12 +79,17 @@ public class RowCollection implements Iterable<Row.Entry> {
this.lastTimeWrote = rc.lastTimeWrote;
}
public RowCollection(final Row rowdef, final int objectCount) {
public RowCollection(final Row rowdef) {
this.rowdef = rowdef;
this.chunkcache = new byte[objectCount * rowdef.objectsize];
this.chunkcount = 0;
this.sortBound = 0;
this.lastTimeWrote = System.currentTimeMillis();
this.chunkcache = new byte[0];
this.chunkcount = 0;
}
public RowCollection(final Row rowdef, final int objectCount) throws RowSpaceExceededException {
this(rowdef);
ensureSize(objectCount);
}
public RowCollection(final Row rowdef, final int objectCount, final byte[] cache, final int sortBound) {
@ -218,6 +223,7 @@ public class RowCollection implements Iterable<Row.Entry> {
}
protected final void ensureSize(final int elements) throws RowSpaceExceededException {
if (elements == 0) return;
long allocram = neededSpaceForEnsuredSize(elements, true);
if (allocram == 0) return;
assert allocram > chunkcache.length : "wrong alloc computation: allocram = " + allocram + ", chunkcache.length = " + chunkcache.length;

19
source/net/yacy/kelondro/index/RowSet.java
Unescape View File

@ -46,11 +46,16 @@ public class RowSet extends RowCollection implements ObjectIndex, Iterable<Row.E
assert rowdef.objectOrder != null;
}
public RowSet(final Row rowdef, final int objectCount) {
public RowSet(final Row rowdef, final int objectCount) throws RowSpaceExceededException {
super(rowdef, objectCount);
assert rowdef.objectOrder != null;
}
public RowSet(final Row rowdef) {
super(rowdef);
assert rowdef.objectOrder != null;
}
/**
* import an exported collection
* @param rowdef
@ -64,16 +69,16 @@ public class RowSet extends RowCollection implements ObjectIndex, Iterable<Row.E
public final static RowSet importRowSet(byte[] b, final Row rowdef) {
assert b.length >= exportOverheadSize : "b.length = " + b.length;
if (b.length < exportOverheadSize) return new RowSet(rowdef, 0);
if (b.length < exportOverheadSize) return new RowSet(rowdef);
final int size = (int) NaturalOrder.decodeLong(b, 0, 4);
assert size >= 0 : "size = " + size;
if (size < 0) return new RowSet(rowdef, 0);
if (size < 0) return new RowSet(rowdef);
final int orderbound = (int) NaturalOrder.decodeLong(b, 10, 4);
assert orderbound >= 0 : "orderbound = " + orderbound;
if (orderbound < 0) return new RowSet(rowdef, 0);
if (orderbound < 0) return new RowSet(rowdef);
final byte[] chunkcache = new byte[size * rowdef.objectsize];
assert b.length - exportOverheadSize == size * rowdef.objectsize;
if (b.length - exportOverheadSize != size * rowdef.objectsize) return new RowSet(rowdef, 0);
if (b.length - exportOverheadSize != size * rowdef.objectsize) return new RowSet(rowdef);
System.arraycopy(b, exportOverheadSize, chunkcache, 0, chunkcache.length);
return new RowSet(rowdef, size, chunkcache, orderbound);
}
@ -486,7 +491,7 @@ public class RowSet extends RowCollection implements ObjectIndex, Iterable<Row.E
"acht......xxxx",
"neun......xxxx",
"zehn......xxxx" };
final RowSet d = new RowSet(new Row("byte[] key-10, Cardinal x-4 {b256}", NaturalOrder.naturalOrder), 0);
final RowSet d = new RowSet(new Row("byte[] key-10, Cardinal x-4 {b256}", NaturalOrder.naturalOrder));
for (int ii = 0; ii < test.length; ii++)
try {
d.add(test[ii].getBytes());
@ -583,7 +588,7 @@ public class RowSet extends RowCollection implements ObjectIndex, Iterable<Row.E
// remove test
final long start = System.currentTimeMillis();
final RowSet c = new RowSet(new Row("byte[] a-12, byte[] b-12", Base64Order.enhancedCoder), 0);
final RowSet c = new RowSet(new Row("byte[] a-12, byte[] b-12", Base64Order.enhancedCoder));
byte[] key;
final int testsize = 5000;
final byte[][] delkeys = new byte[testsize / 5][];

8
source/net/yacy/kelondro/index/RowSetArray.java
Unescape View File

@ -34,17 +34,15 @@ import net.yacy.kelondro.order.StackIterator;
public final class RowSetArray implements ObjectIndex, Iterable<Row.Entry> {
private final int objectCount;
private final Row rowdef;
private final RowSet[] array;
public RowSetArray(final Row rowdef, final int objectCount, final int arraySize) {
public RowSetArray(final Row rowdef, final int arraySize) {
this.array = new RowSet[arraySize];
this.rowdef = rowdef;
for (int i = 0; i < arraySize; i++) {
this.array[i] = null;
}
this.rowdef = rowdef;
this.objectCount = objectCount / arraySize;
}
private final int indexFor(byte[] key) {
@ -58,7 +56,7 @@ public final class RowSetArray implements ObjectIndex, Iterable<Row.Entry> {
private final RowSet accessArray(int i) {
RowSet r = this.array[i];
if (r == null) synchronized (this.array) {
r = new RowSet(this.rowdef, this.objectCount);
r = new RowSet(this.rowdef);
this.array[i] = r;
}
return r;

2
source/net/yacy/kelondro/rwi/AbstractBufferedIndex.java
Unescape View File

@ -46,7 +46,7 @@ public abstract class AbstractBufferedIndex<ReferenceType extends Reference> ext
// this does not use the cache
final Order<ReferenceContainer<ReferenceType>> containerOrder = new ReferenceContainerOrder<ReferenceType>(factory, this.ordering().clone());
if (startHash.length == 0) startHash = null;
ReferenceContainer<ReferenceType> emptyContainer = ReferenceContainer.emptyContainer(factory, startHash, 0);
ReferenceContainer<ReferenceType> emptyContainer = ReferenceContainer.emptyContainer(factory, startHash);
containerOrder.rotate(emptyContainer);
final TreeSet<ReferenceContainer<ReferenceType>> containers = new TreeSet<ReferenceContainer<ReferenceType>>(containerOrder);
final Iterator<ReferenceContainer<ReferenceType>> i = references(startHash, rot, ram);

2
source/net/yacy/kelondro/rwi/AbstractIndex.java
Unescape View File

@ -60,7 +60,7 @@ public abstract class AbstractIndex <ReferenceType extends Reference> implements
// creates a set of indexContainers
// this does not use the cache
final Order<ReferenceContainer<ReferenceType>> containerOrder = new ReferenceContainerOrder<ReferenceType>(factory, this.ordering().clone());
ReferenceContainer<ReferenceType> emptyContainer = ReferenceContainer.emptyContainer(factory, startHash, 0);
ReferenceContainer<ReferenceType> emptyContainer = ReferenceContainer.emptyContainer(factory, startHash);
containerOrder.rotate(emptyContainer);
final TreeSet<ReferenceContainer<ReferenceType>> containers = new TreeSet<ReferenceContainer<ReferenceType>>(containerOrder);
final Iterator<ReferenceContainer<ReferenceType>> i = references(startHash, rot);

4
source/net/yacy/kelondro/rwi/IndexCell.java
Unescape View File

@ -295,7 +295,7 @@ public final class IndexCell<ReferenceType extends Reference> extends AbstractBu
public CloneableIterator<ReferenceContainer<ReferenceType>> references(byte[] starttermHash, boolean rot) {
final Order<ReferenceContainer<ReferenceType>> containerOrder = new ReferenceContainerOrder<ReferenceType>(factory, this.ram.rowdef().getOrdering().clone());
containerOrder.rotate(new ReferenceContainer<ReferenceType>(factory, starttermHash, 0));
containerOrder.rotate(new ReferenceContainer<ReferenceType>(factory, starttermHash));
return new MergeIterator<ReferenceContainer<ReferenceType>>(
this.ram.references(starttermHash, rot),
new MergeIterator<ReferenceContainer<ReferenceType>>(
@ -311,7 +311,7 @@ public final class IndexCell<ReferenceType extends Reference> extends AbstractBu
public CloneableIterator<ReferenceContainer<ReferenceType>> references(byte[] startTermHash, boolean rot, boolean ram) {
final Order<ReferenceContainer<ReferenceType>> containerOrder = new ReferenceContainerOrder<ReferenceType>(factory, this.ram.rowdef().getOrdering().clone());
containerOrder.rotate(new ReferenceContainer<ReferenceType>(factory, startTermHash, 0));
containerOrder.rotate(new ReferenceContainer<ReferenceType>(factory, startTermHash));
if (ram) {
return this.ram.references(startTermHash, rot);
}

19
source/net/yacy/kelondro/rwi/ReferenceContainer.java
Unescape View File

@ -62,7 +62,15 @@ public class ReferenceContainer<ReferenceType extends Reference> extends RowSet
this.termHash = termHash;
}
public ReferenceContainer(final ReferenceFactory<ReferenceType> factory, final byte[] termHash, final int objectCount) {
public ReferenceContainer(final ReferenceFactory<ReferenceType> factory, final byte[] termHash) {
super(factory.getRow());
assert termHash == null || (termHash[2] != '@' && termHash.length == this.rowdef.primaryKeyLength);
this.termHash = termHash;
this.factory = factory;
this.lastTimeWrote = 0;
}
public ReferenceContainer(final ReferenceFactory<ReferenceType> factory, final byte[] termHash, final int objectCount) throws RowSpaceExceededException {
super(factory.getRow(), objectCount);
assert termHash == null || (termHash[2] != '@' && termHash.length == this.rowdef.primaryKeyLength);
this.termHash = termHash;
@ -76,8 +84,13 @@ public class ReferenceContainer<ReferenceType extends Reference> extends RowSet
return newContainer;
}
public static <ReferenceType extends Reference> ReferenceContainer<ReferenceType> emptyContainer(final ReferenceFactory<ReferenceType> factory, final byte[] termHash, final int elementCount) {
assert termHash == null || (termHash[2] != '@' && termHash.length == factory.getRow().primaryKeyLength);
public static <ReferenceType extends Reference> ReferenceContainer<ReferenceType> emptyContainer(final ReferenceFactory<ReferenceType> factory, final byte[] termHash) {
assert termHash == null || (termHash[2] != '@' && termHash.length == factory.getRow().primaryKeyLength);
return new ReferenceContainer<ReferenceType>(factory, termHash);
}
public static <ReferenceType extends Reference> ReferenceContainer<ReferenceType> emptyContainer(final ReferenceFactory<ReferenceType> factory, final byte[] termHash, final int elementCount) throws RowSpaceExceededException {
assert termHash == null || (termHash[2] != '@' && termHash.length == factory.getRow().primaryKeyLength);
return new ReferenceContainer<ReferenceType>(factory, termHash, elementCount);
}

2
source/net/yacy/kelondro/rwi/ReferenceContainerArray.java
Unescape View File

@ -329,7 +329,7 @@ public final class ReferenceContainerArray<ReferenceType extends Reference> {
final Row payloadrow) throws IOException, RowSpaceExceededException {
System.out.println("CELL REFERENCE COLLECTION startup");
HandleMap references = new HandleMap(payloadrow.primaryKeyLength, termOrder, 4, 0, 1000000);
HandleMap references = new HandleMap(payloadrow.primaryKeyLength, termOrder, 4, 1000000, 0);
String[] files = heapLocation.list();
for (String f: files) {
if (f.length() < 22 || !f.startsWith("text.index") || !f.endsWith(".blob")) continue;

25
source/net/yacy/kelondro/rwi/ReferenceContainerCache.java
Unescape View File

@ -282,25 +282,28 @@ public final class ReferenceContainerCache<ReferenceType extends Reference> exte
* get a indexContainer from a heap
* @param key
* @return the indexContainer if one exist, null otherwise
* @throws
*/
public ReferenceContainer<ReferenceType> get(final byte[] key, Set<String> urlselection) {
ReferenceContainer<ReferenceType> c = this.cache.get(new ByteArray(key));
if (urlselection == null) return c;
if (c == null) return null;
// because this is all in RAM, we must clone the entries (flat)
ReferenceContainer<ReferenceType> c1 = new ReferenceContainer<ReferenceType>(factory, c.getTermHash(), c.size());
Iterator<ReferenceType> e = c.entries();
ReferenceType ee;
while (e.hasNext()) {
ee = e.next();
if (urlselection.contains(ee.metadataHash())) try {
c1.add(ee);
} catch (RowSpaceExceededException e1) {
Log.logException(e1);
break;
try {
ReferenceContainer<ReferenceType> c1 = new ReferenceContainer<ReferenceType>(factory, c.getTermHash(), c.size());
Iterator<ReferenceType> e = c.entries();
ReferenceType ee;
while (e.hasNext()) {
ee = e.next();
if (urlselection.contains(ee.metadataHash())) {
c1.add(ee);
}
}
return c1;
} catch (RowSpaceExceededException e2) {
Log.logException(e2);
}
return c1;
return null;
}
/**

2
source/net/yacy/kelondro/table/SplitTable.java
Unescape View File

@ -68,7 +68,7 @@ public class SplitTable implements ObjectIndex, Iterable<Row.Entry> {
// the table type can be either kelondroFlex or kelondroEco
private static final int EcoFSBufferSize = 20;
static final ObjectIndex dummyIndex = new ObjectIndexCache(new Row(new Column[]{new Column("key", Column.celltype_binary, Column.encoder_bytes, 2, "key")}, NaturalOrder.naturalOrder), 0, 0);
static final ObjectIndex dummyIndex = new ObjectIndexCache(new Row(new Column[]{new Column("key", Column.celltype_binary, Column.encoder_bytes, 2, "key")}, NaturalOrder.naturalOrder), 0);
// the thread pool for the keeperOf executor service
private ExecutorService executor;

8
source/net/yacy/kelondro/table/Table.java
Unescape View File

@ -135,8 +135,8 @@ public class Table implements ObjectIndex, Iterable<Row.Entry> {
table = null; System.gc();
Log.logSevere("TABLE", tablefile.getName() + ": RAM after releasing the table: " + (MemoryControl.available() / 1024 / 1024) + "MB");
}
index = new HandleMap(rowdef.primaryKeyLength, rowdef.objectOrder, 4, records, 100000);
HandleMap errors = new HandleMap(rowdef.primaryKeyLength, NaturalOrder.naturalOrder, 4, records, 10);
index = new HandleMap(rowdef.primaryKeyLength, rowdef.objectOrder, 4, records);
HandleMap errors = new HandleMap(rowdef.primaryKeyLength, NaturalOrder.naturalOrder, 4, records);
Log.logInfo("TABLE", tablefile + ": TABLE " + tablefile.toString() + " has table copy " + ((table == null) ? "DISABLED" : "ENABLED"));
// read all elements from the file into the copy table
@ -695,8 +695,8 @@ public class Table implements ObjectIndex, Iterable<Row.Entry> {
}
// initialize index and copy table
table = (table == null) ? null : new RowSet(taildef, 1);
index = new HandleMap(rowdef.primaryKeyLength, rowdef.objectOrder, 4, 1, 100000);
table = (table == null) ? null : new RowSet(taildef);
index = new HandleMap(rowdef.primaryKeyLength, rowdef.objectOrder, 4, 100000);
}
public Row row() {

Write Preview
Loading…
Cancel
Save