boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

- set status of kelondroTree, kelondroBLOBTree and kelondroFlexTable to deprecated

Browse Source 
- removed initialization and/or usage of kelondroFlexTable (should meanwhile not be used any more)

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5396 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 17 years ago
parent 7535fd7447
commit 8cb7170b75
8 changed files with 68 additions and 178 deletions
Show Stats Download Patch File Download Diff File

10
source/dbtest.java
Unescape View File

@ -13,10 +13,8 @@ import java.util.Random;
import javax.imageio.ImageIO;
import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroCache;
import de.anomic.kelondro.kelondroCloneableIterator;
import de.anomic.kelondro.kelondroEcoTable;
import de.anomic.kelondro.kelondroFlexTable;
import de.anomic.kelondro.kelondroIndex;
import de.anomic.kelondro.kelondroIntBytesMap;
import de.anomic.kelondro.kelondroProfile;
@ -198,14 +196,6 @@ public class dbtest {
if (dbe.equals("kelondroRowSet")) {
return new kelondroRowSet(testRow, 0);
}
if (dbe.equals("kelondroTree")) {
final File tablefile = new File(tablename + ".kelondro.db");
return new kelondroCache(new kelondroTree(tablefile, true, 0, testRow));
}
if (dbe.equals("kelondroFlexTable")) {
final File tablepath = new File(tablename).getParentFile();
return new kelondroFlexTable(tablepath, new File(tablename).getName(), testRow, 0, true);
}
if (dbe.equals("kelondroSplitTable")) {
final File tablepath = new File(tablename).getParentFile();
return new kelondroSplitTable(tablepath, new File(tablename).getName(), testRow, true);

4
source/de/anomic/kelondro/kelondroBLOBTree.java
Unescape View File

@ -55,6 +55,10 @@ public class kelondroBLOBTree implements kelondroBLOB {
private final kelondroRow rowdef;
private File file;
/**
* Deprecated Class. Please use kelondroBLOBHeap instead
*/
@Deprecated
public kelondroBLOBTree(final File file, final boolean useNodeCache, final boolean useObjectCache, final int key,
final int nodesize, final char fillChar, final kelondroByteOrder objectOrder, final boolean usetree, final boolean writebuffer, final boolean resetOnFail) {
// creates or opens a dynamic tree

60
source/de/anomic/kelondro/kelondroCollectionIndex.java
Unescape View File

@ -105,12 +105,6 @@ public class kelondroCollectionIndex {
final String sn = fillZ(Integer.toHexString(serialNumber).toUpperCase(), 2);
return new File(path, filenameStub + "." + lf + "." + cs + "." + pn + "." + sn + ".kca"); // kelondro collection array
}
private static File propertyFile(final File path, final String filenameStub, final int loadfactor, final int chunksize) {
final String lf = fillZ(Integer.toHexString(loadfactor).toUpperCase(), 2);
final String cs = fillZ(Integer.toHexString(chunksize).toUpperCase(), 4);
return new File(path, filenameStub + "." + lf + "." + cs + ".properties");
}
public kelondroCollectionIndex(final File path, final String filenameStub, final int keyLength, final kelondroByteOrder indexOrder,
final int loadfactor, final int maxpartitions, final kelondroRow rowdef, boolean useCommons) throws IOException {
@ -130,7 +124,9 @@ public class kelondroCollectionIndex {
this.commonsPath.mkdirs();
}
final File f = new File(path, filenameStub + ".index");
if (f.isDirectory()) {
kelondroFlexTable.delete(path, filenameStub + ".index");
}
if (f.exists()) {
serverLog.logFine("STARTUP", "OPENING COLLECTION INDEX");
@ -161,16 +157,9 @@ public class kelondroCollectionIndex {
serverLog.logFine("STARTUP", "STARTED INITIALIZATION OF NEW COLLECTION INDEX WITH " + initialSpace + " ENTRIES. THIS WILL TAKE SOME TIME. " + (serverMemory.available() / 1024 / 1024) + "MB AVAILABLE.");
final kelondroRow indexRowdef = indexRow(keyLength, indexOrder);
final long necessaryRAM4fullTable = minimumRAM4Eco + (indexRowdef.objectsize + 4) * initialSpace * 3 / 2;
final long necessaryRAM4fullIndex = minimumRAM4Eco + (indexRowdef.primaryKeyLength + 4) * initialSpace * 3 / 2;
// initialize (new generation) index table from file
if (serverMemory.request(necessaryRAM4fullTable, false)) {
index = new kelondroEcoTable(f, indexRowdef, kelondroEcoTable.tailCacheUsageAuto, EcoFSBufferSize, initialSpace);
} else if (serverMemory.request(necessaryRAM4fullIndex, false)) {
index = new kelondroEcoTable(f, indexRowdef, kelondroEcoTable.tailCacheDenyUsage, EcoFSBufferSize, initialSpace);
} else {
index = new kelondroFlexTable(path, filenameStub + ".index", indexRowdef, initialSpace, true);
}
index = new kelondroEcoTable(f, indexRowdef, (serverMemory.request(necessaryRAM4fullTable, false)) ? kelondroEcoTable.tailCacheUsageAuto : kelondroEcoTable.tailCacheDenyUsage, EcoFSBufferSize, initialSpace);
// open array files
this.arrays = new HashMap<String, kelondroFixedWidthArray>(); // all entries will be dynamically created with getArray()
@ -283,35 +272,18 @@ public class kelondroCollectionIndex {
final kelondroRow indexRowdef = indexRow(keylength, indexOrder);
kelondroIndex theindex;
if (f.isDirectory()) {
// use a flextable
theindex = new kelondroCache(new kelondroFlexTable(path, filenameStub + ".index", indexRowdef, initialSpace, true));
// save/check property file for this array
final File propfile = propertyFile(path, filenameStub, loadfactor, rowdef.objectsize);
Map<String, String> props = new HashMap<String, String>();
if (propfile.exists()) {
props = serverFileUtils.loadHashMap(propfile);
final String stored_rowdef = props.get("rowdef");
if ((rowdef != null) && (!(rowdef.subsumes(new kelondroRow(stored_rowdef, rowdef.objectOrder, 0))))) {
System.out.println("FATAL ERROR: stored rowdef '" + stored_rowdef + "' does not match with new rowdef '" +
rowdef + "' for array cluster '" + path + "/" + filenameStub + "'");
System.exit(-1);
}
}
props.put("rowdef", rowdef.toString());
serverFileUtils.saveMap(propfile, props, "CollectionIndex properties");
kelondroFlexTable.delete(path, filenameStub + ".index");
}
// open a ecotable
final long records = f.length() / indexRowdef.objectsize;
final long necessaryRAM4fullTable = minimumRAM4Eco + (indexRowdef.objectsize + 4) * records * 3 / 2;
final boolean fullCache = serverMemory.request(necessaryRAM4fullTable, false);
if (fullCache) {
theindex = new kelondroEcoTable(f, indexRowdef, kelondroEcoTable.tailCacheUsageAuto, EcoFSBufferSize, initialSpace);
//if (!((kelondroEcoTable) theindex).usesFullCopy()) theindex = new kelondroCache(theindex);
} else {
// open a ecotable
final long records = f.length() / indexRowdef.objectsize;
final long necessaryRAM4fullTable = minimumRAM4Eco + (indexRowdef.objectsize + 4) * records * 3 / 2;
final boolean fullCache = serverMemory.request(necessaryRAM4fullTable, false);
if (fullCache) {
theindex = new kelondroEcoTable(f, indexRowdef, kelondroEcoTable.tailCacheUsageAuto, EcoFSBufferSize, initialSpace);
//if (!((kelondroEcoTable) theindex).usesFullCopy()) theindex = new kelondroCache(theindex);
} else {
//theindex = new kelondroCache(new kelondroEcoTable(f, indexRowdef, kelondroEcoTable.tailCacheDenyUsage, EcoFSBufferSize, initialSpace));
theindex = new kelondroEcoTable(f, indexRowdef, kelondroEcoTable.tailCacheDenyUsage, EcoFSBufferSize, initialSpace);
}
//theindex = new kelondroCache(new kelondroEcoTable(f, indexRowdef, kelondroEcoTable.tailCacheDenyUsage, EcoFSBufferSize, initialSpace));
theindex = new kelondroEcoTable(f, indexRowdef, kelondroEcoTable.tailCacheDenyUsage, EcoFSBufferSize, initialSpace);
}
return theindex;
}
@ -918,7 +890,7 @@ public class kelondroCollectionIndex {
// printout of index
collectionIndex.close();
final kelondroFlexTable index = new kelondroFlexTable(path, filenameStub + ".index", kelondroCollectionIndex.indexRow(9, kelondroNaturalOrder.naturalOrder), 0, true);
final kelondroEcoTable index = new kelondroEcoTable(new File(path, filenameStub + ".index"), kelondroCollectionIndex.indexRow(9, kelondroNaturalOrder.naturalOrder), kelondroEcoTable.tailCacheUsageAuto, 0, 0);
index.print();
index.close();
} catch (final IOException e) {

14
source/de/anomic/kelondro/kelondroEcoTable.java
Unescape View File

@ -676,6 +676,20 @@ public class kelondroEcoTable implements kelondroIndex {
}
}
public void print() throws IOException {
System.out.println("PRINTOUT of table, length=" + size());
Entry row;
byte[] key;
kelondroCloneableIterator<byte[]> i = keys(true, null);
while (i.hasNext()) {
System.out.print("row " + i + ": ");
key = i.next();
row = get(key);
System.out.println(row.toString());
}
System.out.println("EndOfTable");
}
public static void main(final String[] args) {
// open a file, add one entry and exit
final File f = new File(args[0]);

82
source/de/anomic/kelondro/kelondroFlexTable.java
Unescape View File

@ -47,6 +47,10 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr
protected kelondroBytesIntMap index;
private boolean RAMIndex;
/**
* Deprecated Class. Please use kelondroEcoTable instead
*/
@Deprecated
public kelondroFlexTable(final File path, final String tablename, final kelondroRow rowdef, int minimumSpace, final boolean resetOnFail) {
// the buffersize applies to a possible load of the ram-index
// the minimumSpace is a initial allocation space for the index; names the number of index slots
@ -72,43 +76,21 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr
System.out.println("*** Last Startup time: " + stt + " milliseconds");
final long start = System.currentTimeMillis();
if (serverMemory.request(neededRAM, false)) {
// we can use a RAM index
if (indexfile.exists()) {
// delete existing index file
System.out.println("*** Delete File index " + indexfile);
indexfile.delete();
}
// fill the index
System.out.print("*** Loading RAM index for " + size() + " entries from " + newpath + "; available RAM = " + (serverMemory.available() >> 20) + " MB, allocating " + (neededRAM >> 20) + " MB for index.");
index = initializeRamIndex(minimumSpace);
// we use a RAM index
if (indexfile.exists()) {
// delete existing index file
System.out.println("*** Delete File index " + indexfile);
indexfile.delete();
}
System.out.println(" -done-");
System.out.println(index.size() + " index entries initialized and sorted from " + super.col[0].size() + " keys.");
RAMIndex = true;
tableTracker.put(this.filename(), this);
} else {
// too less ram for a ram index
kelondroIndex ki;
if (indexfile.exists()) {
// use existing index file
System.out.println("*** Using File index " + indexfile);
ki = new kelondroCache(kelondroTree.open(indexfile, true, 0, treeIndexRow(rowdef.width(0), rowdef.objectOrder), 2, 80));
RAMIndex = false;
} else {
// generate new index file
System.out.println("*** Generating File index for " + size() + " entries from " + indexfile);
System.out.println("*** Cause: too less RAM (" + serverMemory.available() + " Bytes) configured. Assign at least " + (neededRAM / 1024 / 1024) + " MB more RAM to enable a RAM index.");
ki = initializeTreeIndex(indexfile, 0, rowdef.objectOrder);
// fill the index
System.out.print("*** Loading RAM index for " + size() + " entries from " + newpath + "; available RAM = " + (serverMemory.available() >> 20) + " MB, allocating " + (neededRAM >> 20) + " MB for index.");
index = initializeRamIndex(minimumSpace);
System.out.println(" -done-");
System.out.println(ki.size() + " entries indexed from " + super.col[0].size() + " keys.");
RAMIndex = false;
}
index = new kelondroBytesIntMap(ki);
assert this.size() == index.size() : "content.size() = " + this.size() + ", index.size() = " + index.size();
}
System.out.println(" -done-");
System.out.println(index.size() + " index entries initialized and sorted from " + super.col[0].size() + " keys.");
RAMIndex = true;
tableTracker.put(this.filename(), this);
// check consistency
final ArrayList<Integer[]> doubles = index.removeDoubles();
@ -185,36 +167,6 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr
return ri;
}
private kelondroIndex initializeTreeIndex(final File indexfile, final long preloadTime, final kelondroByteOrder objectOrder) throws IOException {
final kelondroIndex treeindex = new kelondroCache(new kelondroTree(indexfile, true, preloadTime, treeIndexRow(rowdef.primaryKeyLength, objectOrder), 2, 80));
final Iterator<kelondroNode> content = super.col[0].contentNodes(-1);
kelondroNode node;
kelondroRow.Entry indexentry;
int i, c = 0;
final int all = super.col[0].size();
final long start = System.currentTimeMillis();
long last = start;
while (content.hasNext()) {
node = content.next();
i = node.handle().hashCode();
indexentry = treeindex.row().newEntry();
indexentry.setCol(0, node.getValueRow());
indexentry.setCol(1, i);
treeindex.addUnique(indexentry);
c++;
if (System.currentTimeMillis() - last > 30000) {
System.out.println(".. generated " + c + "/" + all + " entries, " + ((System.currentTimeMillis() - start) / c * (all - c) / 60000) + " minutes remaining");
System.out.flush();
last = System.currentTimeMillis();
}
}
return treeindex;
}
private static final kelondroRow treeIndexRow(final int keywidth, final kelondroByteOrder objectOrder) {
return new kelondroRow("byte[] key-" + keywidth + ", int reference-4 {b256}", objectOrder, 0);
}
public synchronized kelondroRow.Entry get(final byte[] key) throws IOException {
if (index == null) return null; // case may happen during shutdown
final int pos = index.geti(key);

35
source/de/anomic/kelondro/kelondroSplitTable.java
Unescape View File

@ -47,7 +47,6 @@ import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import de.anomic.server.NamePrefixThreadFactory;
import de.anomic.server.serverMemory;
import de.anomic.server.serverProcessor;
import de.anomic.server.logging.serverLog;
@ -57,7 +56,6 @@ public class kelondroSplitTable implements kelondroIndex {
// the set is divided into tables with different entry date
// the table type can be either kelondroFlex or kelondroEco
private static final long minimumRAM4Eco = 80 * 1024 * 1024;
private static final int EcoFSBufferSize = 20;
static final kelondroIndex dummyIndex = new kelondroRAMIndex(new kelondroRow(new kelondroColumn[]{new kelondroColumn("key", kelondroColumn.celltype_binary, kelondroColumn.encoder_bytes, 2, "key")}, kelondroNaturalOrder.naturalOrder, 0), 0);
@ -137,12 +135,11 @@ public class kelondroSplitTable implements kelondroIndex {
f = new File(path, maxf);
if (f.isDirectory()) {
// this is a kelonodroFlex table
serverLog.logInfo("kelondroSplitTable", "opening partial flex table " + f);
table = new kelondroFlexTable(path, maxf, rowdef, 0, resetOnFail);
} else {
serverLog.logInfo("kelondroSplitTable", "opening partial eco table " + f);
table = new kelondroEcoTable(f, rowdef, kelondroEcoTable.tailCacheUsageAuto, EcoFSBufferSize, 0);
kelondroFlexTable.delete(path, maxf);
serverLog.logInfo("kelondroSplitTable", "replaced partial flex table " + f + " by new eco table");
}
serverLog.logInfo("kelondroSplitTable", "opening partial eco table " + f);
table = new kelondroEcoTable(f, rowdef, kelondroEcoTable.tailCacheUsageAuto, EcoFSBufferSize, 0);
tables.put(date, table);
}
}
@ -239,21 +236,13 @@ public class kelondroSplitTable implements kelondroIndex {
final File f = new File(path, tablename + "." + suffix);
if (f.exists()) {
if (f.isDirectory()) {
// open a flex table
table = new kelondroFlexTable(path, tablename + "." + suffix, rowdef, 0, true);
} else {
// open a eco table
table = new kelondroEcoTable(f, rowdef, kelondroEcoTable.tailCacheDenyUsage, EcoFSBufferSize, 0);
kelondroFlexTable.delete(path, tablename + "." + suffix);
}
// open a eco table
table = new kelondroEcoTable(f, rowdef, kelondroEcoTable.tailCacheDenyUsage, EcoFSBufferSize, 0);
} else {
// make new table
if (serverMemory.request(minimumRAM4Eco, true)) {
// enough memory for a ecoTable
table = new kelondroEcoTable(f, rowdef, kelondroEcoTable.tailCacheDenyUsage, EcoFSBufferSize, 0);
} else {
// use the flex table
table = new kelondroFlexTable(path, tablename + "." + suffix, rowdef, 0, true);
}
table = new kelondroEcoTable(f, rowdef, kelondroEcoTable.tailCacheDenyUsage, EcoFSBufferSize, 0);
}
tables.put(suffix, table);
}
@ -320,13 +309,7 @@ public class kelondroSplitTable implements kelondroIndex {
kelondroIndex table = tables.get(suffix);
if (table == null) {
// make new table
if (serverMemory.request(minimumRAM4Eco, true)) {
// enough memory for a ecoTable
table = new kelondroEcoTable(new File(path, tablename + "." + suffix), rowdef, kelondroEcoTable.tailCacheDenyUsage, EcoFSBufferSize, 0);
} else {
// use the flex table
table = new kelondroFlexTable(path, tablename + "." + suffix, rowdef, 0, true);
}
table = new kelondroEcoTable(new File(path, tablename + "." + suffix), rowdef, kelondroEcoTable.tailCacheDenyUsage, EcoFSBufferSize, 0);
tables.put(suffix, table);
}
table.addUnique(row);

37
source/de/anomic/kelondro/kelondroTree.java
Unescape View File

@ -74,10 +74,15 @@ public class kelondroTree extends kelondroCachedRecords implements kelondroIndex
protected int readAheadChunkSize = 100;
protected long lastIteratorCount = readAheadChunkSize;
/**
* Deprecated Class. Please use kelondroEcoTable instead
*/
@Deprecated
public kelondroTree(final File file, final boolean useNodeCache, final long preloadTime, final kelondroRow rowdef) throws IOException {
this(file, useNodeCache, preloadTime, rowdef, rowdef.columns() /* txtProps */, 80 /* txtPropWidth */);
}
@Deprecated
public kelondroTree(final File file, final boolean useNodeCache, final long preloadTime, final kelondroRow rowdef, final int txtProps, final int txtPropsWidth) throws IOException {
// opens an existing tree file or creates a new tree file
super(file, useNodeCache, preloadTime,
@ -91,10 +96,7 @@ public class kelondroTree extends kelondroCachedRecords implements kelondroIndex
this.loopDetectionOrder = new kelondroByteOrder.StringOrder(rowdef.objectOrder);
}
public static final kelondroTree open(final File file, final boolean useNodeCache, final long preloadTime, final kelondroRow rowdef) {
return open(file, useNodeCache, preloadTime, rowdef, rowdef.columns() /* txtProps */, 80 /* txtPropWidth */);
}
@Deprecated
public static final kelondroTree open(final File file, final boolean useNodeCache, final long preloadTime, final kelondroRow rowdef, final int txtProps, final int txtPropsWidth) {
// opens new or existing file; in case that any error occur the file is deleted again and it is tried to create the file again
// if that fails, the method returns null
@ -113,33 +115,6 @@ public class kelondroTree extends kelondroCachedRecords implements kelondroIndex
}
}
public kelondroTree(final kelondroRA ra, final String filename, final boolean useNodeCache, final long preloadTime, final kelondroRow rowdef, final boolean exitOnFail) {
// this creates a new tree within a kelondroRA
this(ra, filename, useNodeCache, preloadTime, rowdef, new kelondroNaturalOrder(true), rowdef.columns() /* txtProps */, 80 /* txtPropWidth */, exitOnFail);
}
public kelondroTree(final kelondroRA ra, final String filename, final boolean useNodeCache, final long preloadTime, final kelondroRow rowdef, final kelondroByteOrder objectOrder, final int txtProps, final int txtPropsWidth, final boolean exitOnFail) {
// this creates a new tree within a kelondroRA
super(ra, filename, useNodeCache, preloadTime,
thisOHBytes, thisOHHandles, rowdef,
thisFHandles, txtProps, txtPropsWidth, exitOnFail);
try {
setHandle(root, null); // define the root value
} catch (final IOException e) {
super.logFailure("cannot set root handle / " + e.getMessage());
if (exitOnFail) System.exit(-1);
throw new RuntimeException("cannot set root handle / " + e.getMessage());
}
super.setLogger(log);
this.loopDetectionOrder = new kelondroByteOrder.StringOrder(rowdef.objectOrder);
}
public kelondroTree(final kelondroRA ra, final String filename, final boolean useNodeCache, final long preloadTime) throws IOException {
// this opens a file with an existing tree in a kelondroRA
super(ra, filename, useNodeCache, preloadTime);
super.setLogger(log);
}
public void clear() throws IOException {
super.clear();
setHandle(root, null);

4
source/de/anomic/plasma/plasmaRankingCRProcess.java
Unescape View File

@ -34,7 +34,7 @@ import de.anomic.kelondro.kelondroAttrSeq;
import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroBitfield;
import de.anomic.kelondro.kelondroCollectionIndex;
import de.anomic.kelondro.kelondroFlexTable;
import de.anomic.kelondro.kelondroEcoTable;
import de.anomic.kelondro.kelondroIndex;
import de.anomic.kelondro.kelondroMicroDate;
import de.anomic.kelondro.kelondroRow;
@ -242,7 +242,7 @@ public class plasmaRankingCRProcess {
kelondroCollectionIndex newseq = null;
if (newdb) {
final File path = to_file.getParentFile(); // path to storage place
newacc = new kelondroFlexTable(path, CRG_accname, CRG_accrow, 0, false);
newacc = new kelondroEcoTable(new File(path, CRG_accname), CRG_accrow, kelondroEcoTable.tailCacheUsageAuto, 0, 0);
newseq = new kelondroCollectionIndex(path, CRG_seqname, 12, kelondroBase64Order.enhancedCoder, 2, 9, CRG_colrow, false);
} else {
if (!(to_file.exists())) {

Write Preview
Loading…
Cancel
Save