boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

more refactoring of indexer and kelondro classes;

Browse Source 
- integrating the indexer into kelondro as package 'text'
- renaming of classes in kelondro.index

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5663 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 16 years ago
parent 404bc21da9
commit 6ffc6e3389
50 changed files with 161 additions and 147 deletions
Show Stats Download Patch File Download Diff File

8
htroot/IndexControlRWIs_p.java
Unescape View File

@ -37,13 +37,13 @@ import java.util.Set;
import de.anomic.data.listManager;
import de.anomic.http.httpRequestHeader;
import de.anomic.index.indexAbstractReferenceBlacklist;
import de.anomic.index.ReferenceContainer;
import de.anomic.index.indexContainerCache;
import de.anomic.index.Reference;
import de.anomic.index.ReferenceRow;
import de.anomic.index.URLMetadata;
import de.anomic.index.Word;
import de.anomic.kelondro.order.Bitfield;
import de.anomic.kelondro.text.Reference;
import de.anomic.kelondro.text.ReferenceContainer;
import de.anomic.kelondro.text.ReferenceRow;
import de.anomic.kelondro.text.Word;
import de.anomic.plasma.plasmaSearchAPI;
import de.anomic.plasma.plasmaSearchEvent;
import de.anomic.plasma.plasmaSearchRankingProcess;

2
htroot/yacy/search.java
Unescape View File

@ -35,9 +35,9 @@ import java.util.Set;
import java.util.TreeSet;
import de.anomic.http.httpRequestHeader;
import de.anomic.index.ReferenceContainer;
import de.anomic.kelondro.order.Base64Order;
import de.anomic.kelondro.order.Bitfield;
import de.anomic.kelondro.text.ReferenceContainer;
import de.anomic.kelondro.util.SortStack;
import de.anomic.net.natLib;
import de.anomic.plasma.plasmaProfiling;

2
htroot/yacy/transferRWI.java
Unescape View File

@ -32,8 +32,8 @@ import java.util.Iterator;
import java.util.List;
import de.anomic.http.httpRequestHeader;
import de.anomic.index.ReferenceRow;
import de.anomic.index.indexReferenceBlacklist;
import de.anomic.kelondro.text.ReferenceRow;
import de.anomic.kelondro.util.FileUtils;
import de.anomic.kelondro.util.Log;
import de.anomic.plasma.plasmaSwitchboard;

2
htroot/yacysearch.java
Unescape View File

@ -32,8 +32,8 @@ import java.util.TreeSet;
import de.anomic.http.httpRequestHeader;
import de.anomic.index.URLMetadata;
import de.anomic.index.Word;
import de.anomic.kelondro.order.Bitfield;
import de.anomic.kelondro.text.Word;
import de.anomic.kelondro.util.MemoryControl;
import de.anomic.kelondro.util.SetTools;
import de.anomic.kelondro.util.Log;

6
source/dbtest.java
Unescape View File

@ -15,7 +15,7 @@ import javax.imageio.ImageIO;
import de.anomic.kelondro.index.Row;
import de.anomic.kelondro.index.RowSet;
import de.anomic.kelondro.index.ObjectIndex;
import de.anomic.kelondro.index.IntBytesMap;
import de.anomic.kelondro.index.ObjectArrayCache;
import de.anomic.kelondro.order.Base64Order;
import de.anomic.kelondro.order.CloneableIterator;
import de.anomic.kelondro.order.NaturalOrder;
@ -425,7 +425,7 @@ public class dbtest {
int p, rc=0;
final ArrayList<Long> ra = new ArrayList<Long>();
final HashSet<Long> jcontrol = new HashSet<Long>();
final IntBytesMap kcontrol = new IntBytesMap(1, 0);
final ObjectArrayCache kcontrol = new ObjectArrayCache(1, 0);
for (int i = 0; i < writeCount; i++) {
r = Math.abs(random.nextLong() % 1000);
jcontrol.add(Long.valueOf(r));
@ -465,7 +465,7 @@ public class dbtest {
int p, rc=0;
final ArrayList<Long> ra = new ArrayList<Long>();
final HashSet<Long> jcontrol = new HashSet<Long>();
final IntBytesMap kcontrol = new IntBytesMap(1, 0);
final ObjectArrayCache kcontrol = new ObjectArrayCache(1, 0);
for (int i = 0; i < writeCount; i++) {
//if (i == 30) random = new Random(randomstart);
r = Math.abs(random.nextLong() % 1000);

2
source/de/anomic/data/bookmarksDB.java
Unescape View File

@ -64,12 +64,12 @@ import de.anomic.crawler.CrawlEntry;
import de.anomic.crawler.CrawlProfile;
import de.anomic.htmlFilter.htmlFilterContentScraper;
import de.anomic.htmlFilter.htmlFilterWriter;
import de.anomic.index.Word;
import de.anomic.kelondro.blob.BLOBTree;
import de.anomic.kelondro.blob.MapView;
import de.anomic.kelondro.order.CloneableIterator;
import de.anomic.kelondro.order.DateFormatter;
import de.anomic.kelondro.order.NaturalOrder;
import de.anomic.kelondro.text.Word;
import de.anomic.kelondro.util.kelondroException;
import de.anomic.kelondro.util.Log;
import de.anomic.kelondro.util.FileUtils;

2
source/de/anomic/index/URLMetadata.java
Unescape View File

@ -40,6 +40,8 @@ import de.anomic.kelondro.order.Bitfield;
import de.anomic.kelondro.order.DateFormatter;
import de.anomic.kelondro.order.Digest;
import de.anomic.kelondro.order.NaturalOrder;
import de.anomic.kelondro.text.Reference;
import de.anomic.kelondro.text.ReferenceRow;
import de.anomic.kelondro.util.FileUtils;
import de.anomic.kelondro.util.kelondroException;
import de.anomic.plasma.plasmaSearchQuery;

1
source/de/anomic/index/URLMetadataRepository.java
Unescape View File

@ -48,6 +48,7 @@ import de.anomic.kelondro.index.Row;
import de.anomic.kelondro.index.ObjectIndex;
import de.anomic.kelondro.order.CloneableIterator;
import de.anomic.kelondro.table.SplitTable;
import de.anomic.kelondro.text.Reference;
import de.anomic.kelondro.util.ScoreCluster;
import de.anomic.kelondro.util.Log;
import de.anomic.yacy.yacyURL;

4
source/de/anomic/index/indexCollectionRI.java
Unescape View File

@ -53,6 +53,8 @@ import de.anomic.kelondro.order.RotateIterator;
import de.anomic.kelondro.table.EcoTable;
import de.anomic.kelondro.table.FixedWidthArray;
import de.anomic.kelondro.table.FlexTable;
import de.anomic.kelondro.text.Index;
import de.anomic.kelondro.text.ReferenceContainer;
import de.anomic.kelondro.util.FileUtils;
import de.anomic.kelondro.util.MemoryControl;
import de.anomic.kelondro.util.kelondroException;
@ -60,7 +62,7 @@ import de.anomic.kelondro.util.kelondroOutOfLimitsException;
import de.anomic.kelondro.util.Log;
import de.anomic.yacy.yacyURL;
public class indexCollectionRI implements ReverseIndex {
public class indexCollectionRI implements Index {
private static final int loadfactor = 4;
private static final int serialNumber = 0;

1
source/de/anomic/index/indexContainerBLOBArray.java
Unescape View File

@ -37,6 +37,7 @@ import de.anomic.kelondro.blob.BLOBArray;
import de.anomic.kelondro.index.Row;
import de.anomic.kelondro.index.RowSet;
import de.anomic.kelondro.order.CloneableIterator;
import de.anomic.kelondro.text.ReferenceContainer;
public final class indexContainerBLOBArray {

5
source/de/anomic/index/indexContainerCache.java
Unescape View File

@ -44,11 +44,14 @@ import de.anomic.kelondro.blob.HeapWriter;
import de.anomic.kelondro.order.CloneableIterator;
import de.anomic.kelondro.order.Base64Order;
import de.anomic.kelondro.order.ByteOrder;
import de.anomic.kelondro.text.Index;
import de.anomic.kelondro.text.ReferenceContainer;
import de.anomic.kelondro.text.ReferenceRow;
import de.anomic.kelondro.util.Log;
import de.anomic.kelondro.index.Row;
import de.anomic.kelondro.index.RowSet;
public final class indexContainerCache implements Iterable<ReferenceContainer>, ReverseIndex {
public final class indexContainerCache implements Iterable<ReferenceContainer>, Index {
private final Row payloadrow;
private SortedMap<String, ReferenceContainer> cache;

1
source/de/anomic/index/indexContainerOrder.java
Unescape View File

@ -28,6 +28,7 @@ package de.anomic.index;
import de.anomic.kelondro.order.AbstractOrder;
import de.anomic.kelondro.order.Order;
import de.anomic.kelondro.text.ReferenceContainer;
public class indexContainerOrder extends AbstractOrder<ReferenceContainer> implements Order<ReferenceContainer>, Cloneable {

8
source/de/anomic/kelondro/blob/HeapReader.java
Unescape View File

@ -34,7 +34,7 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.concurrent.ExecutionException;
import de.anomic.kelondro.index.BytesLongMap;
import de.anomic.kelondro.index.LongHandleIndex;
import de.anomic.kelondro.io.CachedRandomAccess;
import de.anomic.kelondro.order.ByteOrder;
import de.anomic.kelondro.order.CloneableIterator;
@ -45,7 +45,7 @@ import de.anomic.kelondro.util.Log;
public class HeapReader {
protected int keylength; // the length of the primary key
protected BytesLongMap index; // key/seek relation for used records
protected LongHandleIndex index; // key/seek relation for used records
protected Gap free; // set of {seek, size} pairs denoting space and position of free records
protected final File heapFile; // the file of the heap
protected final ByteOrder ordering; // the ordering on keys
@ -106,7 +106,7 @@ public class HeapReader {
// there is an index and a gap file:
// read the index file:
try {
this.index = new BytesLongMap(this.keylength, this.ordering, fif);
this.index = new LongHandleIndex(this.keylength, this.ordering, fif);
} catch (IOException e) {
e.printStackTrace();
return false;
@ -132,7 +132,7 @@ public class HeapReader {
// this initializes the this.index object by reading positions from the heap file
this.free = new Gap();
BytesLongMap.initDataConsumer indexready = BytesLongMap.asynchronusInitializer(keylength, this.ordering, 0, Math.max(10, (int) (Runtime.getRuntime().freeMemory() / (10 * 1024 * 1024))));
LongHandleIndex.initDataConsumer indexready = LongHandleIndex.asynchronusInitializer(keylength, this.ordering, 0, Math.max(10, (int) (Runtime.getRuntime().freeMemory() / (10 * 1024 * 1024))));
byte[] key = new byte[keylength];
int reclen;
long seek = 0;

6
source/de/anomic/kelondro/blob/HeapWriter.java
Unescape View File

@ -30,7 +30,7 @@ import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import de.anomic.kelondro.index.BytesLongMap;
import de.anomic.kelondro.index.LongHandleIndex;
import de.anomic.kelondro.order.ByteOrder;
import de.anomic.kelondro.order.Digest;
import de.anomic.kelondro.util.Log;
@ -38,7 +38,7 @@ import de.anomic.kelondro.util.Log;
public final class HeapWriter {
private int keylength; // the length of the primary key
private BytesLongMap index; // key/seek relation for used records
private LongHandleIndex index; // key/seek relation for used records
private final File heapFile; // the file of the heap
private DataOutputStream os; // the output stream where the BLOB is written
private long seek; // the current write position
@ -71,7 +71,7 @@ public final class HeapWriter {
public HeapWriter(final File heapFile, final int keylength, final ByteOrder ordering) throws IOException {
this.heapFile = heapFile;
this.keylength = keylength;
this.index = new BytesLongMap(keylength, ordering, 10);
this.index = new LongHandleIndex(keylength, ordering, 10);
this.os = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(heapFile), 1024 * 1024));
this.seek = 0;
}

26
source/de/anomic/kelondro/index/BytesIntMap.java → source/de/anomic/kelondro/index/IntegerHandleIndex.java
Unescape View File

@ -37,14 +37,14 @@ import java.util.concurrent.Future;
import de.anomic.kelondro.order.ByteOrder;
import de.anomic.kelondro.order.CloneableIterator;
public class BytesIntMap {
public class IntegerHandleIndex {
private final Row rowdef;
private ObjectIndex index;
public BytesIntMap(final int keylength, final ByteOrder objectOrder, final int space) {
public IntegerHandleIndex(final int keylength, final ByteOrder objectOrder, final int space) {
this.rowdef = new Row(new Column[]{new Column("key", Column.celltype_binary, Column.encoder_bytes, keylength, "key"), new Column("int c-4 {b256}")}, objectOrder, 0);
this.index = new RAMIndex(rowdef, space);
this.index = new ObjectIndexCache(rowdef, space);
}
public Row row() {
@ -156,27 +156,27 @@ public class BytesIntMap {
* @return
*/
public static initDataConsumer asynchronusInitializer(final int keylength, final ByteOrder objectOrder, final int space, int bufferSize) {
initDataConsumer initializer = new initDataConsumer(new BytesIntMap(keylength, objectOrder, space), bufferSize);
initDataConsumer initializer = new initDataConsumer(new IntegerHandleIndex(keylength, objectOrder, space), bufferSize);
ExecutorService service = Executors.newSingleThreadExecutor();
initializer.setResult(service.submit(initializer));
service.shutdown();
return initializer;
}
public static class initDataConsumer implements Callable<BytesIntMap> {
public static class initDataConsumer implements Callable<IntegerHandleIndex> {
private BlockingQueue<entry> cache;
private BytesIntMap map;
private Future<BytesIntMap> result;
private IntegerHandleIndex map;
private Future<IntegerHandleIndex> result;
private boolean sortAtEnd;
public initDataConsumer(BytesIntMap map, int bufferCount) {
public initDataConsumer(IntegerHandleIndex map, int bufferCount) {
this.map = map;
cache = new ArrayBlockingQueue<entry>(bufferCount);
sortAtEnd = false;
}
protected void setResult(Future<BytesIntMap> result) {
protected void setResult(Future<IntegerHandleIndex> result) {
this.result = result;
}
@ -214,11 +214,11 @@ public class BytesIntMap {
* @throws InterruptedException
* @throws ExecutionException
*/
public BytesIntMap result() throws InterruptedException, ExecutionException {
public IntegerHandleIndex result() throws InterruptedException, ExecutionException {
return this.result.get();
}
public BytesIntMap call() throws IOException {
public IntegerHandleIndex call() throws IOException {
try {
entry c;
while ((c = cache.take()) != poisonEntry) {
@ -227,8 +227,8 @@ public class BytesIntMap {
} catch (InterruptedException e) {
e.printStackTrace();
}
if (sortAtEnd && map.index instanceof RAMIndex) {
((RAMIndex) map.index).finishInitialization();
if (sortAtEnd && map.index instanceof ObjectIndexCache) {
((ObjectIndexCache) map.index).finishInitialization();
}
return map;
}

26
source/de/anomic/kelondro/index/BytesLongMap.java → source/de/anomic/kelondro/index/LongHandleIndex.java
Unescape View File

@ -45,10 +45,10 @@ import java.util.concurrent.Future;
import de.anomic.kelondro.order.ByteOrder;
import de.anomic.kelondro.order.CloneableIterator;
public class BytesLongMap {
public class LongHandleIndex {
private final Row rowdef;
private RAMIndex index;
private ObjectIndexCache index;
/**
* initialize a BytesLongMap
@ -58,9 +58,9 @@ public class BytesLongMap {
* @param objectOrder
* @param space
*/
public BytesLongMap(final int keylength, final ByteOrder objectOrder, final int space) {
public LongHandleIndex(final int keylength, final ByteOrder objectOrder, final int space) {
this.rowdef = new Row(new Column[]{new Column("key", Column.celltype_binary, Column.encoder_bytes, keylength, "key"), new Column("long c-8 {b256}")}, objectOrder, 0);
this.index = new RAMIndex(rowdef, space);
this.index = new ObjectIndexCache(rowdef, space);
}
/**
@ -70,7 +70,7 @@ public class BytesLongMap {
* @param file
* @throws IOException
*/
public BytesLongMap(final int keylength, final ByteOrder objectOrder, final File file) throws IOException {
public LongHandleIndex(final int keylength, final ByteOrder objectOrder, final File file) throws IOException {
this(keylength, objectOrder, (int) (file.length() / (keylength + 8)));
// read the index dump and fill the index
InputStream is = new BufferedInputStream(new FileInputStream(file), 1024 * 1024);
@ -200,7 +200,7 @@ public class BytesLongMap {
* @return
*/
public static initDataConsumer asynchronusInitializer(final int keylength, final ByteOrder objectOrder, final int space, int bufferSize) {
initDataConsumer initializer = new initDataConsumer(new BytesLongMap(keylength, objectOrder, space), bufferSize);
initDataConsumer initializer = new initDataConsumer(new LongHandleIndex(keylength, objectOrder, space), bufferSize);
ExecutorService service = Executors.newSingleThreadExecutor();
initializer.setResult(service.submit(initializer));
service.shutdown();
@ -217,18 +217,18 @@ public class BytesLongMap {
}
private static final entry poisonEntry = new entry(new byte[0], 0);
public static class initDataConsumer implements Callable<BytesLongMap> {
public static class initDataConsumer implements Callable<LongHandleIndex> {
private BlockingQueue<entry> cache;
private BytesLongMap map;
private Future<BytesLongMap> result;
private LongHandleIndex map;
private Future<LongHandleIndex> result;
public initDataConsumer(BytesLongMap map, int bufferCount) {
public initDataConsumer(LongHandleIndex map, int bufferCount) {
this.map = map;
cache = new ArrayBlockingQueue<entry>(bufferCount);
}
protected void setResult(Future<BytesLongMap> result) {
protected void setResult(Future<LongHandleIndex> result) {
this.result = result;
}
@ -265,11 +265,11 @@ public class BytesLongMap {
* @throws InterruptedException
* @throws ExecutionException
*/
public BytesLongMap result() throws InterruptedException, ExecutionException {
public LongHandleIndex result() throws InterruptedException, ExecutionException {
return this.result.get();
}
public BytesLongMap call() throws IOException {
public LongHandleIndex call() throws IOException {
try {
entry c;
while ((c = cache.take()) != poisonEntry) {

2
source/de/anomic/kelondro/index/Array.java → source/de/anomic/kelondro/index/ObjectArray.java
Unescape View File

@ -27,7 +27,7 @@ package de.anomic.kelondro.index;
import java.io.IOException;
public interface Array {
public interface ObjectArray {
public int size();

6
source/de/anomic/kelondro/index/IntBytesMap.java → source/de/anomic/kelondro/index/ObjectArrayCache.java
Unescape View File

@ -31,7 +31,7 @@ import java.util.Random;
import de.anomic.kelondro.order.NaturalOrder;
public class IntBytesMap {
public class ObjectArrayCache {
// we use two indexes: one for initialization, and one for data aquired during runtime
// this has a gread advantage, if the setup-data is large. Then a re-organisation of
@ -45,7 +45,7 @@ public class IntBytesMap {
private RowSet index1;
//private final kelondroOrder<kelondroRow.Entry> entryOrder;
public IntBytesMap(final int payloadSize, final int initSize) {
public ObjectArrayCache(final int payloadSize, final int initSize) {
this.rowdef = new Row("Cardinal key-4 {b256}, byte[] payload-" + payloadSize, NaturalOrder.naturalOrder, 0);
this.index0 = new RowSet(rowdef, initSize);
this.index1 = null;
@ -218,7 +218,7 @@ public class IntBytesMap {
int p, rc = 0;
final ArrayList<Long> ra = new ArrayList<Long>();
final HashSet<Long> jcontrol = new HashSet<Long>();
final IntBytesMap kcontrol = new IntBytesMap(1, 0);
final ObjectArrayCache kcontrol = new ObjectArrayCache(1, 0);
for (int i = 0; i < 1000000; i++) {
r = Math.abs(random.nextLong() % 10000);
//System.out.println("add " + r);

4
source/de/anomic/kelondro/index/RAMIndex.java → source/de/anomic/kelondro/index/ObjectIndexCache.java
Unescape View File

@ -35,13 +35,13 @@ import de.anomic.kelondro.order.CloneableIterator;
import de.anomic.kelondro.order.MergeIterator;
import de.anomic.kelondro.order.StackIterator;
public class RAMIndex implements ObjectIndex {
public class ObjectIndexCache implements ObjectIndex {
private final Row rowdef;
private RowSet index0, index1;
private final Row.EntryComparator entryComparator;
public RAMIndex(final Row rowdef, final int initialspace) {
public ObjectIndexCache(final Row rowdef, final int initialspace) {
this.rowdef = rowdef;
this.entryComparator = new Row.EntryComparator(rowdef.objectOrder);
reset(initialspace);

6
source/de/anomic/kelondro/table/CachedRecords.java
Unescape View File

@ -34,7 +34,7 @@ import java.util.Map;
import java.util.TreeMap;
import de.anomic.kelondro.index.Row;
import de.anomic.kelondro.index.IntBytesMap;
import de.anomic.kelondro.index.ObjectArrayCache;
import de.anomic.kelondro.io.RandomAccessInterface;
import de.anomic.kelondro.io.RandomAccessRecords;
import de.anomic.kelondro.util.MemoryControl;
@ -51,7 +51,7 @@ public class CachedRecords extends AbstractRecords implements RandomAccessRecord
private static final long memStartShrink = 20 * 1024 * 1024; // a limit for the node cache to start with shrinking if less than this memory amount is available
// caching buffer
private IntBytesMap cacheHeaders; // the cache; holds overhead values and key element
private ObjectArrayCache cacheHeaders; // the cache; holds overhead values and key element
private int readHit;
private int readMiss;
private int writeUnique;
@ -88,7 +88,7 @@ public class CachedRecords extends AbstractRecords implements RandomAccessRecord
private void initCache(final boolean useNodeCache, final long preloadTime) {
if (useNodeCache) {
this.cacheHeaders = new IntBytesMap(this.headchunksize, 0);
this.cacheHeaders = new ObjectArrayCache(this.headchunksize, 0);
} else {
this.cacheHeaders = null;
}

8
source/de/anomic/kelondro/table/EcoTable.java
Unescape View File

@ -38,7 +38,7 @@ import java.util.Map;
import java.util.TreeMap;
import java.util.TreeSet;
import de.anomic.kelondro.index.BytesIntMap;
import de.anomic.kelondro.index.IntegerHandleIndex;
import de.anomic.kelondro.index.Column;
import de.anomic.kelondro.index.Row;
import de.anomic.kelondro.index.RowCollection;
@ -75,7 +75,7 @@ public class EcoTable implements ObjectIndex {
public static final long maxarraylength = 134217727L; // that may be the maxmimum size of array length in some JVMs
private static final long minmemremaining = 20 * 1024 * 1024; // if less than this memory is remaininig, the memory copy of a table is abandoned
RowSet table;
BytesIntMap index;
IntegerHandleIndex index;
BufferedEcoFS file;
Row rowdef;
int fail;
@ -132,7 +132,7 @@ public class EcoTable implements ObjectIndex {
table = null; System.gc();
Log.logSevere("ECOTABLE", tablefile + ": RAM after releasing the table: " + (MemoryControl.available() / 1024 / 1024) + "MB");
}
index = new BytesIntMap(rowdef.primaryKeyLength, rowdef.objectOrder, records);
index = new IntegerHandleIndex(rowdef.primaryKeyLength, rowdef.objectOrder, records);
Log.logInfo("ECOTABLE", tablefile + ": EcoTable " + tablefile.toString() + " has table copy " + ((table == null) ? "DISABLED" : "ENABLED"));
// read all elements from the file into the copy table
@ -564,7 +564,7 @@ public class EcoTable implements ObjectIndex {
// initialize index and copy table
table = (table == null) ? null : new RowSet(taildef, 1);
index = new BytesIntMap(rowdef.primaryKeyLength, rowdef.objectOrder, 1);
index = new IntegerHandleIndex(rowdef.primaryKeyLength, rowdef.objectOrder, 1);
}
public Row row() {

4
source/de/anomic/kelondro/table/FixedWidthArray.java
Unescape View File

@ -33,12 +33,12 @@ import java.util.Iterator;
import java.util.Map;
import java.util.TreeMap;
import de.anomic.kelondro.index.Array;
import de.anomic.kelondro.index.ObjectArray;
import de.anomic.kelondro.index.Row;
import de.anomic.kelondro.io.RandomAccessInterface;
import de.anomic.kelondro.order.NaturalOrder;
public class FixedWidthArray extends FullRecords implements Array {
public class FixedWidthArray extends FullRecords implements ObjectArray {
// define the Over-Head-Array
private static short thisOHBytes = 0; // our record definition does not need extra bytes

12
source/de/anomic/kelondro/table/FlexTable.java
Unescape View File

@ -35,7 +35,7 @@ import java.util.Map;
import java.util.TreeMap;
import java.util.TreeSet;
import de.anomic.kelondro.index.BytesIntMap;
import de.anomic.kelondro.index.IntegerHandleIndex;
import de.anomic.kelondro.index.Row;
import de.anomic.kelondro.index.RowCollection;
import de.anomic.kelondro.index.RowSet;
@ -52,7 +52,7 @@ public class FlexTable extends FlexWidthArray implements ObjectIndex {
private static TreeMap<String, FlexTable> tableTracker = new TreeMap<String, FlexTable>();
// class objects
protected BytesIntMap index;
protected IntegerHandleIndex index;
private boolean RAMIndex;
/**
@ -112,7 +112,7 @@ public class FlexTable extends FlexWidthArray implements ObjectIndex {
} catch (final IOException e) {
if (resetOnFail) {
RAMIndex = true;
index = new BytesIntMap(super.row().column(0).cellwidth, super.rowdef.objectOrder, 0);
index = new IntegerHandleIndex(super.row().column(0).cellwidth, super.rowdef.objectOrder, 0);
} else {
throw new kelondroException(e.getMessage());
}
@ -122,7 +122,7 @@ public class FlexTable extends FlexWidthArray implements ObjectIndex {
public void clear() throws IOException {
super.reset();
RAMIndex = true;
index = new BytesIntMap(super.row().column(0).cellwidth, super.rowdef.objectOrder, 0);
index = new IntegerHandleIndex(super.row().column(0).cellwidth, super.rowdef.objectOrder, 0);
}
public static int staticSize(final File path, final String tablename) {
@ -145,10 +145,10 @@ public class FlexTable extends FlexWidthArray implements ObjectIndex {
return index.has(key);
}
private BytesIntMap initializeRamIndex(final int initialSpace) {
private IntegerHandleIndex initializeRamIndex(final int initialSpace) {
final int space = Math.max(super.col[0].size(), initialSpace) + 1;
if (space < 0) throw new kelondroException("wrong space: " + space);
final BytesIntMap ri = new BytesIntMap(super.row().column(0).cellwidth, super.rowdef.objectOrder, space);
final IntegerHandleIndex ri = new IntegerHandleIndex(super.row().column(0).cellwidth, super.rowdef.objectOrder, space);
final Iterator<Node> content = super.col[0].contentNodes(-1);
Node node;
int i;

4
source/de/anomic/kelondro/table/FlexWidthArray.java
Unescape View File

@ -32,14 +32,14 @@ import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import de.anomic.kelondro.index.Array;
import de.anomic.kelondro.index.ObjectArray;
import de.anomic.kelondro.index.Column;
import de.anomic.kelondro.index.Row;
import de.anomic.kelondro.order.NaturalOrder;
import de.anomic.kelondro.util.kelondroException;
import de.anomic.kelondro.util.Log;
public class FlexWidthArray implements Array {
public class FlexWidthArray implements ObjectArray {
protected FixedWidthArray[] col;
protected Row rowdef;

4
source/de/anomic/kelondro/table/SplitTable.java
Unescape View File

@ -48,7 +48,7 @@ import java.util.concurrent.TimeUnit;
import de.anomic.kelondro.blob.Cache;
import de.anomic.kelondro.index.Column;
import de.anomic.kelondro.index.RAMIndex;
import de.anomic.kelondro.index.ObjectIndexCache;
import de.anomic.kelondro.index.Row;
import de.anomic.kelondro.index.RowCollection;
import de.anomic.kelondro.index.ObjectIndex;
@ -67,7 +67,7 @@ public class SplitTable implements ObjectIndex {
// the table type can be either kelondroFlex or kelondroEco
private static final int EcoFSBufferSize = 20;
static final ObjectIndex dummyIndex = new RAMIndex(new Row(new Column[]{new Column("key", Column.celltype_binary, Column.encoder_bytes, 2, "key")}, NaturalOrder.naturalOrder, 0), 0);
static final ObjectIndex dummyIndex = new ObjectIndexCache(new Row(new Column[]{new Column("key", Column.celltype_binary, Column.encoder_bytes, 2, "key")}, NaturalOrder.naturalOrder, 0), 0);
// the thread pool for the keeperOf executor service
private ExecutorService executor;

4
source/de/anomic/index/ReverseIndex.java → source/de/anomic/kelondro/text/Index.java
Unescape View File

@ -26,14 +26,14 @@
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package de.anomic.index;
package de.anomic.kelondro.text;
import java.io.IOException;
import java.util.Set;
import de.anomic.kelondro.order.CloneableIterator;
public interface ReverseIndex {
public interface Index {
/**
* add references to the reverse index

7
source/de/anomic/index/ReverseIndexCache.java → source/de/anomic/kelondro/text/IndexCache.java
Unescape View File

@ -24,7 +24,7 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package de.anomic.index;
package de.anomic.kelondro.text;
import java.io.File;
import java.io.IOException;
@ -32,13 +32,14 @@ import java.util.ArrayList;
import java.util.Iterator;
import java.util.Set;
import de.anomic.index.indexContainerCache;
import de.anomic.kelondro.index.Row;
import de.anomic.kelondro.order.CloneableIterator;
import de.anomic.kelondro.util.MemoryControl;
import de.anomic.kelondro.util.ScoreCluster;
import de.anomic.kelondro.util.Log;
public final class ReverseIndexCache implements ReverseIndex, ReverseIndexReader, Iterable<ReferenceContainer> {
public final class IndexCache implements Index, IndexReader, Iterable<ReferenceContainer> {
// class variables
private final ScoreCluster<String> hashScore;
@ -52,7 +53,7 @@ public final class ReverseIndexCache implements ReverseIndex, ReverseIndexReader
private indexContainerCache heap;
@SuppressWarnings("unchecked")
public ReverseIndexCache(
public IndexCache(
final File databaseRoot,
final Row payloadrow,
final int entityCacheMaxSize,

9
source/de/anomic/index/ReverseIndexCell.java → source/de/anomic/kelondro/text/IndexCell.java
Unescape View File

@ -24,13 +24,16 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package de.anomic.index;
package de.anomic.kelondro.text;
import java.io.File;
import java.io.IOException;
import java.util.HashSet;
import java.util.Set;
import de.anomic.index.indexContainerBLOBArray;
import de.anomic.index.indexContainerCache;
import de.anomic.index.indexContainerOrder;
import de.anomic.kelondro.index.Row;
import de.anomic.kelondro.order.CloneableIterator;
import de.anomic.kelondro.order.MergeIterator;
@ -47,14 +50,14 @@ import de.anomic.kelondro.order.Order;
* another BLOB file in the index array.
*/
public final class ReverseIndexCell implements ReverseIndex {
public final class IndexCell implements Index {
// class variables
private indexContainerBLOBArray array;
private indexContainerCache ram;
private int maxRamEntries;
public ReverseIndexCell(
public IndexCell(
final File cellPath,
final Row payloadrow,
final int maxRamEntries

4
source/de/anomic/index/ReverseIndexReader.java → source/de/anomic/kelondro/text/IndexReader.java
Unescape View File

@ -24,13 +24,13 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package de.anomic.index;
package de.anomic.kelondro.text;
import java.util.Set;
import de.anomic.kelondro.order.CloneableIterator;
public interface ReverseIndexReader {
public interface IndexReader {
public int size();
public boolean hasReferences(String wordHash); // should only be used if in case that true is returned the getContainer is NOT called

6
source/de/anomic/index/indexPhrase.java → source/de/anomic/kelondro/text/Phrase.java
Unescape View File

@ -24,18 +24,18 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package de.anomic.index;
package de.anomic.kelondro.text;
import java.util.HashSet;
public class indexPhrase {
public class Phrase {
// object carries statistics for words and sentences
private int count; // number of occurrences
private final int handle; // unique handle, is initialized with sentence counter
private final HashSet<Integer> hash; //
public indexPhrase(final int handle) {
public Phrase(final int handle) {
this.count = 1;
this.handle = handle;
this.hash = new HashSet<Integer>();

2
source/de/anomic/index/Reference.java → source/de/anomic/kelondro/text/Reference.java
Unescape View File

@ -24,7 +24,7 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package de.anomic.index;
package de.anomic.kelondro.text;
import de.anomic.kelondro.order.Bitfield;

4
source/de/anomic/index/ReferenceContainer.java → source/de/anomic/kelondro/text/ReferenceContainer.java
Unescape View File

@ -24,7 +24,7 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package de.anomic.index;
package de.anomic.kelondro.text;
import java.lang.reflect.Method;
import java.util.Collection;
@ -207,7 +207,7 @@ public class ReferenceContainer extends RowSet {
static {
Method meth = null;
try {
final Class<?> c = Class.forName("de.anomic.index.ReferenceContainer");
final Class<?> c = Class.forName("de.anomic.kelonodro.text.ReferenceContainer");
meth = c.getMethod("mergeUnique", new Class[]{Object.class, Object.class});
} catch (final SecurityException e) {
System.out.println("Error while initializing containerMerge.SecurityException: " + e.getMessage());

2
source/de/anomic/index/ReferenceOrder.java → source/de/anomic/kelondro/text/ReferenceOrder.java
Unescape View File

@ -24,7 +24,7 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package de.anomic.index;
package de.anomic.kelondro.text;
import java.util.ArrayList;
import java.util.HashMap;

2
source/de/anomic/index/ReferenceRow.java → source/de/anomic/kelondro/text/ReferenceRow.java
Unescape View File

@ -24,7 +24,7 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package de.anomic.index;
package de.anomic.kelondro.text;
import de.anomic.kelondro.index.Column;
import de.anomic.kelondro.index.Row;

2
source/de/anomic/index/ReferenceVars.java → source/de/anomic/kelondro/text/ReferenceVars.java
Unescape View File

@ -24,7 +24,7 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package de.anomic.index;
package de.anomic.kelondro.text;
import de.anomic.kelondro.order.Bitfield;
import de.anomic.kelondro.order.MicroDate;

2
source/de/anomic/index/Word.java → source/de/anomic/kelondro/text/Word.java
Unescape View File

@ -24,7 +24,7 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package de.anomic.index;
package de.anomic.kelondro.text;
import java.util.HashSet;
import java.util.Iterator;

14
source/de/anomic/plasma/plasmaCondenser.java
Unescape View File

@ -48,10 +48,10 @@ import java.util.TreeSet;
import de.anomic.htmlFilter.htmlFilterContentScraper;
import de.anomic.htmlFilter.htmlFilterImageEntry;
import de.anomic.index.indexPhrase;
import de.anomic.index.Reference;
import de.anomic.index.Word;
import de.anomic.kelondro.order.Bitfield;
import de.anomic.kelondro.text.Phrase;
import de.anomic.kelondro.text.Reference;
import de.anomic.kelondro.text.Word;
import de.anomic.kelondro.util.SetTools;
import de.anomic.language.identification.Identificator;
import de.anomic.yacy.yacyURL;
@ -284,7 +284,7 @@ public final class plasmaCondenser {
String k;
int wordlen;
Word wsp, wsp1;
indexPhrase psp;
Phrase psp;
int wordHandle;
int wordHandleCount = 0;
int sentenceHandleCount = 0;
@ -295,7 +295,7 @@ public final class plasmaCondenser {
boolean comb_indexof = false, last_last = false, last_index = false;
RandomAccessFile fa;
final boolean dumpWords = false;
final HashMap<StringBuilder, indexPhrase> sentences = new HashMap<StringBuilder, indexPhrase>();
final HashMap<StringBuilder, Phrase> sentences = new HashMap<StringBuilder, Phrase>();
if (dumpWords) try {
fa = new RandomAccessFile(new File("dump.txt"), "rw");
@ -339,7 +339,7 @@ public final class plasmaCondenser {
} else {
// create new sentence
idx = sentenceHandleCount++;
sentences.put(sentence, new indexPhrase(idx));
sentences.put(sentence, new Phrase(idx));
}
// store to the words a link to this sentence
it = currsentwords.iterator();
@ -392,7 +392,7 @@ public final class plasmaCondenser {
psp.inc();
sentences.put(sentence, psp);
} else {
sentences.put(sentence, new indexPhrase(sentenceHandleCount++));
sentences.put(sentence, new Phrase(sentenceHandleCount++));
}
}

6
source/de/anomic/plasma/plasmaDbImporter.java
Unescape View File

@ -6,11 +6,11 @@ import java.util.TreeSet;
import de.anomic.crawler.AbstractImporter;
import de.anomic.crawler.Importer;
import de.anomic.index.ReferenceContainer;
import de.anomic.index.Reference;
import de.anomic.index.ReferenceRow;
import de.anomic.index.URLMetadata;
import de.anomic.kelondro.order.DateFormatter;
import de.anomic.kelondro.text.Reference;
import de.anomic.kelondro.text.ReferenceContainer;
import de.anomic.kelondro.text.ReferenceRow;
public class plasmaDbImporter extends AbstractImporter implements Importer {

2
source/de/anomic/plasma/plasmaRankingCRProcess.java
Unescape View File

@ -31,7 +31,6 @@ import java.io.IOException;
import java.util.Iterator;
import de.anomic.index.indexCollectionRI;
import de.anomic.index.ReferenceContainer;
import de.anomic.kelondro.index.Row;
import de.anomic.kelondro.index.RowSet;
import de.anomic.kelondro.index.ObjectIndex;
@ -41,6 +40,7 @@ import de.anomic.kelondro.order.CloneableIterator;
import de.anomic.kelondro.order.DateFormatter;
import de.anomic.kelondro.order.MicroDate;
import de.anomic.kelondro.table.EcoTable;
import de.anomic.kelondro.text.ReferenceContainer;
import de.anomic.kelondro.util.MemoryControl;
import de.anomic.kelondro.util.AttrSeq;
import de.anomic.kelondro.util.FileUtils;

2
source/de/anomic/plasma/plasmaSearchAPI.java
Unescape View File

@ -33,11 +33,11 @@ import java.util.Iterator;
import java.util.List;
import de.anomic.data.listManager;
import de.anomic.index.Reference;
import de.anomic.index.indexReferenceBlacklist;
import de.anomic.index.URLMetadata;
import de.anomic.kelondro.order.Bitfield;
import de.anomic.kelondro.order.DateFormatter;
import de.anomic.kelondro.text.Reference;
import de.anomic.server.serverObjects;
import de.anomic.yacy.yacySeed;
import de.anomic.yacy.yacyURL;

6
source/de/anomic/plasma/plasmaSearchEvent.java
Unescape View File

@ -37,11 +37,11 @@ import java.util.TreeSet;
import java.util.concurrent.ConcurrentHashMap;
import de.anomic.crawler.ResultURLs;
import de.anomic.index.ReferenceContainer;
import de.anomic.index.Reference;
import de.anomic.index.ReferenceVars;
import de.anomic.index.URLMetadata;
import de.anomic.kelondro.order.Bitfield;
import de.anomic.kelondro.text.Reference;
import de.anomic.kelondro.text.ReferenceContainer;
import de.anomic.kelondro.text.ReferenceVars;
import de.anomic.kelondro.util.MemoryControl;
import de.anomic.kelondro.util.SetTools;
import de.anomic.kelondro.util.SortStack;

2
source/de/anomic/plasma/plasmaSearchQuery.java
Unescape View File

@ -28,10 +28,10 @@ import java.util.TreeSet;
import de.anomic.htmlFilter.htmlFilterAbstractScraper;
import de.anomic.htmlFilter.htmlFilterCharacterCoding;
import de.anomic.index.Word;
import de.anomic.kelondro.order.Base64Order;
import de.anomic.kelondro.order.Bitfield;
import de.anomic.kelondro.order.NaturalOrder;
import de.anomic.kelondro.text.Word;
import de.anomic.kelondro.util.SetTools;
import de.anomic.yacy.yacySeed;
import de.anomic.yacy.yacySeedDB;

10
source/de/anomic/plasma/plasmaSearchRankingProcess.java
Unescape View File

@ -37,14 +37,14 @@ import java.util.TreeSet;
import java.util.concurrent.ConcurrentHashMap;
import de.anomic.htmlFilter.htmlFilterContentScraper;
import de.anomic.index.ReferenceContainer;
import de.anomic.index.Reference;
import de.anomic.index.ReferenceOrder;
import de.anomic.index.ReferenceVars;
import de.anomic.index.URLMetadata;
import de.anomic.index.Word;
import de.anomic.kelondro.index.BinSearch;
import de.anomic.kelondro.order.Digest;
import de.anomic.kelondro.text.Reference;
import de.anomic.kelondro.text.ReferenceContainer;
import de.anomic.kelondro.text.ReferenceOrder;
import de.anomic.kelondro.text.ReferenceVars;
import de.anomic.kelondro.text.Word;
import de.anomic.kelondro.util.ScoreCluster;
import de.anomic.kelondro.util.SortStack;
import de.anomic.kelondro.util.FileUtils;

2
source/de/anomic/plasma/plasmaSnippetCache.java
Unescape View File

@ -45,7 +45,7 @@ import de.anomic.http.httpClient;
import de.anomic.http.httpResponseHeader;
import de.anomic.index.indexDocumentMetadata;
import de.anomic.index.URLMetadata;
import de.anomic.index.Word;
import de.anomic.kelondro.text.Word;
import de.anomic.kelondro.util.ScoreCluster;
import de.anomic.kelondro.util.SetTools;
import de.anomic.kelondro.util.Log;

30
source/de/anomic/plasma/plasmaWordIndex.java
Unescape View File

@ -42,16 +42,10 @@ import de.anomic.crawler.IndexingStack;
import de.anomic.htmlFilter.htmlFilterContentScraper;
import de.anomic.http.httpdProxyCacheEntry;
import de.anomic.index.indexCollectionRI;
import de.anomic.index.ReferenceContainer;
import de.anomic.index.indexContainerOrder;
import de.anomic.index.ReverseIndexCache;
import de.anomic.index.ReverseIndex;
import de.anomic.index.Reference;
import de.anomic.index.ReferenceRow;
import de.anomic.index.indexReferenceBlacklist;
import de.anomic.index.URLMetadataRepository;
import de.anomic.index.URLMetadata;
import de.anomic.index.Word;
import de.anomic.index.URLMetadataRepository.Export;
import de.anomic.kelondro.index.RowCollection;
import de.anomic.kelondro.order.Base64Order;
@ -60,6 +54,12 @@ import de.anomic.kelondro.order.CloneableIterator;
import de.anomic.kelondro.order.MergeIterator;
import de.anomic.kelondro.order.Order;
import de.anomic.kelondro.order.RotateIterator;
import de.anomic.kelondro.text.Index;
import de.anomic.kelondro.text.IndexCache;
import de.anomic.kelondro.text.Reference;
import de.anomic.kelondro.text.ReferenceContainer;
import de.anomic.kelondro.text.ReferenceRow;
import de.anomic.kelondro.text.Word;
import de.anomic.kelondro.util.MemoryControl;
import de.anomic.kelondro.util.kelondroException;
import de.anomic.kelondro.util.Log;
@ -70,7 +70,7 @@ import de.anomic.xml.RSSMessage;
import de.anomic.yacy.yacySeedDB;
import de.anomic.yacy.yacyURL;
public final class plasmaWordIndex implements ReverseIndex {
public final class plasmaWordIndex implements Index {
// environment constants
public static final long wCacheMaxAge = 1000 * 60 * 30; // milliseconds; 30 minutes
@ -96,7 +96,7 @@ public final class plasmaWordIndex implements ReverseIndex {
public static final long CRAWL_PROFILE_SNIPPET_GLOBAL_MEDIA_RECRAWL_CYCLE = 60L * 24L * 30L;
private final ReverseIndexCache indexCache;
private final IndexCache indexCache;
private final indexCollectionRI collections; // new database structure to replace AssortmentCluster and FileCluster
private final Log log;
public URLMetadataRepository referenceURL;
@ -147,15 +147,15 @@ public final class plasmaWordIndex implements ReverseIndex {
if (!(textindexcache.exists())) textindexcache.mkdirs();
if (new File(textindexcache, "index.dhtin.blob").exists()) {
// migration of the both caches into one
this.indexCache = new ReverseIndexCache(textindexcache, ReferenceRow.urlEntryRow, entityCacheMaxSize, wCacheMaxChunk, wCacheMaxAge, "index.dhtout.blob", log);
ReverseIndexCache dhtInCache = new ReverseIndexCache(textindexcache, ReferenceRow.urlEntryRow, entityCacheMaxSize, wCacheMaxChunk, wCacheMaxAge, "index.dhtin.blob", log);
this.indexCache = new IndexCache(textindexcache, ReferenceRow.urlEntryRow, entityCacheMaxSize, wCacheMaxChunk, wCacheMaxAge, "index.dhtout.blob", log);
IndexCache dhtInCache = new IndexCache(textindexcache, ReferenceRow.urlEntryRow, entityCacheMaxSize, wCacheMaxChunk, wCacheMaxAge, "index.dhtin.blob", log);
for (ReferenceContainer c: dhtInCache) {
this.indexCache.addReferences(c);
}
new File(textindexcache, "index.dhtin.blob").delete();
} else {
// read in new BLOB
this.indexCache = new ReverseIndexCache(textindexcache, ReferenceRow.urlEntryRow, entityCacheMaxSize, wCacheMaxChunk, wCacheMaxAge, "index.dhtout.blob", log);
this.indexCache = new IndexCache(textindexcache, ReferenceRow.urlEntryRow, entityCacheMaxSize, wCacheMaxChunk, wCacheMaxAge, "index.dhtout.blob", log);
}
// create collections storage path
@ -449,7 +449,7 @@ public final class plasmaWordIndex implements ReverseIndex {
// calculate the real size in bytes of the index cache
long cacheBytes = 0;
final long entryBytes = ReferenceRow.urlEntryRow.objectsize;
final ReverseIndexCache cache = (indexCache);
final IndexCache cache = (indexCache);
synchronized (cache) {
final Iterator<ReferenceContainer> it = cache.referenceIterator(null, false, true);
while (it.hasNext()) cacheBytes += it.next().size() * entryBytes;
@ -461,7 +461,7 @@ public final class plasmaWordIndex implements ReverseIndex {
indexCache.setMaxWordCount(maxWords);
}
public void cacheFlushControl(final ReverseIndexCache theCache) {
public void cacheFlushControl(final IndexCache theCache) {
// check for forced flush
int cs = cacheSize();
if (cs > 0) {
@ -512,11 +512,11 @@ public final class plasmaWordIndex implements ReverseIndex {
}
}
private synchronized void flushCacheOne(final ReverseIndexCache ram) {
private synchronized void flushCacheOne(final IndexCache ram) {
if (ram.size() > 0) collections.addReferences(flushContainer(ram));
}
private ReferenceContainer flushContainer(final ReverseIndexCache ram) {
private ReferenceContainer flushContainer(final IndexCache ram) {
String wordHash;
ReferenceContainer c;
wordHash = ram.maxScoreWordHash();

10
source/de/anomic/yacy/dht/Dispatcher.java
Unescape View File

@ -31,11 +31,11 @@ import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import de.anomic.index.ReferenceContainer;
import de.anomic.index.ReverseIndex;
import de.anomic.index.ReferenceRow;
import de.anomic.index.URLMetadataRepository;
import de.anomic.kelondro.order.Base64Order;
import de.anomic.kelondro.text.Index;
import de.anomic.kelondro.text.ReferenceContainer;
import de.anomic.kelondro.text.ReferenceRow;
import de.anomic.kelondro.util.Log;
import de.anomic.server.serverProcessor;
import de.anomic.yacy.yacySeed;
@ -80,7 +80,7 @@ public class Dispatcher {
private HashMap<String, Transmission.Chunk> transmissionCloud;
// the backend is used to store the remaining indexContainers in case that the object is closed
private ReverseIndex backend;
private Index backend;
// the seed database
private yacySeedDB seeds;
@ -95,7 +95,7 @@ public class Dispatcher {
private Transmission transmission;
public Dispatcher(
final ReverseIndex backend,
final Index backend,
final URLMetadataRepository repository,
final yacySeedDB seeds,
final boolean gzipBody,

10
source/de/anomic/yacy/dht/Transmission.java
Unescape View File

@ -30,13 +30,13 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import de.anomic.index.ReferenceContainer;
import de.anomic.index.indexContainerCache;
import de.anomic.index.ReverseIndex;
import de.anomic.index.ReferenceRow;
import de.anomic.index.URLMetadataRepository;
import de.anomic.index.URLMetadata;
import de.anomic.kelondro.index.Row;
import de.anomic.kelondro.text.Index;
import de.anomic.kelondro.text.ReferenceContainer;
import de.anomic.kelondro.text.ReferenceRow;
import de.anomic.kelondro.util.Log;
import de.anomic.server.serverProcessorJob;
import de.anomic.yacy.yacyClient;
@ -48,7 +48,7 @@ public class Transmission {
private Log log;
private URLMetadataRepository repository;
private yacySeedDB seeds;
private ReverseIndex backend;
private Index backend;
private boolean gzipBody4Transfer;
private int timeout4Transfer;
@ -56,7 +56,7 @@ public class Transmission {
Log log,
URLMetadataRepository repository,
yacySeedDB seeds,
ReverseIndex backend,
Index backend,
boolean gzipBody4Transfer,
int timeout4Transfer) {
this.log = log;

8
source/de/anomic/yacy/yacyClient.java
Unescape View File

@ -66,16 +66,16 @@ import de.anomic.http.httpClient;
import de.anomic.http.httpResponse;
import de.anomic.http.httpRemoteProxyConfig;
import de.anomic.http.httpRequestHeader;
import de.anomic.index.ReferenceContainer;
import de.anomic.index.indexContainerCache;
import de.anomic.index.Reference;
import de.anomic.index.ReferenceRow;
import de.anomic.index.indexReferenceBlacklist;
import de.anomic.index.URLMetadata;
import de.anomic.index.Word;
import de.anomic.kelondro.order.Base64Order;
import de.anomic.kelondro.order.Bitfield;
import de.anomic.kelondro.order.Digest;
import de.anomic.kelondro.text.Reference;
import de.anomic.kelondro.text.ReferenceContainer;
import de.anomic.kelondro.text.ReferenceRow;
import de.anomic.kelondro.text.Word;
import de.anomic.kelondro.util.ByteBuffer;
import de.anomic.kelondro.util.FileUtils;
import de.anomic.plasma.plasmaSearchRankingProcess;

2
source/de/anomic/yacy/yacySeed.java
Unescape View File

@ -55,10 +55,10 @@ import java.util.Random;
import java.util.Set;
import java.util.TreeMap;
import de.anomic.index.Word;
import de.anomic.kelondro.order.Base64Order;
import de.anomic.kelondro.order.DateFormatter;
import de.anomic.kelondro.order.Digest;
import de.anomic.kelondro.text.Word;
import de.anomic.net.natLib;
import de.anomic.server.serverCodings;
import de.anomic.server.serverDomains;

8
source/yacy.java
Unescape View File

@ -52,17 +52,17 @@ import de.anomic.http.httpClient;
import de.anomic.http.httpResponse;
import de.anomic.http.httpRequestHeader;
import de.anomic.http.httpd;
import de.anomic.index.ReferenceContainer;
import de.anomic.index.Reference;
import de.anomic.index.ReferenceRow;
import de.anomic.index.URLMetadataRepository;
import de.anomic.index.URLMetadata;
import de.anomic.index.Word;
import de.anomic.kelondro.blob.BLOBHeap;
import de.anomic.kelondro.blob.MapDataMining;
import de.anomic.kelondro.index.RowCollection;
import de.anomic.kelondro.order.Base64Order;
import de.anomic.kelondro.order.DateFormatter;
import de.anomic.kelondro.text.Reference;
import de.anomic.kelondro.text.ReferenceContainer;
import de.anomic.kelondro.text.ReferenceRow;
import de.anomic.kelondro.text.Word;
import de.anomic.kelondro.util.MemoryControl;
import de.anomic.kelondro.util.ScoreCluster;
import de.anomic.kelondro.util.Log;

Write Preview
Loading…
Cancel
Save