boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

- added large number of generics

Browse Source 
- redesign of ordering structures in kelondro (old did not work with strict generics)
- 50% IO reduction during read access on kelondroFlex (ommiting of read on index table)

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4320 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 17 years ago
parent eff372e20c
commit a5054c038d
50 changed files with 521 additions and 632 deletions
Show Stats Download Patch File Download Diff File

4
htroot/yacy/hello.java
Unescape View File

@ -198,7 +198,7 @@ public final class hello {
if (count > 100) { count = 100; }
// latest seeds
final Map ySeeds = yacyCore.seedDB.seedsByAge(true, count); // peerhash/yacySeed relation
final Map<String, yacySeed> ySeeds = yacyCore.seedDB.seedsByAge(true, count); // peerhash/yacySeed relation
// attach also my own seed
seeds.append("seed0=").append(yacyCore.seedDB.mySeed().genSeedStr(key)).append(serverCore.CRLF_STRING);
@ -207,7 +207,7 @@ public final class hello {
// attach other seeds
if (ySeeds != null) {
seeds.ensureCapacity((ySeeds.size() + 1) * 768);
Iterator si = ySeeds.values().iterator();
Iterator<yacySeed> si = ySeeds.values().iterator();
yacySeed s;
while (si.hasNext()) {
s = (yacySeed) si.next();

4
htroot/yacy/profile.java
Unescape View File

@ -92,9 +92,9 @@ public final class profile {
if (fileIn != null) try { fileIn.close(); fileIn = null; } catch (Exception e) {}
}
Iterator it = ((Map)profile).keySet().iterator();
Iterator<String> it = ((Map)profile).keySet().iterator();
while (it.hasNext()) {
key=(String)it.next();
key = it.next();
value=profile.getProperty(key, "").replaceAll("\r","").replaceAll("\n","\\\\n");
if( !(key.equals("")) && !(value.equals("")) ){
prop.put("list_"+count+"_key", key);

45
htroot/yacy/search.java
Unescape View File

@ -45,6 +45,7 @@ import de.anomic.plasma.plasmaSearchEvent;
import de.anomic.plasma.plasmaSearchQuery;
import de.anomic.plasma.plasmaSearchRankingProfile;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.plasma.plasmaSearchEvent.ResultEntry;
import de.anomic.server.serverCore;
import de.anomic.server.serverObjects;
import de.anomic.server.serverProfiling;
@ -120,7 +121,7 @@ public final class search {
// tell all threads to do nothing for a specific time
sb.intermissionAllThreads(2 * duetime);
TreeSet abstractSet = ((abstracts.length() == 0) || (abstracts.equals("auto"))) ? null : plasmaSearchQuery.hashes2Set(abstracts);
TreeSet<String> abstractSet = ((abstracts.length() == 0) || (abstracts.equals("auto"))) ? null : plasmaSearchQuery.hashes2Set(abstracts);
// store accessing peer
if (yacyCore.seedDB == null) {
@ -130,8 +131,8 @@ public final class search {
}
// prepare search
final TreeSet queryhashes = plasmaSearchQuery.hashes2Set(query);
final TreeSet excludehashes = (exclude.length() == 0) ? new TreeSet(kelondroBase64Order.enhancedCoder) : plasmaSearchQuery.hashes2Set(exclude);
final TreeSet<String> queryhashes = plasmaSearchQuery.hashes2Set(query);
final TreeSet<String> excludehashes = (exclude.length() == 0) ? new TreeSet<String>(kelondroBase64Order.enhancedComparator) : plasmaSearchQuery.hashes2Set(exclude);
final long timestamp = System.currentTimeMillis();
// prepare a search profile
@ -142,25 +143,25 @@ public final class search {
int indexabstractContainercount = 0;
int joincount = 0;
plasmaSearchQuery theQuery = null;
ArrayList accu = null;
ArrayList<ResultEntry> accu = null;
long urlRetrievalAllTime = 0, snippetComputationAllTime = 0;
if ((query.length() == 0) && (abstractSet != null)) {
// this is _not_ a normal search, only a request for index abstracts
theQuery = new plasmaSearchQuery(null, abstractSet, new TreeSet(kelondroBase64Order.enhancedCoder), rankingProfile, maxdist, prefer, plasmaSearchQuery.contentdomParser(contentdom), false, count, 0, duetime, filter, plasmaSearchQuery.SEARCHDOM_LOCAL, null, -1, null, false);
theQuery = new plasmaSearchQuery(null, abstractSet, new TreeSet<String>(kelondroBase64Order.enhancedComparator), rankingProfile, maxdist, prefer, plasmaSearchQuery.contentdomParser(contentdom), false, count, 0, duetime, filter, plasmaSearchQuery.SEARCHDOM_LOCAL, null, -1, null, false);
theQuery.domType = plasmaSearchQuery.SEARCHDOM_LOCAL;
yacyCore.log.logInfo("INIT HASH SEARCH (abstracts only): " + plasmaSearchQuery.anonymizedQueryHashes(theQuery.queryHashes) + " - " + theQuery.displayResults() + " links");
long timer = System.currentTimeMillis();
Map[] containers = sb.wordIndex.localSearchContainers(theQuery, plasmaSearchQuery.hashes2Set(urls));
Map<String, indexContainer>[] containers = sb.wordIndex.localSearchContainers(theQuery, plasmaSearchQuery.hashes2Set(urls));
serverProfiling.update("SEARCH", new plasmaProfiling.searchEvent(theQuery.id(true), plasmaSearchEvent.COLLECTION, containers[0].size(), System.currentTimeMillis() - timer));
if (containers != null) {
Iterator ci = containers[0].entrySet().iterator();
Map.Entry entry;
Iterator<Map.Entry<String, indexContainer>> ci = containers[0].entrySet().iterator();
Map.Entry<String, indexContainer> entry;
String wordhash;
while (ci.hasNext()) {
entry = (Map.Entry) ci.next();
wordhash = (String) entry.getKey();
indexContainer container = (indexContainer) entry.getValue();
entry = ci.next();
wordhash = entry.getKey();
indexContainer container = entry.getValue();
indexabstractContainercount += container.size();
indexabstract.append("indexabstract." + wordhash + "=").append(indexContainer.compressIndex(container, null, 1000).toString()).append(serverCore.CRLF_STRING);
}
@ -188,18 +189,18 @@ public final class search {
} else {
// attach information about index abstracts
StringBuffer indexcount = new StringBuffer();
Map.Entry entry;
Iterator i = theSearch.IACount.entrySet().iterator();
Map.Entry<String, Integer> entry;
Iterator<Map.Entry<String, Integer>> i = theSearch.IACount.entrySet().iterator();
while (i.hasNext()) {
entry = (Map.Entry) i.next();
entry = i.next();
indexcount.append("indexcount.").append((String) entry.getKey()).append('=').append(((Integer) entry.getValue()).toString()).append(serverCore.CRLF_STRING);
}
if (abstractSet != null) {
// if a specific index-abstract is demanded, attach it here
i = abstractSet.iterator();
Iterator<String> j = abstractSet.iterator();
String wordhash;
while (i.hasNext()) {
wordhash = (String) i.next();
while (j.hasNext()) {
wordhash = (String) j.next();
indexabstractContainercount += ((Integer) theSearch.IACount.get(wordhash)).intValue();
indexabstract.append("indexabstract." + wordhash + "=").append((String) theSearch.IAResults.get(wordhash)).append(serverCore.CRLF_STRING);
}
@ -238,9 +239,9 @@ public final class search {
// prepare reference hints
long timer = System.currentTimeMillis();
Set ws = theSearch.references(10);
Set<String> ws = theSearch.references(10);
StringBuffer refstr = new StringBuffer();
Iterator j = ws.iterator();
Iterator<String> j = ws.iterator();
while (j.hasNext()) {
refstr.append(",").append((String) j.next());
}
@ -282,15 +283,15 @@ public final class search {
// prepare search statistics
Long trackerHandle = new Long(System.currentTimeMillis());
HashMap searchProfile = theQuery.resultProfile(joincount, System.currentTimeMillis() - timestamp, urlRetrievalAllTime, snippetComputationAllTime);
HashMap<String, Object> searchProfile = theQuery.resultProfile(joincount, System.currentTimeMillis() - timestamp, urlRetrievalAllTime, snippetComputationAllTime);
String client = (String) header.get("CLIENTIP");
searchProfile.put("host", client);
yacySeed remotepeer = yacyCore.seedDB.lookupByIP(natLib.getInetAddress(client), true, false, false);
searchProfile.put("peername", (remotepeer == null) ? "unknown" : remotepeer.getName());
searchProfile.put("time", trackerHandle);
sb.remoteSearches.add(searchProfile);
TreeSet handles = (TreeSet) sb.remoteSearchTracker.get(client);
if (handles == null) handles = new TreeSet();
TreeSet<Long> handles = sb.remoteSearchTracker.get(client);
if (handles == null) handles = new TreeSet<Long>();
handles.add(trackerHandle);
sb.remoteSearchTracker.put(client, handles);

2
htroot/yacy/transferRWI.java
Unescape View File

@ -123,7 +123,7 @@ public final class transferRWI {
final long startProcess = System.currentTimeMillis();
// decode request
final List v = nxTools.strings(indexes, null);
final List<String> v = nxTools.strings(indexes, null);
// free memory
indexes = null;

14
htroot/yacysearch.java
Unescape View File

@ -200,7 +200,7 @@ public class yacysearch {
}
// filter out stopwords
final TreeSet filtered = kelondroMSetTools.joinConstructive(query[0], plasmaSwitchboard.stopwords);
final TreeSet<String> filtered = kelondroMSetTools.joinConstructive(query[0], plasmaSwitchboard.stopwords);
if (filtered.size() > 0) {
kelondroMSetTools.excludeDestructive(query[0], plasmaSwitchboard.stopwords);
}
@ -217,7 +217,7 @@ public class yacysearch {
sb.wordIndex.removeWordReferences(query[0], delHash);
// make new news message with negative voting
HashMap map = new HashMap();
HashMap<String, String> map = new HashMap<String, String>();
map.put("urlhash", delHash);
map.put("vote", "negative");
map.put("refid", "");
@ -238,7 +238,7 @@ public class yacysearch {
document = plasmaSnippetCache.retrieveDocument(comp.url(), true, 5000, true);
if (document != null) {
// create a news message
HashMap map = new HashMap();
HashMap<String, String> map = new HashMap<String, String>();
map.put("url", comp.url().toNormalform(false, true).replace(',', '|'));
map.put("title", comp.title().replace(',', ' '));
map.put("description", ((document == null) ? comp.title() : document.getTitle()).replace(',', ' '));
@ -255,7 +255,7 @@ public class yacysearch {
final boolean globalsearch = (global) && (yacyonline);
// do the search
TreeSet queryHashes = plasmaCondenser.words2hashes(query[0]);
TreeSet<String> queryHashes = plasmaCondenser.words2hashes(query[0]);
plasmaSearchQuery theQuery = new plasmaSearchQuery(
querystring,
queryHashes,
@ -311,14 +311,14 @@ public class yacysearch {
// prepare search statistics
Long trackerHandle = new Long(System.currentTimeMillis());
HashMap searchProfile = theQuery.resultProfile(theSearch.getLocalCount() + theSearch.getGlobalCount(), System.currentTimeMillis() - timestamp, theSearch.getURLRetrievalTime(), theSearch.getSnippetComputationTime());
HashMap<String, Object> searchProfile = theQuery.resultProfile(theSearch.getLocalCount() + theSearch.getGlobalCount(), System.currentTimeMillis() - timestamp, theSearch.getURLRetrievalTime(), theSearch.getSnippetComputationTime());
searchProfile.put("querystring", theQuery.queryString);
searchProfile.put("time", trackerHandle);
searchProfile.put("host", client);
searchProfile.put("offset", new Integer(0));
sb.localSearches.add(searchProfile);
TreeSet handles = (TreeSet) sb.localSearchTracker.get(client);
if (handles == null) handles = new TreeSet();
TreeSet<Long> handles = sb.localSearchTracker.get(client);
if (handles == null) handles = new TreeSet<Long>();
handles.add(trackerHandle);
sb.localSearchTracker.put(client, handles);

20
htroot/yacysearchitem.java
Unescape View File

@ -93,21 +93,21 @@ public class yacysearchitem {
if (bottomline) {
// attach the bottom line with search references (topwords)
final Set references = theSearch.references(20);
final Set<String> references = theSearch.references(20);
if (references.size() > 0) {
// get the topwords
final TreeSet topwords = new TreeSet(kelondroNaturalOrder.naturalOrder);
final TreeSet<String> topwords = new TreeSet<String>(kelondroNaturalOrder.naturalComparator);
String tmp = "";
Iterator i = references.iterator();
Iterator<String> i = references.iterator();
while (i.hasNext()) {
tmp = (String) i.next();
tmp = i.next();
if (tmp.matches("[a-z]+")) {
topwords.add(tmp);
}
}
// filter out the badwords
final TreeSet filteredtopwords = kelondroMSetTools.joinConstructive(topwords, plasmaSwitchboard.badwords);
final TreeSet<String> filteredtopwords = kelondroMSetTools.joinConstructive(topwords, plasmaSwitchboard.badwords);
if (filteredtopwords.size() > 0) {
kelondroMSetTools.excludeDestructive(topwords, plasmaSwitchboard.badwords);
}
@ -122,7 +122,7 @@ public class yacysearchitem {
if (rss) {
String word;
int hintcount = 0;
final Iterator iter = topwords.iterator();
final Iterator<String> iter = topwords.iterator();
while (iter.hasNext()) {
word = (String) iter.next();
if (word != null) {
@ -137,7 +137,7 @@ public class yacysearchitem {
} else {
String word;
int hintcount = 0;
final Iterator iter = topwords.iterator();
final Iterator<String> iter = topwords.iterator();
while (iter.hasNext()) {
word = (String) iter.next();
if (word != null) {
@ -209,7 +209,7 @@ public class yacysearchitem {
prop.put("content_ybr", plasmaSearchRankingProcess.ybr(result.hash()));
prop.putNum("content_size", result.filesize());
TreeSet[] query = theQuery.queryWords();
TreeSet<String>[] query = theQuery.queryWords();
yacyURL wordURL = null;
try {
prop.putHTML("content_words", URLEncoder.encode(query[0].toString(),"UTF-8"));
@ -227,7 +227,7 @@ public class yacysearchitem {
// image search; shows thumbnails
// iterate over all images in the result
prop.put("content", theQuery.contentdom + 1); // switch on specific content
ArrayList /* of plasmaSnippetCache.MediaSnippet */ images = result.mediaSnippets();
ArrayList<plasmaSnippetCache.MediaSnippet> images = result.mediaSnippets();
if (images != null) {
plasmaSnippetCache.MediaSnippet ms;
yacyURL url;
@ -253,7 +253,7 @@ public class yacysearchitem {
(theQuery.contentdom == plasmaSearchQuery.CONTENTDOM_APP)) {
// any other media content
prop.put("content", theQuery.contentdom + 1); // switch on specific content
ArrayList /* of plasmaSnippetCache.MediaSnippet */ media = result.mediaSnippets();
ArrayList<plasmaSnippetCache.MediaSnippet> media = result.mediaSnippets();
if (item == 0) col = true;
if (media != null) {
plasmaSnippetCache.MediaSnippet ms;

18
source/dbtest.java
Unescape View File

@ -7,13 +7,13 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Random;
import javax.imageio.ImageIO;
import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroCache;
import de.anomic.kelondro.kelondroCloneableIterator;
import de.anomic.kelondro.kelondroFlexSplitTable;
import de.anomic.kelondro.kelondroFlexTable;
import de.anomic.kelondro.kelondroIndex;
@ -357,14 +357,14 @@ public class dbtest {
}
if (command.equals("list")) {
Iterator i = null;
kelondroCloneableIterator<kelondroRow.Entry> i = null;
if (table instanceof kelondroSplittedTree) i = ((kelondroSplittedTree) table).rows(true, null);
if (table instanceof kelondroTree) i = ((kelondroTree) table).rows(true, null);
if (table instanceof kelondroSQLTable) i = ((kelondroSQLTable) table).rows(true, null);
byte[][] row;
kelondroRow.Entry row;
while (i.hasNext()) {
row = (byte[][]) i.next();
for (int j = 0; j < row.length; j++) System.out.print(new String(row[j]) + ",");
row = i.next();
for (int j = 0; j < row.columns(); j++) System.out.print(row.getColString(j, null) + ",");
System.out.println();
}
}
@ -380,8 +380,8 @@ public class dbtest {
long r;
Long R;
int p, rc=0;
ArrayList ra = new ArrayList();
HashSet jcontrol = new HashSet();
ArrayList<Long> ra = new ArrayList<Long>();
HashSet<Long> jcontrol = new HashSet<Long>();
kelondroIntBytesMap kcontrol = new kelondroIntBytesMap(1, 0);
for (int i = 0; i < writeCount; i++) {
r = Math.abs(random.nextLong() % 1000);
@ -420,8 +420,8 @@ public class dbtest {
long r;
Long R;
int p, rc=0;
ArrayList ra = new ArrayList();
HashSet jcontrol = new HashSet();
ArrayList<Long> ra = new ArrayList<Long>();
HashSet<Long> jcontrol = new HashSet<Long>();
kelondroIntBytesMap kcontrol = new kelondroIntBytesMap(1, 0);
for (int i = 0; i < writeCount; i++) {
//if (i == 30) random = new Random(randomstart);

1
source/de/anomic/index/indexCachedRI.java
Unescape View File

@ -255,7 +255,6 @@ public class indexCachedRI implements indexRI {
return wordContainers(startHash, false, rot);
}
@SuppressWarnings("unchecked")
public kelondroCloneableIterator<indexContainer> wordContainers(String startHash, boolean ramOnly, boolean rot) {
kelondroCloneableIterator<indexContainer> i;
if (ramOnly) {

85
source/de/anomic/index/indexContainerOrder.java
Unescape View File

@ -26,25 +26,19 @@
package de.anomic.index;
import de.anomic.kelondro.kelondroNode;
import de.anomic.kelondro.kelondroAbstractOrder;
import de.anomic.kelondro.kelondroOrder;
import de.anomic.kelondro.kelondroRow;
import de.anomic.kelondro.kelondroRow.Entry;
public class indexContainerOrder implements kelondroOrder {
public class indexContainerOrder extends kelondroAbstractOrder<indexContainer> implements kelondroOrder<indexContainer> {
private kelondroOrder embeddedOrder;
private kelondroOrder<byte[]> embeddedOrder;
public indexContainerOrder(kelondroOrder embedOrder) {
public indexContainerOrder(kelondroOrder<byte[]> embedOrder) {
this.embeddedOrder = embedOrder;
}
public boolean wellformed(byte[] a) {
return embeddedOrder.wellformed(a);
}
public boolean wellformed(byte[] a, int astart, int alength) {
return embeddedOrder.wellformed(a, astart, alength);
public boolean wellformed(indexContainer a) {
return embeddedOrder.wellformed(a.getWordHash().getBytes());
}
public void direction(boolean ascending) {
@ -59,16 +53,13 @@ public class indexContainerOrder implements kelondroOrder {
return this.embeddedOrder.compare(a.getWordHash().getBytes(), b.getWordHash().getBytes());
}
public byte[] zero() {
return this.embeddedOrder.zero();
}
public void rotate(byte[] zero) {
this.embeddedOrder.rotate(zero);
public void rotate(indexContainer zero) {
this.embeddedOrder.rotate(zero.getWordHash().getBytes());
this.zero = new indexContainer(new String(this.embeddedOrder.zero()), zero);
}
public Object clone() {
return new indexContainerOrder((kelondroOrder) this.embeddedOrder.clone());
public kelondroOrder<indexContainer> clone() {
return new indexContainerOrder(this.embeddedOrder.clone());
}
public String signature() {
@ -79,57 +70,13 @@ public class indexContainerOrder implements kelondroOrder {
return this.embeddedOrder.cardinal(key);
}
public int compare(Object a, Object b) {
if ((a instanceof byte[]) && (b instanceof byte[])) {
return this.embeddedOrder.compare((byte[]) a, (byte[]) b);
}
if ((a instanceof Integer) && (b instanceof Integer)) {
return ((Integer) a).compareTo((Integer) b);
}
if ((a instanceof String) && (b instanceof String)) {
return this.embeddedOrder.compare(((String) a).getBytes(), ((String) b).getBytes());
}
if ((a instanceof kelondroNode) && (b instanceof kelondroNode)) {
return this.embeddedOrder.compare((kelondroNode) a, (kelondroNode) b);
}
if ((a instanceof kelondroRow.Entry) && (b instanceof kelondroRow.Entry)) {
return this.embeddedOrder.compare((kelondroRow.Entry) a, (kelondroRow.Entry) b);
}
if ((a instanceof indexContainer) && (b instanceof indexContainer)) {
return this.embeddedOrder.compare((indexContainer) a, (indexContainer) b);
}
Class<? extends Object> ac = a.getClass();
Class<? extends Object> bc = b.getClass();
if (ac.equals(bc)) {
throw new UnsupportedOperationException("compare not implemented for this object type: " + ac);
} else {
throw new UnsupportedOperationException("compare must be applied to same object classes; here a = " + ac + ", b = " + bc);
}
}
public int compare(String a, String b) {
return this.embeddedOrder.compare(a, b);
}
public int compare(kelondroNode a, kelondroNode b) {
return this.embeddedOrder.compare(a, b);
}
public int compare(Entry a, Entry b) {
return this.embeddedOrder.compare(a, b);
}
public int compare(byte[] a, byte[] b) {
return this.embeddedOrder.compare(a, b);
}
public int compare(byte[] a, int aoffset, int alength, byte[] b, int boffset, int blength) {
return this.embeddedOrder.compare(a, aoffset, alength, b, boffset, blength);
}
public boolean equals(kelondroOrder otherOrder) {
public boolean equals(kelondroOrder<indexContainer> otherOrder) {
if (!(otherOrder instanceof indexContainerOrder)) return false;
return this.embeddedOrder.equals(((indexContainerOrder) otherOrder).embeddedOrder);
}
public long cardinal(indexContainer key) {
return this.embeddedOrder.cardinal(key.getWordHash().getBytes());
}
}

22
source/de/anomic/index/indexRWIEntryOrder.java
Unescape View File

@ -39,7 +39,7 @@ import de.anomic.plasma.plasmaSearchRankingProcess;
import de.anomic.plasma.plasmaSearchRankingProfile;
import de.anomic.yacy.yacyURL;
public class indexRWIEntryOrder extends kelondroAbstractOrder implements kelondroOrder {
public class indexRWIEntryOrder extends kelondroAbstractOrder<indexRWIEntry> implements kelondroOrder<indexRWIEntry> {
private indexRWIVarEntry min, max;
private plasmaSearchRankingProfile ranking;
private kelondroMScoreCluster<String> doms;
@ -102,7 +102,7 @@ public class indexRWIEntryOrder extends kelondroAbstractOrder implements kelondr
if (this.doms.size() > 0) this.maxdomcount = this.doms.getMaxScore();
}
public Object clone() {
public kelondroOrder<indexRWIEntry> clone() {
return null;
}
@ -152,15 +152,9 @@ public class indexRWIEntryOrder extends kelondroAbstractOrder implements kelondr
return Long.MAX_VALUE - r; // returns a reversed number: the lower the number the better the ranking. This is used for simple sorting with a TreeMap
}
public int compare(byte[] a, byte[] b) {
long ca = cardinal(new indexRWIRowEntry(a));
long cb = cardinal(new indexRWIRowEntry(b));
return (ca > cb) ? 1 : (ca < cb) ? -1 : 0;
}
public int compare(byte[] a, int aoffset, int alength, byte[] b, int boffset, int blength) {
long ca = cardinal(new indexRWIRowEntry(a, aoffset, false));
long cb = cardinal(new indexRWIRowEntry(b, boffset, false));
public int compare(indexRWIEntry a, indexRWIEntry b) {
long ca = cardinal(a);
long cb = cardinal(b);
return (ca > cb) ? 1 : (ca < cb) ? -1 : 0;
}
@ -168,11 +162,7 @@ public class indexRWIEntryOrder extends kelondroAbstractOrder implements kelondr
return "rx";
}
public boolean wellformed(byte[] a) {
return true;
}
public boolean wellformed(byte[] a, int astart, int alength) {
public boolean wellformed(indexRWIEntry a) {
return true;
}

67
source/de/anomic/kelondro/kelondroAbstractOrder.java
Unescape View File

@ -45,83 +45,36 @@
package de.anomic.kelondro;
import de.anomic.index.indexContainer;
public abstract class kelondroAbstractOrder implements kelondroOrder {
public abstract class kelondroAbstractOrder<A> implements kelondroOrder<A> {
protected byte[] zero = null;
protected A zero = null;
protected boolean asc = true;
public abstract Object clone();
abstract public kelondroOrder<A> clone();
public A zero() {
return zero;
}
public void direction(boolean ascending) {
asc = ascending;
}
public long partition(byte[] key, int forks) {
public long partition(A key, int forks) {
final long d = (Long.MAX_VALUE / forks) + ((Long.MAX_VALUE % forks) + 1) / forks;
return cardinal(key) / d;
}
public int compare(Object a, Object b) {
if ((a instanceof byte[]) && (b instanceof byte[])) {
return compare((byte[]) a, (byte[]) b);
}
if ((a instanceof Integer) && (b instanceof Integer)) {
return compare((Integer) a, (Integer) b);
}
if ((a instanceof String) && (b instanceof String)) {
return compare(((String) a).getBytes(), ((String) b).getBytes());
}
if ((a instanceof kelondroNode) && (b instanceof kelondroNode)) {
return compare((kelondroNode) a, (kelondroNode) b);
}
if ((a instanceof kelondroRow.Entry) && (b instanceof kelondroRow.Entry)) {
return compare((kelondroRow.Entry) a, (kelondroRow.Entry) b);
}
Class<? extends Object> ac = a.getClass();
Class<? extends Object> bc = b.getClass();
if (ac.equals(bc)) {
throw new UnsupportedOperationException("compare not implemented for this object type: " + ac);
} else {
throw new UnsupportedOperationException("compare must be applied to same object classes; here a = " + ac + ", b = " + bc);
}
}
public int compare(kelondroNode a, kelondroNode b) {
return compare(a.getKey(), b.getKey());
}
public int compare(String a, String b) {
return compare(a.getBytes(), b.getBytes());
}
public int compare(kelondroRow.Entry a, kelondroRow.Entry b) {
return a.compareTo(b);
}
public int compare(indexContainer a, indexContainer b) {
return compare(a.getWordHash(), b.getWordHash());
}
public int compare(Integer a, Integer b) {
return a.compareTo(b);
}
public byte[] zero() {
return zero;
}
public void rotate(byte[] newzero) {
public void rotate(A newzero) {
this.zero = newzero;
}
public boolean equals(kelondroOrder otherOrder) {
public boolean equals(kelondroOrder<A> otherOrder) {
if (otherOrder == null) return false;
String thisSig = this.signature();
String otherSig = otherOrder.signature();
if ((thisSig == null) || (otherSig == null)) return false;
return thisSig.equals(otherSig);
}
}

8
source/de/anomic/kelondro/kelondroAbstractRecords.java
Unescape View File

@ -370,7 +370,7 @@ public abstract class kelondroAbstractRecords implements kelondroRecords {
}
assignRowdef(rowdef);
if (fileExisted) {
kelondroOrder oldOrder = readOrderType();
kelondroByteOrder oldOrder = readOrderType();
if ((oldOrder != null) && (!(oldOrder.equals(rowdef.objectOrder)))) {
writeOrderType(); // write new order type
//throw new IOException("wrong object order upon initialization. new order is " + rowdef.objectOrder.toString() + ", old order was " + oldOrder.toString());
@ -580,7 +580,7 @@ public abstract class kelondroAbstractRecords implements kelondroRecords {
} catch (IOException e) {}
}
private kelondroOrder readOrderType() {
private kelondroByteOrder readOrderType() {
try {
byte[] d = getDescription();
String s = new String(d).substring(0, 2);
@ -590,8 +590,8 @@ public abstract class kelondroAbstractRecords implements kelondroRecords {
}
}
public static kelondroOrder orderBySignature(String signature) {
kelondroOrder oo = null;
public static kelondroByteOrder orderBySignature(String signature) {
kelondroByteOrder oo = null;
if (oo == null) oo = kelondroNaturalOrder.bySignature(signature);
if (oo == null) oo = kelondroBase64Order.bySignature(signature);
if (oo == null) oo = new kelondroNaturalOrder(true);

11
source/de/anomic/kelondro/kelondroBase64Order.java
Unescape View File

@ -51,7 +51,7 @@ import java.util.Comparator;
import de.anomic.server.logging.serverLog;
public class kelondroBase64Order extends kelondroAbstractOrder implements kelondroOrder, kelondroCoding, Comparator<Object> {
public class kelondroBase64Order extends kelondroAbstractOrder<byte[]> implements kelondroByteOrder, kelondroCoding, Comparator<byte[]> {
protected static final char[] alpha_standard = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/".toCharArray();
protected static final char[] alpha_enhanced = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_".toCharArray();
@ -73,6 +73,8 @@ public class kelondroBase64Order extends kelondroAbstractOrder implements kelond
public static final kelondroBase64Order standardCoder = new kelondroBase64Order(true, true);
public static final kelondroBase64Order enhancedCoder = new kelondroBase64Order(true, false);
public static final Comparator<String> standardComparator = new kelondroByteOrder.StringOrder(standardCoder);
public static final Comparator<String> enhancedComparator = new kelondroByteOrder.StringOrder(enhancedCoder);
private boolean rfc1113compliant;
private final char[] alpha;
@ -95,9 +97,9 @@ public class kelondroBase64Order extends kelondroAbstractOrder implements kelond
return z;
}
public Object clone() {
public kelondroOrder<byte[]> clone() {
kelondroBase64Order o = new kelondroBase64Order(this.asc, this.rfc1113compliant);
o.rotate(this.zero);
o.rotate(zero);
return o;
}
@ -115,7 +117,7 @@ public class kelondroBase64Order extends kelondroAbstractOrder implements kelond
return true;
}
public final static kelondroOrder bySignature(String signature) {
public final static kelondroByteOrder bySignature(String signature) {
if (signature.equals("Bd")) return new kelondroBase64Order(false, false);
if (signature.equals("bd")) return new kelondroBase64Order(false, true);
if (signature.equals("Bu")) return new kelondroBase64Order(true, false);
@ -396,5 +398,4 @@ public class kelondroBase64Order extends kelondroAbstractOrder implements kelond
System.out.println(b64.decodeString(s[1]));
}
}
}

2
source/de/anomic/kelondro/kelondroBinSearch.java
Unescape View File

@ -51,7 +51,7 @@ public class kelondroBinSearch {
private byte[] chunks;
private int chunksize;
private int count;
private kelondroOrder objectOrder = new kelondroNaturalOrder(true);
private kelondroByteOrder objectOrder = new kelondroNaturalOrder(true);
public kelondroBinSearch(byte[] chunks, int chunksize) {
this.chunks = chunks;

4
source/de/anomic/kelondro/kelondroByteArray.java
Unescape View File

@ -192,11 +192,11 @@ public class kelondroByteArray {
this.offset = 0;
}
public int compareTo(kelondroByteArray b, kelondroOrder order) {
public int compareTo(kelondroByteArray b, kelondroByteOrder order) {
return order.compare(this.buffer, this.offset, this.length, b.buffer, b.offset, b.length);
}
public int compareTo(int aoffset, int alength, kelondroByteArray b, int boffset, int blength, kelondroOrder order) {
public int compareTo(int aoffset, int alength, kelondroByteArray b, int boffset, int blength, kelondroByteOrder order) {
return order.compare(this.buffer, this.offset + aoffset, alength, b.buffer, b.offset + boffset, blength);
}
}

50
source/de/anomic/kelondro/kelondroByteOrder.java
Unescape View File

@ -0,0 +1,50 @@
// kelondroByteOrder.java
// -----------------------------
// (C) 2008 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
// first published 10.01.2008 on http://yacy.net
//
// This is a part of YaCy, a peer-to-peer based web search engine
//
// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
// $LastChangedRevision: 1986 $
// $LastChangedBy: orbiter $
//
// LICENSE
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package de.anomic.kelondro;
import java.util.Comparator;
public interface kelondroByteOrder extends kelondroOrder<byte[]> {
public boolean wellformed(byte[] a, int start, int len);
public int compare(byte[] a, int astart, int alen, byte[] b, int bstart, int blen);
public static class StringOrder implements Comparator<String> {
public kelondroByteOrder baseOrder;
public StringOrder(kelondroByteOrder base) {
this.baseOrder = base;
}
public int compare(String s1, String s2) {
return baseOrder.compare(s1.getBytes(), s2.getBytes());
}
}
}

2
source/de/anomic/kelondro/kelondroBytesIntMap.java
Unescape View File

@ -38,7 +38,7 @@ public class kelondroBytesIntMap {
this.rowdef = ki.row();
}
public kelondroBytesIntMap(int keylength, kelondroOrder objectOrder, int space) {
public kelondroBytesIntMap(int keylength, kelondroByteOrder objectOrder, int space) {
this.rowdef = new kelondroRow(new kelondroColumn[]{new kelondroColumn("key", kelondroColumn.celltype_binary, kelondroColumn.encoder_bytes, keylength, "key"), new kelondroColumn("int c-4 {b256}")}, objectOrder, 0);
this.index = new kelondroRAMIndex(rowdef, space);
}

1
source/de/anomic/kelondro/kelondroCloneableMapIterator.java
Unescape View File

@ -54,7 +54,6 @@ public class kelondroCloneableMapIterator<E> implements kelondroCloneableIterato
this.last = null;
}
@SuppressWarnings("unchecked")
public kelondroCloneableMapIterator<E> clone(Object modifier) {
return new kelondroCloneableMapIterator(map, modifier);
}

12
source/de/anomic/kelondro/kelondroCollectionIndex.java
Unescape View File

@ -74,7 +74,7 @@ public class kelondroCollectionIndex {
private static final int idx_col_lastread = 6; // a time stamp, update time in days since 1.1.2000
private static final int idx_col_lastwrote = 7; // a time stamp, update time in days since 1.1.2000
private static kelondroRow indexRow(int keylength, kelondroOrder payloadOrder) {
private static kelondroRow indexRow(int keylength, kelondroByteOrder payloadOrder) {
return new kelondroRow(
"byte[] key-" + keylength + "," +
"int chunksize-4 {b256}," +
@ -111,7 +111,7 @@ public class kelondroCollectionIndex {
return new File(path, filenameStub + "." + lf + "." + cs + ".properties");
}
public kelondroCollectionIndex(File path, String filenameStub, int keyLength, kelondroOrder indexOrder,
public kelondroCollectionIndex(File path, String filenameStub, int keyLength, kelondroByteOrder indexOrder,
long preloadTime, int loadfactor, int maxpartitions, kelondroRow rowdef) throws IOException {
// the buffersize is number of bytes that are only used if the kelondroFlexTable is backed up with a kelondroTree
this.path = path;
@ -161,7 +161,7 @@ public class kelondroCollectionIndex {
}
}
private void openAllArrayFiles(boolean indexGeneration, kelondroOrder indexOrder) throws IOException {
private void openAllArrayFiles(boolean indexGeneration, kelondroByteOrder indexOrder) throws IOException {
String[] list = this.path.list();
kelondroFixedWidthArray array;
@ -222,7 +222,7 @@ public class kelondroCollectionIndex {
}
}
private kelondroIndex openIndexFile(File path, String filenameStub, kelondroOrder indexOrder,
private kelondroIndex openIndexFile(File path, String filenameStub, kelondroByteOrder indexOrder,
long preloadTime, int loadfactor, kelondroRow rowdef, int initialSpace) throws IOException {
// open/create index table
kelondroIndex theindex = new kelondroCache(new kelondroFlexTable(path, filenameStub + ".index", preloadTime, indexRow(keylength, indexOrder), initialSpace, true), true, false);
@ -246,7 +246,7 @@ public class kelondroCollectionIndex {
return theindex;
}
private kelondroFixedWidthArray openArrayFile(int partitionNumber, int serialNumber, kelondroOrder indexOrder, boolean create) throws IOException {
private kelondroFixedWidthArray openArrayFile(int partitionNumber, int serialNumber, kelondroByteOrder indexOrder, boolean create) throws IOException {
File f = arrayFile(path, filenameStub, loadfactor, payloadrow.objectsize, partitionNumber, serialNumber);
int load = arrayCapacity(partitionNumber);
kelondroRow rowdef = new kelondroRow(
@ -261,7 +261,7 @@ public class kelondroCollectionIndex {
return a;
}
private kelondroFixedWidthArray getArray(int partitionNumber, int serialNumber, kelondroOrder indexOrder, int chunksize) {
private kelondroFixedWidthArray getArray(int partitionNumber, int serialNumber, kelondroByteOrder indexOrder, int chunksize) {
String accessKey = partitionNumber + "-" + chunksize;
kelondroFixedWidthArray array = (kelondroFixedWidthArray) arrays.get(accessKey);
if (array != null) return array;

3
source/de/anomic/kelondro/kelondroDyn.java
Unescape View File

@ -71,7 +71,7 @@ public class kelondroDyn {
private kelondroRow rowdef;
public kelondroDyn(File file, boolean useNodeCache, boolean useObjectCache, long preloadTime, int key,
int nodesize, char fillChar, kelondroOrder objectOrder, boolean usetree, boolean writebuffer, boolean resetOnFail) {
int nodesize, char fillChar, kelondroByteOrder objectOrder, boolean usetree, boolean writebuffer, boolean resetOnFail) {
// creates or opens a dynamic tree
rowdef = new kelondroRow("byte[] key-" + (key + counterlen) + ", byte[] node-" + nodesize, objectOrder, 0);
kelondroIndex fbi;
@ -160,7 +160,6 @@ public class kelondroDyn {
nextKey = n();
}
@SuppressWarnings("unchecked")
public dynKeyIterator clone(Object modifier) {
return new dynKeyIterator(ri.clone(modifier));
}

14
source/de/anomic/kelondro/kelondroFlexSplitTable.java
Unescape View File

@ -45,11 +45,13 @@ public class kelondroFlexSplitTable implements kelondroIndex {
private kelondroRow rowdef;
private File path;
private String tablename;
private kelondroOrder<kelondroRow.Entry> entryOrder;
public kelondroFlexSplitTable(File path, String tablename, long preloadTime, kelondroRow rowdef, boolean resetOnFail) {
this.path = path;
this.tablename = tablename;
this.rowdef = rowdef;
this.entryOrder = new kelondroRow.EntryComparator(rowdef.objectOrder);
init(preloadTime, resetOnFail);
}
@ -192,7 +194,7 @@ public class kelondroFlexSplitTable implements kelondroIndex {
return (kelondroRow.Entry) keeper[1];
}
public synchronized void putMultiple(List rows) throws IOException {
public synchronized void putMultiple(List<kelondroRow.Entry> rows) throws IOException {
throw new UnsupportedOperationException("not yet implemented");
}
@ -287,8 +289,8 @@ public class kelondroFlexSplitTable implements kelondroIndex {
}
}
public synchronized kelondroCloneableIterator keys(boolean up, byte[] firstKey) throws IOException {
HashSet<kelondroCloneableIterator<?>> set = new HashSet<kelondroCloneableIterator<?>>();
public synchronized kelondroCloneableIterator<byte[]> keys(boolean up, byte[] firstKey) throws IOException {
HashSet<kelondroCloneableIterator<byte[]>> set = new HashSet<kelondroCloneableIterator<byte[]>>();
Iterator<kelondroIndex> i = tables.values().iterator();
while (i.hasNext()) {
set.add(i.next().keys(up, firstKey));
@ -296,13 +298,13 @@ public class kelondroFlexSplitTable implements kelondroIndex {
return kelondroMergeIterator.cascade(set, rowdef.objectOrder, kelondroMergeIterator.simpleMerge, up);
}
public synchronized kelondroCloneableIterator rows(boolean up, byte[] firstKey) throws IOException {
HashSet<kelondroCloneableIterator<?>> set = new HashSet<kelondroCloneableIterator<?>>();
public synchronized kelondroCloneableIterator<kelondroRow.Entry> rows(boolean up, byte[] firstKey) throws IOException {
HashSet<kelondroCloneableIterator<kelondroRow.Entry>> set = new HashSet<kelondroCloneableIterator<kelondroRow.Entry>>();
Iterator<kelondroIndex> i = tables.values().iterator();
while (i.hasNext()) {
set.add(i.next().rows(up, firstKey));
}
return kelondroMergeIterator.cascade(set, rowdef.objectOrder, kelondroMergeIterator.simpleMerge, up);
return kelondroMergeIterator.cascade(set, entryOrder, kelondroMergeIterator.simpleMerge, up);
}
public final int cacheObjectChunkSize() {

44
source/de/anomic/kelondro/kelondroFlexTable.java
Unescape View File

@ -153,52 +153,30 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr
}
System.out.print(" -ordering- ");
System.out.flush();
//int sbu = ri.size();
//if (ri.size() != sbu) serverLog.logSevere("kelondroFlexTable.initializeRamIndex: " + tablename, "; size before uniq = " + sbu + ", after uniq = " + ri.size());
return ri;
}
private kelondroIndex initializeTreeIndex(File indexfile, long preloadTime, kelondroOrder objectOrder) throws IOException {
kelondroIndex treeindex = new kelondroCache(new kelondroTree(indexfile, true, preloadTime, treeIndexRow(rowdef.primaryKeyLength, objectOrder), 2, 80), true, false);
Iterator<kelondroNode> content = super.col[0].contentNodes(-1);
kelondroNode node;
kelondroRow.Entry indexentry;
int i, c = 0, all = super.col[0].size();
long start = System.currentTimeMillis();
long last = start;
while (content.hasNext()) {
node = content.next();
i = node.handle().hashCode();
indexentry = treeindex.row().newEntry();
indexentry.setCol(0, node.getValueRow());
indexentry.setCol(1, i);
treeindex.addUnique(indexentry);
c++;
if (System.currentTimeMillis() - last > 30000) {
System.out.println(".. generated " + c + "/" + all + " entries, " + ((System.currentTimeMillis() - start) / c * (all - c) / 60000) + " minutes remaining");
System.out.flush();
last = System.currentTimeMillis();
}
}
return treeindex;
}
private static final kelondroRow treeIndexRow(int keywidth, kelondroOrder objectOrder) {
return new kelondroRow("byte[] key-" + keywidth + ", int reference-4 {b256}", objectOrder, 0);
}
public synchronized kelondroRow.Entry get(byte[] key) throws IOException {
if (index == null) return null; // case may happen during shutdown
int pos = index.geti(key);
assert this.size() == index.size() : "content.size() = " + this.size() + ", index.size() = " + index.size();
if (pos < 0) return null;
// i may be greater than this.size(), because this table may have deleted entries
// pos may be greater than this.size(), because this table may have deleted entries
// the deleted entries are subtracted from the 'real' tablesize,
// so the size may be smaller than an index to a row entry
/*if (kelondroAbstractRecords.debugmode) {
kelondroRow.Entry result = super.get(pos);
assert result != null;
assert rowdef.objectOrder.compare(result.getPrimaryKeyBytes(), key) == 0 : "key and row does not match; key = " + serverLog.arrayList(key, 0, key.length) + " row.key = " + serverLog.arrayList(result.getPrimaryKeyBytes(), 0, rowdef.primaryKeyLength);
return result;
} else {*/
// assume that the column for the primary key is 0,
// and the column 0 is stored in a file only for that column
// then we don't need to lookup from that file, because we already know the value (it's the key)
kelondroRow.Entry result = super.getOmitCol0(pos, key);
assert result != null;
return result;
//}
}
public synchronized void putMultiple(List<kelondroRow.Entry> rows) throws IOException {
@ -298,7 +276,7 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr
assert this.size() == index.size() : "content.size() = " + this.size() + ", index.size() = " + index.size();
return null;
}
kelondroRow.Entry r = super.get(i);
kelondroRow.Entry r = super.getOmitCol0(i, key);
if (r == null) {
serverLog.logSevere("kelondroFlexTable", "remove(): index failure; the index pointed to a cell which is empty. content.size() = " + this.size() + ", index.size() = " + ((index == null) ? 0 : index.size()));
// patch bug ***** FIND CAUSE! (see also: put)

17
source/de/anomic/kelondro/kelondroFlexWidthArray.java
Unescape View File

@ -307,6 +307,7 @@ public class kelondroFlexWidthArray implements kelondroArray {
public synchronized kelondroRow.Entry get(int index) throws IOException {
kelondroRow.Entry e = col[0].getIfValid(index);
//assert e != null;
if (e == null) return null; // probably a deleted entry
kelondroRow.Entry p = rowdef.newEntry();
p.setCol(0, e.getColBytes(0));
@ -321,6 +322,22 @@ public class kelondroFlexWidthArray implements kelondroArray {
return p;
}
public synchronized kelondroRow.Entry getOmitCol0(int index, byte[] col0) throws IOException {
assert col[0].row().columns() == 1;
kelondroRow.Entry p = rowdef.newEntry();
kelondroRow.Entry e;
p.setCol(0, col0);
int r = 1;
while (r < rowdef.columns()) {
e = col[r].get(index);
for (int i = 0; i < col[r].row().columns(); i++) {
p.setCol(r + i, e.getColBytes(i));
}
r = r + col[r].row().columns();
}
return p;
}
public synchronized void remove(int index) throws IOException {
int r = 0;

10
source/de/anomic/kelondro/kelondroIntBytesMap.java
Unescape View File

@ -40,11 +40,13 @@ public class kelondroIntBytesMap {
private kelondroRow rowdef;
private kelondroRowSet index0, index1;
private kelondroOrder<kelondroRow.Entry> entryOrder;
public kelondroIntBytesMap(int payloadSize, int initSize) {
rowdef = new kelondroRow("Cardinal key-4 {b256}, byte[] payload-" + payloadSize, kelondroNaturalOrder.naturalOrder, 0);
index0 = new kelondroRowSet(rowdef, initSize);
index1 = null;
this.rowdef = new kelondroRow("Cardinal key-4 {b256}, byte[] payload-" + payloadSize, kelondroNaturalOrder.naturalOrder, 0);
this.index0 = new kelondroRowSet(rowdef, initSize);
this.index1 = null;
this.entryOrder = new kelondroRow.EntryComparator(rowdef.objectOrder);
}
public long memoryNeededForGrow() {
@ -193,7 +195,7 @@ public class kelondroIntBytesMap {
return new kelondroMergeIterator<kelondroRow.Entry>(
index0.rows(true, null),
index1.rows(true, null),
kelondroRow.entryComparator,
entryOrder,
kelondroMergeIterator.simpleMerge,
true);
}

2
source/de/anomic/kelondro/kelondroMScoreCluster.java
Unescape View File

@ -288,7 +288,6 @@ public final class kelondroMScoreCluster<E> {
return getScores(maxCount, up, Integer.MIN_VALUE, Integer.MAX_VALUE);
}
@SuppressWarnings("unchecked")
public synchronized E[] getScores(int maxCount, boolean up, int minScore, int maxScore) {
if (maxCount > refkeyDB.size()) maxCount = refkeyDB.size();
E[] s = (E[]) new Object[maxCount];
@ -325,7 +324,6 @@ public final class kelondroMScoreCluster<E> {
E n;
int min, max;
@SuppressWarnings("unchecked")
public komplexScoreIterator(boolean up, int minScore, int maxScore) {
this.up = up;
this.min = minScore;

159
source/de/anomic/kelondro/kelondroMSetTools.java
Unescape View File

@ -62,12 +62,6 @@ public class kelondroMSetTools {
// ------------------------------------------------------------------------------------------------
// helper methods
private static int compare(Object a, Object b, Comparator c) {
if (c != null) return c.compare(a,b);
if ((a instanceof String) && (b instanceof String)) return ((String) a).compareTo((String) b);
throw new ClassCastException();
}
public static int log2a(int x) {
// this computes 1 + log2
// it is the number of bits in x, not the logarithmus by 2
@ -178,7 +172,7 @@ public class kelondroMSetTools {
Map.Entry<A, B> mentry1 = mi1.next();
Map.Entry<A, B> mentry2 = mi2.next();
while (true) {
c = compare(mentry1.getKey(), mentry2.getKey(), comp);
c = comp.compare(mentry1.getKey(), mentry2.getKey());
if (c < 0) {
if (mi1.hasNext()) mentry1 = mi1.next(); else break;
} else if (c > 0) {
@ -198,11 +192,11 @@ public class kelondroMSetTools {
}
// now the same for set-set
public static TreeSet joinConstructive(TreeSet set1, TreeSet set2) {
public static <A> TreeSet<A> joinConstructive(TreeSet<A> set1, TreeSet<A> set2) {
// comparators must be equal
if ((set1 == null) || (set2 == null)) return null;
if (set1.comparator() != set2.comparator()) return null;
if ((set1.size() == 0) || (set2.size() == 0)) return new TreeSet(set1.comparator());
if ((set1.size() == 0) || (set2.size() == 0)) return new TreeSet<A>(set1.comparator());
// decide which method to use
int high = ((set1.size() > set2.size()) ? set1.size() : set2.size());
@ -218,10 +212,10 @@ public class kelondroMSetTools {
return joinConstructiveByEnumeration(set1, set2);
}
private static TreeSet joinConstructiveByTest(TreeSet small, TreeSet large) {
Iterator mi = small.iterator();
TreeSet result = new TreeSet(small.comparator());
Object o;
private static <A> TreeSet<A> joinConstructiveByTest(TreeSet<A> small, TreeSet<A> large) {
Iterator<A> mi = small.iterator();
TreeSet<A> result = new TreeSet<A>(small.comparator());
A o;
while (mi.hasNext()) {
o = mi.next();
if (large.contains(o)) result.add(o);
@ -229,18 +223,18 @@ public class kelondroMSetTools {
return result;
}
private static TreeSet joinConstructiveByEnumeration(TreeSet set1, TreeSet set2) {
private static <A> TreeSet<A> joinConstructiveByEnumeration(TreeSet<A> set1, TreeSet<A> set2) {
// implement pairvise enumeration
Comparator comp = set1.comparator();
Iterator mi = set1.iterator();
Iterator si = set2.iterator();
TreeSet result = new TreeSet(set1.comparator());
Comparator<? super A> comp = set1.comparator();
Iterator<A> mi = set1.iterator();
Iterator<A> si = set2.iterator();
TreeSet<A> result = new TreeSet<A>(set1.comparator());
int c;
if ((mi.hasNext()) && (si.hasNext())) {
Object mobj = mi.next();
Object sobj = si.next();
A mobj = mi.next();
A sobj = si.next();
while (true) {
c = compare(mobj, sobj, comp);
c = comp.compare(mobj, sobj);
if (c < 0) {
if (mi.hasNext()) mobj = mi.next(); else break;
} else if (c > 0) {
@ -256,7 +250,7 @@ public class kelondroMSetTools {
}
// now the same for set-set
public static boolean anymatch(TreeSet set1, TreeSet set2) {
public static <A> boolean anymatch(TreeSet<A> set1, TreeSet<A> set2) {
// comparators must be equal
if ((set1 == null) || (set2 == null)) return false;
if (set1.comparator() != set2.comparator()) return false;
@ -276,9 +270,9 @@ public class kelondroMSetTools {
return anymatchByEnumeration(set1, set2);
}
private static boolean anymatchByTest(TreeSet small, TreeSet large) {
Iterator mi = small.iterator();
Object o;
private static <A> boolean anymatchByTest(TreeSet<A> small, TreeSet<A> large) {
Iterator<A> mi = small.iterator();
A o;
while (mi.hasNext()) {
o = mi.next();
if (large.contains(o)) return true;
@ -286,17 +280,17 @@ public class kelondroMSetTools {
return false;
}
private static boolean anymatchByEnumeration(TreeSet set1, TreeSet set2) {
private static <A> boolean anymatchByEnumeration(TreeSet<A> set1, TreeSet<A> set2) {
// implement pairvise enumeration
Comparator comp = set1.comparator();
Iterator mi = set1.iterator();
Iterator si = set2.iterator();
Comparator<? super A> comp = set1.comparator();
Iterator<A> mi = set1.iterator();
Iterator<A> si = set2.iterator();
int c;
if ((mi.hasNext()) && (si.hasNext())) {
Object mobj = mi.next();
Object sobj = si.next();
A mobj = mi.next();
A sobj = si.next();
while (true) {
c = compare(mobj, sobj, comp);
c = comp.compare(mobj, sobj);
if (c < 0) {
if (mi.hasNext()) mobj = mi.next(); else break;
} else if (c > 0) {
@ -312,7 +306,7 @@ public class kelondroMSetTools {
// ------------------------------------------------------------------------------------------------
// exclude
public static TreeMap excludeConstructive(TreeMap map, TreeSet set) {
public static <A, B> TreeMap<A, B> excludeConstructive(TreeMap<A, B> map, TreeSet<A> set) {
// comparators must be equal
if (map == null) return null;
if (set == null) return map;
@ -322,10 +316,10 @@ public class kelondroMSetTools {
// return excludeConstructiveByEnumeration(map, set);
}
private static TreeMap excludeConstructiveByTestMapInSet(TreeMap map, Set set) {
Iterator mi = map.keySet().iterator();
TreeMap result = new TreeMap(map.comparator());
Object o;
private static <A, B> TreeMap<A, B> excludeConstructiveByTestMapInSet(TreeMap<A, B> map, TreeSet<A> set) {
Iterator<A> mi = map.keySet().iterator();
TreeMap<A, B> result = new TreeMap<A, B>(map.comparator());
A o;
while (mi.hasNext()) {
o = mi.next();
if (!(set.contains(o))) result.put(o, map.get(o));
@ -333,47 +327,7 @@ public class kelondroMSetTools {
return result;
}
/*
private static TreeMap excludeConstructiveByEnumeration(TreeMap map, TreeSet set) {
// returns map without the elements in set
// enumerates objects
Comparator comp = map.comparator();
Iterator mi = map.keySet().iterator();
Iterator si = set.iterator();
TreeMap result = new TreeMap(map.comparator());
int c;
if ((mi.hasNext()) && (si.hasNext())) {
Object mobj = mi.next();
Object sobj = si.next();
while (true) {
c = compare(mobj, sobj, comp);
if (c < 0) {
result.put(mobj, map.get(mobj));
if (mi.hasNext()) mobj = mi.next();
else break;
} else if (c > 0) {
if (si.hasNext()) sobj = si.next();
else break;
} else {
if (mi.hasNext()) mobj = mi.next();
else break;
if (si.hasNext()) sobj = si.next();
else {
// final flush
result.put(mobj, map.get(mobj));
while (mi.hasNext()) {
mobj = mi.next();
result.put(mobj, map.get(mobj));
}
break;
}
}
}
}
return result;
}
*/
public static void excludeDestructive(TreeMap map, TreeSet set) {
public static <A, B> void excludeDestructive(TreeMap<A, B> map, TreeSet<A> set) {
// comparators must be equal
if (map == null) return;
if (set == null) return;
@ -386,18 +340,18 @@ public class kelondroMSetTools {
excludeDestructiveByTestSetInMap(map, set);
}
private static void excludeDestructiveByTestMapInSet(TreeMap map, TreeSet set) {
Iterator mi = map.keySet().iterator();
private static <A, B> void excludeDestructiveByTestMapInSet(TreeMap<A, B> map, TreeSet<A> set) {
Iterator<A> mi = map.keySet().iterator();
while (mi.hasNext()) if (set.contains(mi.next())) mi.remove();
}
private static void excludeDestructiveByTestSetInMap(TreeMap map, TreeSet set) {
Iterator si = set.iterator();
private static <A, B> void excludeDestructiveByTestSetInMap(TreeMap<A, B> map, TreeSet<A> set) {
Iterator<A> si = set.iterator();
while (si.hasNext()) map.remove(si.next());
}
// and the same again with set-set
public static void excludeDestructive(TreeSet set1, TreeSet set2) {
public static <A> void excludeDestructive(TreeSet<A> set1, TreeSet<A> set2) {
// comparators must be equal
if (set1 == null) return;
if (set2 == null) return;
@ -410,20 +364,20 @@ public class kelondroMSetTools {
excludeDestructiveByTestLargeInSmall(set1, set2);
}
private static void excludeDestructiveByTestSmallInLarge(TreeSet small, TreeSet large) {
Iterator mi = small.iterator();
private static <A> void excludeDestructiveByTestSmallInLarge(TreeSet<A> small, TreeSet<A> large) {
Iterator<A> mi = small.iterator();
while (mi.hasNext()) if (large.contains(mi.next())) mi.remove();
}
private static void excludeDestructiveByTestLargeInSmall(TreeSet large, TreeSet small) {
Iterator si = small.iterator();
private static <A> void excludeDestructiveByTestLargeInSmall(TreeSet<A> large, TreeSet<A> small) {
Iterator<A> si = small.iterator();
while (si.hasNext()) large.remove(si.next());
}
// ------------------------------------------------------------------------------------------------
public static TreeMap loadMap(String filename, String sep) {
TreeMap map = new TreeMap();
public static TreeMap<String, String> loadMap(String filename, String sep) {
TreeMap<String, String> map = new TreeMap<String, String>();
BufferedReader br = null;
try {
br = new BufferedReader(new InputStreamReader(new FileInputStream(filename)));
@ -441,8 +395,8 @@ public class kelondroMSetTools {
return map;
}
public static TreeMap /* <String,ArrayList<String>> */ loadMapMultiValsPerKey(String filename, String sep) {
TreeMap map = new TreeMap();
public static TreeMap<String, ArrayList<String>> loadMapMultiValsPerKey(String filename, String sep) {
TreeMap<String, ArrayList<String>> map = new TreeMap<String, ArrayList<String>>();
BufferedReader br = null;
try {
br = new BufferedReader(new InputStreamReader(new FileInputStream(filename)));
@ -453,9 +407,8 @@ public class kelondroMSetTools {
if ((line.length() > 0) && (!(line.startsWith("#"))) && ((pos = line.indexOf(sep)) > 0)) {
key = line.substring(0, pos).trim().toLowerCase();
value = line.substring(pos + sep.length()).trim();
if (!map.containsKey(key))
map.put(key, new ArrayList());
((ArrayList)map.get(key)).add(value);
if (!map.containsKey(key)) map.put(key, new ArrayList<String>());
map.get(key).add(value);
}
}
} catch (IOException e) {
@ -465,8 +418,8 @@ public class kelondroMSetTools {
return map;
}
public static TreeSet loadList(File file, Comparator c) {
TreeSet list = new TreeSet(c);
public static TreeSet<String> loadList(File file, Comparator<String> c) {
TreeSet<String> list = new TreeSet<String>(c);
if (!(file.exists())) return list;
BufferedReader br = null;
@ -485,12 +438,12 @@ public class kelondroMSetTools {
return list;
}
public static String setToString(Set set, char separator) {
Iterator i = set.iterator();
public static String setToString(Set<String> set, char separator) {
Iterator<String> i = set.iterator();
StringBuffer sb = new StringBuffer(set.size() * 7);
if (i.hasNext()) sb.append(i.next().toString());
if (i.hasNext()) sb.append(i.next());
while (i.hasNext()) {
sb.append(separator).append(i.next().toString());
sb.append(separator).append(i.next());
}
return new String(sb);
}
@ -499,8 +452,8 @@ public class kelondroMSetTools {
public static void main(String[] args) {
TreeMap m = new TreeMap();
TreeMap s = new TreeMap();
TreeMap<String, String> m = new TreeMap<String, String>();
TreeMap<String, String> s = new TreeMap<String, String>();
m.put("a", "a");
m.put("x", "x");
m.put("f", "f");
@ -530,7 +483,7 @@ public class kelondroMSetTools {
System.out.println("Join=" + joinConstructiveByTest(m, s, true));
System.out.println("Join=" + joinConstructiveByTest(m, s, true));
System.out.println("Join=" + joinConstructive(m, s, true));
System.out.println("Exclude=" + excludeConstructiveByTestMapInSet(m, s.keySet()));
//System.out.println("Exclude=" + excludeConstructiveByTestMapInSet(m, s.keySet()));
/*
for (int low = 0; low < 10; low++)

1
source/de/anomic/kelondro/kelondroMapObjects.java
Unescape View File

@ -45,7 +45,6 @@ public class kelondroMapObjects extends kelondroObjects {
this(dyn, cachesize, null, null, null, null, null);
}
@SuppressWarnings("unchecked")
public kelondroMapObjects(kelondroDyn dyn, int cachesize, String[] sortfields, String[] longaccfields, String[] doubleaccfields, Method externalInitializer, Object externalHandler) {
super(dyn, cachesize);

6
source/de/anomic/kelondro/kelondroMapTable.java
Unescape View File

@ -64,19 +64,19 @@ public class kelondroMapTable {
}
public void declareMaps(
String tablename, int keysize, kelondroOrder objectOrder, int nodesize, int cacheslots,
String tablename, int keysize, kelondroByteOrder objectOrder, int nodesize, int cacheslots,
char fillChar, boolean resetOnFail) {
declareMaps(tablename, keysize, objectOrder, nodesize, cacheslots, null, null, null, fillChar, resetOnFail);
}
public void declareMaps(
String tablename, int keysize, kelondroOrder objectOrder, int nodesize, int cacheslots,
String tablename, int keysize, kelondroByteOrder objectOrder, int nodesize, int cacheslots,
String[] sortfields, String[] longaccfields, String[] doubleaccfields, char fillChar, boolean resetOnFail) {
declareMaps(tablename, keysize, objectOrder, nodesize, cacheslots, sortfields, longaccfields, doubleaccfields, fillChar, 0, resetOnFail);
}
public void declareMaps(
String tablename, int keysize, kelondroOrder objectOrder, int nodesize, int cacheslots,
String tablename, int keysize, kelondroByteOrder objectOrder, int nodesize, int cacheslots,
String[] sortfields, String[] longaccfields, String[] doubleaccfields, char fillChar,
long preloadTime, boolean resetOnFail) {
if (mTables.containsKey(tablename)) throw new RuntimeException("kelondroTables.declareMap: table '" + tablename + "' declared twice.");

13
source/de/anomic/kelondro/kelondroMergeIterator.java
Unescape View File

@ -67,7 +67,6 @@ public class kelondroMergeIterator<E> implements kelondroCloneableIterator<E> {
nextb();
}
@SuppressWarnings("unchecked")
public kelondroMergeIterator<E> clone(Object modifier) {
return new kelondroMergeIterator<E>(a.clone(modifier), b.clone(modifier), comp, merger, up);
}
@ -100,7 +99,6 @@ public class kelondroMergeIterator<E> implements kelondroCloneableIterator<E> {
return (na != null) || (nb != null);
}
@SuppressWarnings("unchecked")
public E next() {
E s;
if (na == null) {
@ -148,21 +146,20 @@ public class kelondroMergeIterator<E> implements kelondroCloneableIterator<E> {
throw new java.lang.UnsupportedOperationException("merge does not support remove");
}
public static kelondroCloneableIterator<?> cascade(Set<kelondroCloneableIterator<?>> /*of*/ iterators, Comparator<?> c, Method merger, boolean up) {
public static <A> kelondroCloneableIterator<A> cascade(Set<kelondroCloneableIterator<A>> /*of*/ iterators, kelondroOrder<A> c, Method merger, boolean up) {
// this extends the ability to combine two iterators
// to the abiliy of combining a set of iterators
if (iterators == null) return null;
if (iterators.size() == 0) return null;
return cascade(iterators.iterator(), c, merger, up);
return cascade((Set<kelondroCloneableIterator<A>>) iterators.iterator(), c, merger, up);
}
@SuppressWarnings("unchecked")
private static kelondroCloneableIterator<?> cascade(Iterator<?> /*of*/ iiterators, Comparator<?> c, Method merger, boolean up) {
private static <A> kelondroCloneableIterator<A> cascade(Iterator<A> /*of*/ iiterators, kelondroOrder<A> c, Method merger, boolean up) {
if (iiterators == null) return null;
if (!(iiterators.hasNext())) return null;
kelondroCloneableIterator<?> one = (kelondroCloneableIterator<?>) iiterators.next();
kelondroCloneableIterator<A> one = (kelondroCloneableIterator<A>) iiterators.next();
if (!(iiterators.hasNext())) return one;
return new kelondroMergeIterator(one, cascade(iiterators, c, merger, up), c, merger, up);
return new kelondroMergeIterator<A>(one, cascade(iiterators, c, merger, up), c, merger, up);
}
public static Method simpleMerge = null;

12
source/de/anomic/kelondro/kelondroNaturalOrder.java
Unescape View File

@ -47,10 +47,10 @@ package de.anomic.kelondro;
import java.util.Comparator;
public final class kelondroNaturalOrder extends kelondroAbstractOrder implements kelondroOrder, Comparator<Object>, Cloneable {
public static final kelondroOrder naturalOrder = new kelondroNaturalOrder(true);
public final class kelondroNaturalOrder extends kelondroAbstractOrder<byte[]> implements kelondroByteOrder, Comparator<byte[]>, Cloneable {
public static final kelondroByteOrder naturalOrder = new kelondroNaturalOrder(true);
public static final Comparator<String> naturalComparator = new kelondroByteOrder.StringOrder(naturalOrder);
public kelondroNaturalOrder(boolean ascending) {
this.asc = ascending;
this.zero = null;
@ -64,13 +64,13 @@ public final class kelondroNaturalOrder extends kelondroAbstractOrder implements
return true;
}
public final Object clone() {
public final kelondroOrder<byte[]> clone() {
kelondroNaturalOrder o = new kelondroNaturalOrder(this.asc);
o.rotate(this.zero);
return o;
}
public final static kelondroOrder bySignature(String signature) {
public final static kelondroByteOrder bySignature(String signature) {
if (signature.equals("nd")) return new kelondroNaturalOrder(false);
if (signature.equals("nu")) return new kelondroNaturalOrder(true);
return null;
@ -189,7 +189,7 @@ public final class kelondroNaturalOrder extends kelondroAbstractOrder implements
byte[] t = new byte[12];
for (int i = 0; i < 12; i++) t[i] = (byte) 255;
t[0] = (byte) 127;
kelondroOrder o = new kelondroNaturalOrder(true);
kelondroOrder<byte[]> o = new kelondroNaturalOrder(true);
System.out.println(o.partition(t, 16));
}

27
source/de/anomic/kelondro/kelondroOrder.java
Unescape View File

@ -47,34 +47,25 @@ package de.anomic.kelondro;
import java.util.Comparator;
import de.anomic.index.indexContainer;
public interface kelondroOrder<A> extends Comparator<A> {
public interface kelondroOrder extends Comparator<Object> {
public boolean wellformed(A a); // returns true if and only if a has only characters that belong to the implemented order
public boolean wellformed(byte[] a); // returns true if and only if a has only characters that belong to the implemented order
public boolean wellformed(byte[] a, int astart, int alength);
public Object clone();
public kelondroOrder<A> clone();
public void direction(boolean ascending); // the ordering direction can be changed at any time
public String signature(); // returns a signature String so that different orderings have different signatures
public long partition(byte[] key, int forkes);
public long cardinal(byte[] key); // returns a cardinal number in the range of 0 .. Long.MAX_VALUE
public long partition(A key, int forkes);
public int compare(byte[] a, byte[] b);
public int compare(String a, String b);
public int compare(kelondroNode a, kelondroNode b);
public int compare(indexContainer a, indexContainer b);
public int compare(kelondroRow.Entry a, kelondroRow.Entry b);
public long cardinal(A key); // returns a cardinal number in the range of 0 .. Long.MAX_VALUE
public int compare(byte[] a, int aoffset, int alength, byte[] b, int boffset, int blength);
public int compare(A a, A b);
public byte[] zero(); // returns the zero point of the Ordering; null if not defined
public A zero(); // returns the zero point of the Ordering; null if not defined
public void rotate(byte[] zero); // defines that the ordering rotates, and sets the zero point for the rotation
public void rotate(A zero); // defines that the ordering rotates, and sets the zero point for the rotation
public boolean equals(kelondroOrder o); // used to compare different order objects; they may define the same ordering
public boolean equals(kelondroOrder<A> o); // used to compare different order objects; they may define the same ordering
}

10
source/de/anomic/kelondro/kelondroRAMIndex.java
Unescape View File

@ -35,9 +35,11 @@ public class kelondroRAMIndex implements kelondroIndex {
private kelondroRow rowdef;
private kelondroRowSet index0, index1;
private kelondroRow.EntryComparator entryComparator;
public kelondroRAMIndex(kelondroRow rowdef, int initialspace) {
this.rowdef = rowdef;
this.entryComparator = new kelondroRow.EntryComparator(rowdef.objectOrder);
reset(initialspace);
}
@ -153,7 +155,6 @@ public class kelondroRAMIndex implements kelondroIndex {
return index0.size() + index1.size();
}
@SuppressWarnings("unchecked")
public synchronized kelondroCloneableIterator<byte[]> keys(boolean up, byte[] firstKey) throws IOException {
// returns the key-iterator of the underlying kelondroIndex
if (index1 == null) {
@ -169,7 +170,7 @@ public class kelondroRAMIndex implements kelondroIndex {
return index1.keys(up, firstKey);
}
//assert consistencyAnalysis0() : "consistency problem: " + consistencyAnalysis();
return new kelondroMergeIterator(
return new kelondroMergeIterator<byte[]>(
index0.keys(up, firstKey),
index1.keys(up, firstKey),
rowdef.objectOrder,
@ -177,7 +178,6 @@ public class kelondroRAMIndex implements kelondroIndex {
true);
}
@SuppressWarnings("unchecked")
public synchronized kelondroCloneableIterator<kelondroRow.Entry> rows(boolean up, byte[] firstKey) throws IOException {
// returns the row-iterator of the underlying kelondroIndex
if (index1 == null) {
@ -193,10 +193,10 @@ public class kelondroRAMIndex implements kelondroIndex {
return index1.rows(up, firstKey);
}
//assert consistencyAnalysis0() : "consistency problem: " + consistencyAnalysis();
return new kelondroMergeIterator(
return new kelondroMergeIterator<kelondroRow.Entry>(
index0.rows(up, firstKey),
index1.rows(up, firstKey),
rowdef.objectOrder,
entryComparator,
kelondroMergeIterator.simpleMerge,
true);
}

1
source/de/anomic/kelondro/kelondroRotateIterator.java
Unescape View File

@ -38,7 +38,6 @@ public class kelondroRotateIterator<E> implements kelondroCloneableIterator<E> {
this.clone = (kelondroCloneableIterator<E>) a.clone(modifier);
}
@SuppressWarnings("unchecked")
public kelondroRotateIterator<E> clone(Object modifier) {
return new kelondroRotateIterator<E>(a, modifier);
}

35
source/de/anomic/kelondro/kelondroRow.java
Unescape View File

@ -41,16 +41,16 @@ public final class kelondroRow {
protected kelondroColumn[] row;
protected int[] colstart;
protected kelondroOrder objectOrder;
protected kelondroByteOrder objectOrder;
public int objectsize;
public int primaryKeyIndex, primaryKeyLength;
protected Map<String, Object[]> nickref = null; // a mapping from nicknames to Object[2]{kelondroColumn, Integer(colstart)}
public kelondroRow(kelondroColumn[] row, kelondroOrder objectOrder, int primaryKey) {
public kelondroRow(kelondroColumn[] row, kelondroByteOrder objectOrder, int primaryKey) {
assert objectOrder != null;
this.objectOrder = objectOrder;
this.row = row;
assert (objectOrder != null);
this.objectOrder = objectOrder;
this.colstart = new int[row.length];
this.objectsize = 0;
for (int i = 0; i < row.length; i++) {
@ -61,7 +61,7 @@ public final class kelondroRow {
this.primaryKeyLength = (primaryKey < 0) ? this.objectsize : row[primaryKeyIndex].cellwidth;
}
public kelondroRow(String structure, kelondroOrder objectOrder, int primaryKey) {
public kelondroRow(String structure, kelondroByteOrder objectOrder, int primaryKey) {
assert (objectOrder != null);
this.objectOrder = objectOrder;
// define row with row syntax
@ -94,14 +94,14 @@ public final class kelondroRow {
this.primaryKeyLength = (primaryKey < 0) ? this.objectsize : row[primaryKeyIndex].cellwidth;
}
public final void setOrdering(kelondroOrder objectOrder, int primaryKey) {
public final void setOrdering(kelondroByteOrder objectOrder, int primaryKey) {
assert (objectOrder != null);
this.objectOrder = objectOrder;
this.primaryKeyIndex = primaryKey;
this.primaryKeyLength = (primaryKey < 0) ? this.objectsize : row[primaryKeyIndex].cellwidth;
}
public final kelondroOrder getOrdering() {
public final kelondroOrder<byte[]> getOrdering() {
return this.objectOrder;
}
@ -191,14 +191,33 @@ public final class kelondroRow {
return new EntryIndex(rowinstance, index);
}
public static final Comparator<Entry> entryComparator = new EntryComparator();
public static class EntryComparator extends kelondroAbstractOrder<Entry> implements kelondroOrder<Entry>, Comparator<Entry> {
public static class EntryComparator implements Comparator<Entry> {
kelondroByteOrder base;
public EntryComparator(kelondroByteOrder baseOrder) {
this.base = baseOrder;
}
public int compare(Entry a, Entry b) {
return a.compareTo(b);
}
public kelondroOrder<Entry> clone() {
return new EntryComparator(base);
}
public long cardinal(Entry key) {
return base.cardinal(key.getPrimaryKeyBytes());
}
public String signature() {
return base.signature();
}
public boolean wellformed(Entry a) {
return base.wellformed(a.getPrimaryKeyBytes());
}
}
public class Entry implements Comparable<Entry> {

2
source/de/anomic/kelondro/kelondroRowCollection.java
Unescape View File

@ -98,7 +98,7 @@ public class kelondroRowCollection {
this.lastTimeRead = (exportedCollection.getColLong(exp_last_read) + 10957) * day;
this.lastTimeWrote = (exportedCollection.getColLong(exp_last_wrote) + 10957) * day;
String sortOrderKey = exportedCollection.getColString(exp_order_type, null);
kelondroOrder oldOrder = null;
kelondroByteOrder oldOrder = null;
if ((sortOrderKey == null) || (sortOrderKey.equals("__"))) {
oldOrder = null;
} else {

4
source/de/anomic/kelondro/kelondroRowSet.java
Unescape View File

@ -61,7 +61,7 @@ public class kelondroRowSet extends kelondroRowCollection implements kelondroInd
this.profile = new kelondroProfile();
}
public void setOrdering(kelondroOrder newOrder, int newColumn) {
public void setOrdering(kelondroByteOrder newOrder, int newColumn) {
assert newOrder != null;
if ((rowdef.objectOrder == null) ||
(!(rowdef.objectOrder.signature().equals(newOrder.signature()))) ||
@ -275,7 +275,6 @@ public class kelondroRowSet extends kelondroRowCollection implements kelondroInd
}
}
@SuppressWarnings("unchecked")
public keyIterator clone(Object second) {
return new keyIterator(up, (byte[]) second);
}
@ -331,7 +330,6 @@ public class kelondroRowSet extends kelondroRowCollection implements kelondroInd
}
}
@SuppressWarnings("unchecked")
public rowIterator clone(Object second) {
return new rowIterator(up, (byte[]) second);
}

4
source/de/anomic/kelondro/kelondroSQLTable.java
Unescape View File

@ -61,7 +61,7 @@ public class kelondroSQLTable implements kelondroIndex {
private final String db_pwd_str = "yacy";
private Connection theDBConnection = null;
private final kelondroOrder order = new kelondroNaturalOrder(true);
private final kelondroByteOrder order = new kelondroNaturalOrder(true);
private kelondroRow rowdef;
public kelondroSQLTable(String dbType, kelondroRow rowdef) throws Exception {
@ -262,7 +262,7 @@ public class kelondroSQLTable implements kelondroIndex {
return 0;
}
public kelondroOrder order() {
public kelondroByteOrder order() {
return this.order;
}

9
source/de/anomic/kelondro/kelondroSplittedTree.java
Unescape View File

@ -55,7 +55,7 @@ import java.util.List;
public class kelondroSplittedTree implements kelondroIndex {
private kelondroTree[] ktfs;
private kelondroOrder order;
private kelondroByteOrder order;
private int ff;
private String filename;
@ -69,7 +69,7 @@ public class kelondroSplittedTree implements kelondroIndex {
return new File(path, filenameStub + "." + ff + "." + co + "." + ns + ".ktc");
}
public kelondroSplittedTree(File pathToFiles, String filenameStub, kelondroOrder objectOrder,
public kelondroSplittedTree(File pathToFiles, String filenameStub, kelondroByteOrder objectOrder,
long preloadTime,
int forkfactor, kelondroRow rowdef, int txtProps, int txtPropsWidth) {
try {
@ -118,7 +118,6 @@ public class kelondroSplittedTree implements kelondroIndex {
return ktfs[partition(key)].get(key);
}
@SuppressWarnings("unchecked")
public synchronized void putMultiple(List<kelondroRow.Entry> rows) throws IOException {
Iterator<kelondroRow.Entry> i = rows.iterator();
kelondroRow.Entry row;
@ -188,7 +187,6 @@ public class kelondroSplittedTree implements kelondroIndex {
ktfsI = ktfs[c].rows(up, firstKey); // FIXME: this works only correct with firstKey == null
}
@SuppressWarnings("unchecked")
public rowIterator clone(Object secondKey) {
try {
return new rowIterator(up, (byte[]) secondKey);
@ -255,7 +253,6 @@ public class kelondroSplittedTree implements kelondroIndex {
ktfsI = ktfs[c].keys(up, firstKey); // FIXME: this works only correct with firstKey == null
}
@SuppressWarnings("unchecked")
public keyIterator clone(Object secondKey) {
try {
return new keyIterator(up, (byte[]) secondKey);
@ -305,7 +302,7 @@ public class kelondroSplittedTree implements kelondroIndex {
}
public kelondroOrder order() {
public kelondroByteOrder order() {
return this.order;
}

20
source/de/anomic/kelondro/kelondroTree.java
Unescape View File

@ -50,6 +50,7 @@ import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.util.Comparator;
import java.util.Date;
import java.util.HashSet;
import java.util.Iterator;
@ -87,7 +88,7 @@ public class kelondroTree extends kelondroCachedRecords implements kelondroIndex
// class variables
private final Search writeSearchObj = new Search();
protected kelondroOrder loopDetectionOrder = new kelondroNaturalOrder(true);
protected Comparator<String> loopDetectionOrder;
protected int readAheadChunkSize = 100;
protected long lastIteratorCount = readAheadChunkSize;
@ -105,7 +106,7 @@ public class kelondroTree extends kelondroCachedRecords implements kelondroIndex
setHandle(root, null); // define the root value
}
super.setLogger(log);
this.loopDetectionOrder = new kelondroByteOrder.StringOrder(rowdef.objectOrder);
}
public static final kelondroTree open(File file, boolean useNodeCache, long preloadTime, kelondroRow rowdef) {
@ -135,7 +136,7 @@ public class kelondroTree extends kelondroCachedRecords implements kelondroIndex
this(ra, filename, useNodeCache, preloadTime, rowdef, new kelondroNaturalOrder(true), rowdef.columns() /* txtProps */, 80 /* txtPropWidth */, exitOnFail);
}
public kelondroTree(kelondroRA ra, String filename, boolean useNodeCache, long preloadTime, kelondroRow rowdef, kelondroOrder objectOrder, int txtProps, int txtPropsWidth, boolean exitOnFail) {
public kelondroTree(kelondroRA ra, String filename, boolean useNodeCache, long preloadTime, kelondroRow rowdef, kelondroByteOrder objectOrder, int txtProps, int txtPropsWidth, boolean exitOnFail) {
// this creates a new tree within a kelondroRA
super(ra, filename, useNodeCache, preloadTime,
thisOHBytes, thisOHHandles, rowdef,
@ -148,6 +149,7 @@ public class kelondroTree extends kelondroCachedRecords implements kelondroIndex
throw new RuntimeException("cannot set root handle / " + e.getMessage());
}
super.setLogger(log);
this.loopDetectionOrder = new kelondroByteOrder.StringOrder(rowdef.objectOrder);
}
public kelondroTree(kelondroRA ra, String filename, boolean useNodeCache, long preloadTime) throws IOException {
@ -272,7 +274,7 @@ public class kelondroTree extends kelondroCachedRecords implements kelondroIndex
return;
}
// System.out.println("Comparing key = '" + new String(key) + "' with '" + otherkey + "':"); // debug
c = row().objectOrder.compare(new String(key), keybuffer);
c = row().objectOrder.compare(key, keybuffer.getBytes());
// System.out.println(c); // debug
if (c == 0) {
found = true;
@ -988,10 +990,10 @@ public class kelondroTree extends kelondroCachedRecords implements kelondroIndex
public TreeMap<String, kelondroRow.Entry> rowMap(boolean up, byte[] firstKey, boolean including, int count) throws IOException {
// returns an ordered map of keys/row relations; key objects are of type String, value objects are of type byte[][]
kelondroOrder setOrder = (kelondroOrder) row().objectOrder.clone();
kelondroByteOrder setOrder = (kelondroByteOrder) row().objectOrder.clone();
setOrder.direction(up);
setOrder.rotate(firstKey);
TreeMap<String, kelondroRow.Entry> rows = new TreeMap<String, kelondroRow.Entry>(setOrder);
TreeMap<String, kelondroRow.Entry> rows = new TreeMap<String, kelondroRow.Entry>(this.loopDetectionOrder);
CacheNode n;
String key;
synchronized (this) {
@ -1008,10 +1010,10 @@ public class kelondroTree extends kelondroCachedRecords implements kelondroIndex
public TreeSet<String> keySet(boolean up, boolean rotating, byte[] firstKey, boolean including, int count) throws IOException {
// returns an ordered set of keys; objects are of type String
kelondroOrder setOrder = (kelondroOrder) row().objectOrder.clone();
kelondroByteOrder setOrder = (kelondroByteOrder) row().objectOrder.clone();
setOrder.direction(up);
setOrder.rotate(firstKey);
TreeSet<String> set = new TreeSet<String>(setOrder);
TreeSet<String> set = new TreeSet<String>(this.loopDetectionOrder);
kelondroNode n;
synchronized (this) {
Iterator<CacheNode> i = (firstKey == null) ? new nodeIterator(up, rotating) : new nodeIterator(up, rotating, firstKey, including);
@ -1054,7 +1056,6 @@ public class kelondroTree extends kelondroCachedRecords implements kelondroIndex
lastIteratorCount = 0;
}
@SuppressWarnings("unchecked")
public rowIterator clone(Object secondStart) {
try {
return new rowIterator(inc, (byte[]) secondStart, guessedCountLimit);
@ -1128,7 +1129,6 @@ public class kelondroTree extends kelondroCachedRecords implements kelondroIndex
lastIteratorCount = 0;
}
@SuppressWarnings("unchecked")
public keyIterator clone(Object secondStart) {
try {
return new keyIterator(inc, (byte[]) secondStart, guessedCountLimit);

1
source/de/anomic/kelondro/kelondroXMLTables.java
Unescape View File

@ -68,7 +68,6 @@ public class kelondroXMLTables {
this.tables = new Hashtable<String, Hashtable<String, String>>();
}
@SuppressWarnings("unchecked")
public kelondroXMLTables(File file) throws IOException {
this.propFile = file;
this.timestamp = System.currentTimeMillis();

4
source/de/anomic/plasma/plasmaCondenser.java
Unescape View File

@ -267,7 +267,7 @@ public final class plasmaCondenser {
}
public static final Set<String> words2hashSet(String[] words) {
TreeSet<String> hashes = new TreeSet<String>(kelondroBase64Order.enhancedCoder);
TreeSet<String> hashes = new TreeSet<String>(kelondroBase64Order.enhancedComparator);
for (int i = 0; i < words.length; i++) hashes.add(word2hash(words[i]));
return hashes;
}
@ -280,7 +280,7 @@ public final class plasmaCondenser {
public static final TreeSet<String> words2hashes(Set<String> words) {
Iterator<String> i = words.iterator();
TreeSet<String> hashes = new TreeSet<String>(kelondroBase64Order.enhancedCoder);
TreeSet<String> hashes = new TreeSet<String>(kelondroBase64Order.enhancedComparator);
while (i.hasNext()) hashes.add(word2hash(i.next()));
return hashes;
}

52
source/de/anomic/plasma/plasmaCrawlLURL.java
Unescape View File

@ -86,12 +86,12 @@ public final class plasmaCrawlLURL {
// result stacks;
// these have all entries of form
// strings: urlHash + initiatorHash + ExecutorHash
private final LinkedList externResultStack; // 1 - remote index: retrieved by other peer
private final LinkedList searchResultStack; // 2 - partly remote/local index: result of search queries
private final LinkedList transfResultStack; // 3 - partly remote/local index: result of index transfer
private final LinkedList proxyResultStack; // 4 - local index: result of proxy fetch/prefetch
private final LinkedList lcrawlResultStack; // 5 - local index: result of local crawling
private final LinkedList gcrawlResultStack; // 6 - local index: triggered external
private final LinkedList<String> externResultStack; // 1 - remote index: retrieved by other peer
private final LinkedList<String> searchResultStack; // 2 - partly remote/local index: result of search queries
private final LinkedList<String> transfResultStack; // 3 - partly remote/local index: result of index transfer
private final LinkedList<String> proxyResultStack; // 4 - local index: result of proxy fetch/prefetch
private final LinkedList<String> lcrawlResultStack; // 5 - local index: result of local crawling
private final LinkedList<String> gcrawlResultStack; // 6 - local index: triggered external
// the class object
private kelondroIndex urlIndexFile;
@ -102,12 +102,12 @@ public final class plasmaCrawlLURL {
urlIndexFile = new kelondroFlexSplitTable(new File(indexPath, "PUBLIC/TEXT"), "urls", preloadTime, indexURLEntry.rowdef, false);
// init result stacks
externResultStack = new LinkedList();
searchResultStack = new LinkedList();
transfResultStack = new LinkedList();
proxyResultStack = new LinkedList();
lcrawlResultStack = new LinkedList();
gcrawlResultStack = new LinkedList();
externResultStack = new LinkedList<String>();
searchResultStack = new LinkedList<String>();
transfResultStack = new LinkedList<String>();
proxyResultStack = new LinkedList<String>();
lcrawlResultStack = new LinkedList<String>();
gcrawlResultStack = new LinkedList<String>();
}
public int size() {
@ -312,14 +312,14 @@ public final class plasmaCrawlLURL {
}
}
public kelondroCloneableIterator entries(boolean up, String firstHash) throws IOException {
public kelondroCloneableIterator<indexURLEntry> entries(boolean up, String firstHash) throws IOException {
// enumerates entry elements
return new kiter(up, firstHash);
}
public class kiter implements kelondroCloneableIterator {
public class kiter implements kelondroCloneableIterator<indexURLEntry> {
// enumerates entry elements
private Iterator iter;
private Iterator<kelondroRow.Entry> iter;
private boolean error;
boolean up;
@ -343,10 +343,10 @@ public final class plasmaCrawlLURL {
return this.iter.hasNext();
}
public final Object next() {
public final indexURLEntry next() {
kelondroRow.Entry e = null;
if (this.iter == null) { return null; }
if (this.iter.hasNext()) { e = (kelondroRow.Entry) this.iter.next(); }
if (this.iter.hasNext()) { e = this.iter.next(); }
if (e == null) { return null; }
return new indexURLEntry(e, null, 0);
}
@ -364,9 +364,9 @@ public final class plasmaCrawlLURL {
*/
public void urldbcleanup() {
serverLog log = new serverLog("URLDBCLEANUP");
HashSet damagedURLS = new HashSet();
HashSet<String> damagedURLS = new HashSet<String>();
try {
Iterator eiter = entries(true, null);
Iterator<indexURLEntry> eiter = entries(true, null);
int iteratorCount = 0;
while (eiter.hasNext()) try {
eiter.next();
@ -382,7 +382,7 @@ public final class plasmaCrawlLURL {
try { Thread.sleep(1000); } catch (InterruptedException e) { }
log.logInfo("URLs vorher: " + size() + " Entries loaded during Iteratorloop: " + iteratorCount + " kaputte URLs: " + damagedURLS.size());
Iterator eiter2 = damagedURLS.iterator();
Iterator<String> eiter2 = damagedURLS.iterator();
String urlHash;
while (eiter2.hasNext()) {
urlHash = (String) eiter2.next();
@ -455,7 +455,7 @@ public final class plasmaCrawlLURL {
public void run() {
try {
serverLog.logInfo("URLDBCLEANER", "UrldbCleaner-Thread startet");
final Iterator eiter = entries(true, null);
final Iterator<indexURLEntry> eiter = entries(true, null);
while (eiter.hasNext() && run) {
synchronized (this) {
if (this.pause) {
@ -468,7 +468,7 @@ public final class plasmaCrawlLURL {
}
}
}
final indexURLEntry entry = (indexURLEntry) eiter.next();
final indexURLEntry entry = eiter.next();
if (entry == null) {
serverLog.logFine("URLDBCLEANER", "entry == null");
} else if (entry.hash() == null) {
@ -605,12 +605,12 @@ public final class plasmaCrawlLURL {
pw.println("<link>http://yacy.net</link>");
}
Iterator i = entries(true, null); // iterates indexURLEntry objects
Iterator<indexURLEntry> i = entries(true, null); // iterates indexURLEntry objects
indexURLEntry entry;
indexURLEntry.Components comp;
String url;
loop: while (i.hasNext()) {
entry = (indexURLEntry) i.next();
entry = i.next();
comp = entry.comp();
url = comp.url().toNormalform(true, false);
if (!url.matches(filter)) continue;
@ -683,9 +683,9 @@ public final class plasmaCrawlLURL {
if (args[0].equals("-l")) try {
// arg 1 is path to URLCache
final plasmaCrawlLURL urls = new plasmaCrawlLURL(new File(args[2]), 0);
final Iterator enu = urls.entries(true, null);
final Iterator<indexURLEntry> enu = urls.entries(true, null);
while (enu.hasNext()) {
System.out.println(((indexURLEntry) enu.next()).toString());
System.out.println(enu.next().toString());
}
} catch (Exception e) {
e.printStackTrace();

25
source/de/anomic/plasma/plasmaDHTChunk.java
Unescape View File

@ -52,6 +52,7 @@ import java.util.Iterator;
import de.anomic.index.indexContainer;
import de.anomic.index.indexRWIEntry;
import de.anomic.index.indexRWIRowEntry;
import de.anomic.index.indexURLEntry;
import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroException;
@ -78,7 +79,7 @@ public class plasmaDHTChunk {
private int status = chunkStatus_UNDEFINED;
private String startPointHash = "AAAAAAAAAAAA";
private indexContainer[] indexContainers = null;
private HashMap urlCache; // String (url-hash) / plasmaCrawlLURL.Entry
private HashMap<String, indexURLEntry> urlCache; // String (url-hash) / plasmaCrawlLURL.Entry
private int idxCount;
private long selectionStartTime = 0;
@ -114,7 +115,7 @@ public class plasmaDHTChunk {
return c;
}
public HashMap urlCacheMap() {
public HashMap<String, indexURLEntry> urlCacheMap() {
return urlCache;
}
@ -207,17 +208,17 @@ public class plasmaDHTChunk {
private int selectTransferContainersResource(String hash, boolean ram, int maxcount, int maxtime) throws InterruptedException {
// if (maxcount > 500) { maxcount = 500; } // flooding & OOM reduce
// the hash is a start hash from where the indexes are picked
final ArrayList tmpContainers = new ArrayList(maxcount);
final ArrayList<indexContainer> tmpContainers = new ArrayList<indexContainer>(maxcount);
try {
final Iterator indexContainerIterator = wordIndex.indexContainerSet(hash, ram, true, maxcount).iterator();
final Iterator<indexContainer> indexContainerIterator = wordIndex.indexContainerSet(hash, ram, true, maxcount).iterator();
indexContainer container;
Iterator urlIter;
Iterator<indexRWIRowEntry> urlIter;
indexRWIEntry iEntry;
indexURLEntry lurl;
int refcount = 0;
int wholesize;
urlCache = new HashMap();
urlCache = new HashMap<String, indexURLEntry>();
final double maximumDistance = ((double) peerRedundancy * 2) / ((double) yacyCore.seedDB.sizeConnected());
final long timeout = (maxtime < 0) ? Long.MAX_VALUE : System.currentTimeMillis() + maxtime;
while (
@ -226,7 +227,7 @@ public class plasmaDHTChunk {
((container = (indexContainer) indexContainerIterator.next()) != null) &&
(container.size() > 0) &&
((tmpContainers.size() == 0) ||
(yacyDHTAction.dhtDistance(container.getWordHash(), ((indexContainer) tmpContainers.get(0)).getWordHash()) < maximumDistance)) &&
(yacyDHTAction.dhtDistance(container.getWordHash(), tmpContainers.get(0).getWordHash()) < maximumDistance)) &&
(System.currentTimeMillis() < timeout)
) {
// check for interruption
@ -288,16 +289,16 @@ public class plasmaDHTChunk {
} catch (kelondroException e) {
log.logSevere("selectTransferIndexes database corrupted: " + e.getMessage(), e);
indexContainers = new indexContainer[0];
urlCache = new HashMap();
urlCache = new HashMap<String, indexURLEntry>();
this.status = chunkStatus_FAILED;
return 0;
}
}
public synchronized String deleteTransferIndexes() {
Iterator urlIter;
Iterator<indexRWIRowEntry> urlIter;
indexRWIEntry iEntry;
HashSet urlHashes;
HashSet<String> urlHashes;
String count = "0";
for (int i = 0; i < this.indexContainers.length; i++) {
@ -307,10 +308,10 @@ public class plasmaDHTChunk {
continue;
}
int c = this.indexContainers[i].size();
urlHashes = new HashSet(this.indexContainers[i].size());
urlHashes = new HashSet<String>(this.indexContainers[i].size());
urlIter = this.indexContainers[i].entries();
while (urlIter.hasNext()) {
iEntry = (indexRWIEntry) urlIter.next();
iEntry = urlIter.next();
urlHashes.add(iEntry.urlHash());
}
String wordHash = indexContainers[i].getWordHash();

2
source/de/anomic/plasma/plasmaRankingRCIEvaluation.java
Unescape View File

@ -155,7 +155,7 @@ public class plasmaRankingRCIEvaluation {
public static TreeSet[] genRankingTable(kelondroAttrSeq rci, int[] partition) {
TreeSet[] ranked = new TreeSet[partition.length];
for (int i = 0; i < partition.length; i++) ranked[i] = new TreeSet(kelondroBase64Order.enhancedCoder);
for (int i = 0; i < partition.length; i++) ranked[i] = new TreeSet(kelondroBase64Order.enhancedComparator);
Iterator i = rci.keys();
String key;
kelondroAttrSeq.Entry entry;

17
source/de/anomic/plasma/plasmaSearchEvent.java
Unescape View File

@ -72,11 +72,12 @@ public final class plasmaSearchEvent {
private Thread localSearchThread;
private TreeMap preselectedPeerHashes;
//private Object[] references;
public TreeMap IAResults, IACount;
public TreeMap IAResults;
public TreeMap<String, Integer> IACount;
public String IAmaxcounthash, IAneardhthash;
private int localcount;
private resultWorker[] workerThreads;
private ArrayList resultList; // list of this.Entry objects
private ArrayList<ResultEntry> resultList;
//private int resultListLock; // a pointer that shows that all elements below this pointer are fixed and may not be changed again
private HashMap failedURLs; // a mapping from a urlhash to a fail reason string
TreeSet snippetFetchWordHashes; // a set of word hashes that are used to match with the snippets
@ -605,7 +606,7 @@ public final class plasmaSearchEvent {
}
}
public ArrayList completeResults(long waitingtime) {
public ArrayList<ResultEntry> completeResults(long waitingtime) {
long timeout = System.currentTimeMillis() + waitingtime;
while ((this.resultList.size() < query.neededResults()) && (anyWorkerAlive()) && (System.currentTimeMillis() < timeout)) {
try {Thread.sleep(200);} catch (InterruptedException e) {}
@ -718,7 +719,7 @@ public final class plasmaSearchEvent {
//assert e != null;
}
public Set references(int count) {
public Set<String> references(int count) {
// returns a set of words that are computed as toplist
return this.rankedCache.getReferences(count);
}
@ -730,12 +731,14 @@ public final class plasmaSearchEvent {
private String alternative_urlstring;
private String alternative_urlname;
private plasmaSnippetCache.TextSnippet textSnippet;
private ArrayList /* of plasmaSnippetCache.MediaSnippet */ mediaSnippets;
private ArrayList<plasmaSnippetCache.MediaSnippet> mediaSnippets;
// statistic objects
public long dbRetrievalTime, snippetComputationTime;
public ResultEntry(indexURLEntry urlentry, plasmaWordIndex wordIndex, plasmaSnippetCache.TextSnippet textSnippet, ArrayList mediaSnippets,
public ResultEntry(indexURLEntry urlentry, plasmaWordIndex wordIndex,
plasmaSnippetCache.TextSnippet textSnippet,
ArrayList<plasmaSnippetCache.MediaSnippet> mediaSnippets,
long dbRetrievalTime, long snippetComputationTime) {
this.urlentry = urlentry;
this.urlcomps = urlentry.comp();
@ -796,7 +799,7 @@ public final class plasmaSearchEvent {
public plasmaSnippetCache.TextSnippet textSnippet() {
return this.textSnippet;
}
public ArrayList /* of plasmaSnippetCache.MediaSnippet */ mediaSnippets() {
public ArrayList<plasmaSnippetCache.MediaSnippet> mediaSnippets() {
return this.mediaSnippets;
}
public Date modified() {

10
source/de/anomic/plasma/plasmaSearchQuery.java
Unescape View File

@ -184,8 +184,8 @@ public plasmaSearchQuery(
}
public static TreeSet<String> hashes2Set(String query) {
if (query == null) return new TreeSet<String>(kelondroBase64Order.enhancedCoder);
final TreeSet<String> keyhashes = new TreeSet<String>(kelondroBase64Order.enhancedCoder);
if (query == null) return new TreeSet<String>(kelondroBase64Order.enhancedComparator);
final TreeSet<String> keyhashes = new TreeSet<String>(kelondroBase64Order.enhancedComparator);
for (int i = 0; i < (query.length() / yacySeedDB.commonHashLength); i++) {
keyhashes.add(query.substring(i * yacySeedDB.commonHashLength, (i + 1) * yacySeedDB.commonHashLength));
}
@ -226,7 +226,7 @@ public plasmaSearchQuery(
public static TreeSet<String>[] cleanQuery(String querystring) {
// returns two sets: a query set and a exclude set
if ((querystring == null) || (querystring.length() == 0)) return new TreeSet[]{new TreeSet<String>(kelondroNaturalOrder.naturalOrder), new TreeSet<String>(kelondroNaturalOrder.naturalOrder)};
if ((querystring == null) || (querystring.length() == 0)) return new TreeSet[]{new TreeSet<String>(kelondroNaturalOrder.naturalComparator), new TreeSet<String>(kelondroNaturalOrder.naturalComparator)};
// convert Umlaute
querystring = htmlFilterAbstractScraper.convertUmlaute(new serverCharBuffer(querystring.toCharArray())).toString();
@ -240,8 +240,8 @@ public plasmaSearchQuery(
}
// the string is clean now, but we must generate a set out of it
final TreeSet<String> query = new TreeSet<String>(kelondroNaturalOrder.naturalOrder);
final TreeSet<String> exclude = new TreeSet<String>(kelondroNaturalOrder.naturalOrder);
final TreeSet<String> query = new TreeSet<String>(kelondroNaturalOrder.naturalComparator);
final TreeSet<String> exclude = new TreeSet<String>(kelondroNaturalOrder.naturalComparator);
final String[] a = querystring.split(" ");
for (int i = 0; i < a.length; i++) {
if (a[i].startsWith("-")) {

22
source/de/anomic/plasma/plasmaSwitchboard.java
Unescape View File

@ -173,7 +173,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
// 3) result of index transfer, some of them are here (not possible here)
// 4) proxy-load (initiator is "------------")
// 5) local prefetch/crawling (initiator is own seedHash)
// 6) local fetching for global crawling (other known or unknwon initiator)
// 6) local fetching for global crawling (other known or unknown initiator)
public static final int PROCESSCASE_0_UNKNOWN = 0;
public static final int PROCESSCASE_1_GLOBAL_CRAWLING = 1;
public static final int PROCESSCASE_2_SEARCH_QUERY_RESULT = 2;
@ -183,9 +183,9 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
public static final int PROCESSCASE_6_GLOBAL_CRAWLING = 6;
// couloured list management
public static TreeSet badwords = null;
public static TreeSet blueList = null;
public static TreeSet stopwords = null;
public static TreeSet<String> badwords = null;
public static TreeSet<String> blueList = null;
public static TreeSet<String> stopwords = null;
public static plasmaURLPattern urlBlacklist;
public static wikiParser wikiParser = null;
@ -199,7 +199,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
public File rankingPath;
public File workPath;
public File releasePath;
public HashMap rankingPermissions;
public HashMap<String, String> rankingPermissions;
public plasmaWordIndex wordIndex;
public plasmaCrawlQueues crawlQueues;
public plasmaSwitchboardQueue sbQueue;
@ -230,7 +230,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
public plasmaDHTFlush transferIdxThread = null;
private plasmaDHTChunk dhtTransferChunk = null;
public ArrayList<HashMap<String, Object>> localSearches, remoteSearches; // array of search result properties as HashMaps
public HashMap localSearchTracker, remoteSearchTracker; // mappings from requesting host to a TreeSet of Long(access time)
public HashMap<String, TreeSet<Long>> localSearchTracker, remoteSearchTracker; // mappings from requesting host to a TreeSet of Long(access time)
public long lastseedcheckuptime = -1;
public long indexedPages = 0;
public long lastindexedPages = 0;
@ -985,7 +985,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
// read only once upon first instantiation of this class
String f = getConfig(LIST_BLUE, LIST_BLUE_DEFAULT);
File plasmaBlueListFile = new File(f);
if (f != null) blueList = kelondroMSetTools.loadList(plasmaBlueListFile, kelondroNaturalOrder.naturalOrder); else blueList= new TreeSet();
if (f != null) blueList = kelondroMSetTools.loadList(plasmaBlueListFile, kelondroNaturalOrder.naturalComparator); else blueList= new TreeSet();
this.log.logConfig("loaded blue-list from file " + plasmaBlueListFile.getName() + ", " +
blueList.size() + " entries, " +
ppRamString(plasmaBlueListFile.length()/1024));
@ -1018,7 +1018,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
// load badwords (to filter the topwords)
if (badwords == null) {
File badwordsFile = new File(rootPath, LIST_BADWORDS_DEFAULT);
badwords = kelondroMSetTools.loadList(badwordsFile, kelondroNaturalOrder.naturalOrder);
badwords = kelondroMSetTools.loadList(badwordsFile, kelondroNaturalOrder.naturalComparator);
this.log.logConfig("loaded badwords from file " + badwordsFile.getName() +
", " + badwords.size() + " entries, " +
ppRamString(badwordsFile.length()/1024));
@ -1027,7 +1027,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
// load stopwords
if (stopwords == null) {
File stopwordsFile = new File(rootPath, LIST_STOPWORDS_DEFAULT);
stopwords = kelondroMSetTools.loadList(stopwordsFile, kelondroNaturalOrder.naturalOrder);
stopwords = kelondroMSetTools.loadList(stopwordsFile, kelondroNaturalOrder.naturalComparator);
this.log.logConfig("loaded stopwords from file " + stopwordsFile.getName() + ", " +
stopwords.size() + " entries, " +
ppRamString(stopwordsFile.length()/1024));
@ -1205,8 +1205,8 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
this.incomingCookies = new HashMap();
// init search history trackers
this.localSearchTracker = new HashMap(); // String:TreeSet - IP:set of Long(accessTime)
this.remoteSearchTracker = new HashMap();
this.localSearchTracker = new HashMap<String, TreeSet<Long>>(); // String:TreeSet - IP:set of Long(accessTime)
this.remoteSearchTracker = new HashMap<String, TreeSet<Long>>();
this.localSearches = new ArrayList<HashMap<String, Object>>(); // contains search result properties as HashMaps
this.remoteSearches = new ArrayList<HashMap<String, Object>>();

65
source/de/anomic/plasma/plasmaWordIndex.java
Unescape View File

@ -46,6 +46,7 @@ import de.anomic.index.indexRWIEntry;
import de.anomic.index.indexRWIRowEntry;
import de.anomic.index.indexURLEntry;
import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroByteOrder;
import de.anomic.kelondro.kelondroCloneableIterator;
import de.anomic.kelondro.kelondroMergeIterator;
import de.anomic.kelondro.kelondroOrder;
@ -65,7 +66,7 @@ public final class plasmaWordIndex implements indexRI {
public static final int lowcachedivisor = 320;
public static final int maxCollectionPartition = 7; // should be 7
private final kelondroOrder indexOrder = kelondroBase64Order.enhancedCoder;
private final kelondroByteOrder indexOrder = kelondroBase64Order.enhancedCoder;
private final indexRAMRI dhtOutCache, dhtInCache;
private final indexCollectionRI collections; // new database structure to replace AssortmentCluster and FileCluster
public boolean busyCacheFlush; // shows if a cache flush is currently performed
@ -133,12 +134,8 @@ public final class plasmaWordIndex implements indexRI {
long entryBytes = indexRWIRowEntry.urlEntryRow.objectsize;
indexRAMRI cache = (in ? dhtInCache : dhtOutCache);
synchronized (cache) {
Iterator it = cache.wordContainers(null, false);
indexContainer ic;
while ( it.hasNext() ) {
ic = (indexContainer)it.next();
cacheBytes += ic.size() * entryBytes;
}
Iterator<indexContainer> it = cache.wordContainers(null, false);
while (it.hasNext()) cacheBytes += it.next().size() * entryBytes;
}
return cacheBytes;
@ -216,7 +213,7 @@ public final class plasmaWordIndex implements indexRI {
busyCacheFlush = true;
String wordHash;
ArrayList containerList = new ArrayList();
ArrayList<indexContainer> containerList = new ArrayList<indexContainer>();
count = Math.min(5000, Math.min(count, ram.size()));
boolean collectMax = true;
indexContainer c;
@ -293,15 +290,15 @@ public final class plasmaWordIndex implements indexRI {
int urlComps = htmlFilterContentScraper.urlComps(url.toString()).length;
// iterate over all words of context text
Iterator i = condenser.words().entrySet().iterator();
Map.Entry wentry;
Iterator<Map.Entry<String, plasmaCondenser.wordStatProp>> i = condenser.words().entrySet().iterator();
Map.Entry<String, plasmaCondenser.wordStatProp> wentry;
String word;
indexRWIEntry ientry;
plasmaCondenser.wordStatProp wprop;
while (i.hasNext()) {
wentry = (Map.Entry) i.next();
word = (String) wentry.getKey();
wprop = (plasmaCondenser.wordStatProp) wentry.getValue();
wentry = i.next();
word = wentry.getKey();
wprop = wentry.getValue();
assert (wprop.flags != null);
ientry = new indexRWIRowEntry(url.hash(),
urlLength, urlComps, (document == null) ? urlLength : document.getTitle().length(),
@ -333,7 +330,7 @@ public final class plasmaWordIndex implements indexRI {
return false;
}
public indexContainer getContainer(String wordHash, Set urlselection) {
public indexContainer getContainer(String wordHash, Set<String> urlselection) {
if ((wordHash == null) || (wordHash.length() != yacySeedDB.commonHashLength)) {
// wrong input
return null;
@ -461,7 +458,7 @@ public final class plasmaWordIndex implements indexRI {
public int removeEntryMultiple(Set<String> wordHashes, String urlHash) {
// remove the same url hashes for multiple words
// this is mainly used when correcting a index after a search
Iterator i = wordHashes.iterator();
Iterator<String> i = wordHashes.iterator();
int count = 0;
while (i.hasNext()) {
if (removeEntry((String) i.next(), urlHash)) count++;
@ -500,7 +497,7 @@ public final class plasmaWordIndex implements indexRI {
public void removeEntriesMultiple(Set<String> wordHashes, Set<String> urlHashes) {
// remove the same url hashes for multiple words
// this is mainly used when correcting a index after a search
Iterator i = wordHashes.iterator();
Iterator<String> i = wordHashes.iterator();
while (i.hasNext()) {
removeEntries((String) i.next(), urlHashes);
}
@ -509,7 +506,7 @@ public final class plasmaWordIndex implements indexRI {
public int removeWordReferences(Set<String> words, String urlhash) {
// sequentially delete all word references
// returns number of deletions
Iterator iter = words.iterator();
Iterator<String> iter = words.iterator();
int count = 0;
while (iter.hasNext()) {
// delete the URL reference in this word index
@ -528,13 +525,13 @@ public final class plasmaWordIndex implements indexRI {
if (d > 0) return d; else return dhtOutCache.tryRemoveURLs(urlHash);
}
public synchronized TreeSet indexContainerSet(String startHash, boolean ram, boolean rot, int count) {
public synchronized TreeSet<indexContainer> indexContainerSet(String startHash, boolean ram, boolean rot, int count) {
// creates a set of indexContainers
// this does not use the dhtInCache
kelondroOrder containerOrder = new indexContainerOrder((kelondroOrder) indexOrder.clone());
containerOrder.rotate(startHash.getBytes());
TreeSet containers = new TreeSet(containerOrder);
Iterator i = wordContainers(startHash, ram, rot);
kelondroOrder<indexContainer> containerOrder = new indexContainerOrder(indexOrder.clone());
containerOrder.rotate(emptyContainer(startHash, 0));
TreeSet<indexContainer> containers = new TreeSet<indexContainer>(containerOrder);
Iterator<indexContainer> i = wordContainers(startHash, ram, rot);
if (ram) count = Math.min(dhtOutCache.size(), count);
indexContainer container;
// this loop does not terminate using the i.hasNex() predicate when rot == true
@ -542,7 +539,7 @@ public final class plasmaWordIndex implements indexRI {
// in this case a termination must be ensured with a counter
// It must also be ensured that the counter is in/decreased every loop
while ((count > 0) && (i.hasNext())) {
container = (indexContainer) i.next();
container = i.next();
if ((container != null) && (container.size() > 0)) {
containers.add(container);
}
@ -551,22 +548,22 @@ public final class plasmaWordIndex implements indexRI {
return containers; // this may return less containers as demanded
}
public synchronized kelondroCloneableIterator wordContainers(String startHash, boolean ram, boolean rot) {
kelondroCloneableIterator i = wordContainers(startHash, ram);
public synchronized kelondroCloneableIterator<indexContainer> wordContainers(String startHash, boolean ram, boolean rot) {
kelondroCloneableIterator<indexContainer> i = wordContainers(startHash, ram);
if (rot) {
return new kelondroRotateIterator(i, new String(kelondroBase64Order.zero(startHash.length())));
return new kelondroRotateIterator<indexContainer>(i, new String(kelondroBase64Order.zero(startHash.length())));
} else {
return i;
}
}
public synchronized kelondroCloneableIterator wordContainers(String startWordHash, boolean ram) {
kelondroOrder containerOrder = new indexContainerOrder((kelondroOrder) indexOrder.clone());
containerOrder.rotate(startWordHash.getBytes());
public synchronized kelondroCloneableIterator<indexContainer> wordContainers(String startWordHash, boolean ram) {
kelondroOrder<indexContainer> containerOrder = new indexContainerOrder(indexOrder.clone());
containerOrder.rotate(emptyContainer(startWordHash, 0));
if (ram) {
return dhtOutCache.wordContainers(startWordHash, false);
} else {
return new kelondroMergeIterator(
return new kelondroMergeIterator<indexContainer>(
dhtOutCache.wordContainers(startWordHash, false),
collections.wordContainers(startWordHash, false),
containerOrder,
@ -602,12 +599,12 @@ public final class plasmaWordIndex implements indexRI {
indexContainer container = null;
indexRWIEntry entry = null;
yacyURL url = null;
HashSet urlHashs = new HashSet();
Iterator indexContainerIterator = indexContainerSet(startHash, false, false, 100).iterator();
HashSet<String> urlHashs = new HashSet<String>();
Iterator<indexContainer> indexContainerIterator = indexContainerSet(startHash, false, false, 100).iterator();
while (indexContainerIterator.hasNext() && run) {
waiter();
container = (indexContainer) indexContainerIterator.next();
Iterator containerIterator = container.entries();
Iterator<indexRWIRowEntry> containerIterator = container.entries();
wordHashNow = container.getWordHash();
while (containerIterator.hasNext() && run) {
waiter();
@ -633,7 +630,7 @@ public final class plasmaWordIndex implements indexRI {
}
if (!containerIterator.hasNext()) {
// We may not be finished yet, try to get the next chunk of wordHashes
TreeSet containers = indexContainerSet(container.getWordHash(), false, false, 100);
TreeSet<indexContainer> containers = indexContainerSet(container.getWordHash(), false, false, 100);
indexContainerIterator = containers.iterator();
// Make sure we don't get the same wordhash twice, but don't skip a word
if ((indexContainerIterator.hasNext()) && (!container.getWordHash().equals(((indexContainer) indexContainerIterator.next()).getWordHash()))) {

48
source/de/anomic/yacy/yacyDHTAction.java
Unescape View File

@ -66,7 +66,7 @@ public class yacyDHTAction implements yacyPeerAction {
this.seedCrawlReady = new kelondroMScoreCluster<String>();
// init crawl-ready table
try {
Iterator en = seedDB.seedsConnected(true, false, null, (float) 0.0);
Iterator<yacySeed> en = seedDB.seedsConnected(true, false, null, (float) 0.0);
yacySeed ys;
while (en.hasNext()) {
ys = (yacySeed) en.next();
@ -76,14 +76,14 @@ public class yacyDHTAction implements yacyPeerAction {
}
}
public Iterator getDHTSeeds(boolean up, String firstHash, float minVersion) {
public Iterator<yacySeed> getDHTSeeds(boolean up, String firstHash, float minVersion) {
// enumerates seed-type objects: all seeds with starting point in the middle, rotating at the end/beginning
return new seedDHTEnum(up, firstHash, minVersion);
}
class seedDHTEnum implements Iterator {
class seedDHTEnum implements Iterator<yacySeed> {
Iterator e1, e2;
Iterator<yacySeed> e1, e2;
boolean up;
int steps;
float minVersion;
@ -100,11 +100,11 @@ public class yacyDHTAction implements yacyPeerAction {
return (steps > 0) && ((e2 == null) || (e2.hasNext()));
}
public Object next() {
public yacySeed next() {
if (steps == 0) return null;
steps--;
if ((e1 != null) && (e1.hasNext())) {
Object n = e1.next();
yacySeed n = e1.next();
if (!(e1.hasNext())) {
e1 = null;
e2 = seedDB.seedsConnected(up, false, null, minVersion);
@ -130,7 +130,7 @@ public class yacyDHTAction implements yacyPeerAction {
class providesRemoteCrawlURLsEnum implements Iterator<yacySeed> {
Iterator se;
Iterator<yacySeed> se;
yacySeed nextSeed;
public providesRemoteCrawlURLsEnum() {
@ -171,16 +171,16 @@ public class yacyDHTAction implements yacyPeerAction {
}
public Iterator getAcceptRemoteIndexSeeds(String starthash) {
public Iterator<yacySeed> getAcceptRemoteIndexSeeds(String starthash) {
// returns an enumeration of yacySeed-Objects
// that have the AcceptRemoteIndex-Flag set
// the seeds are enumerated in the right order according DHT
return new acceptRemoteIndexSeedEnum(starthash);
}
class acceptRemoteIndexSeedEnum implements Iterator {
class acceptRemoteIndexSeedEnum implements Iterator<yacySeed> {
Iterator se;
Iterator<yacySeed> se;
yacySeed nextSeed;
public acceptRemoteIndexSeedEnum(String starthash) {
@ -209,7 +209,7 @@ public class yacyDHTAction implements yacyPeerAction {
return null;
}
public Object next() {
public yacySeed next() {
yacySeed next = nextSeed;
nextSeed = nextInternal();
return next;
@ -221,13 +221,13 @@ public class yacyDHTAction implements yacyPeerAction {
}
public Iterator getAcceptRemoteCrawlSeeds(String starthash, boolean available) {
public Iterator<yacySeed> getAcceptRemoteCrawlSeeds(String starthash, boolean available) {
return new acceptRemoteCrawlSeedEnum(starthash, available);
}
class acceptRemoteCrawlSeedEnum implements Iterator {
class acceptRemoteCrawlSeedEnum implements Iterator<yacySeed> {
Iterator se;
Iterator<yacySeed> se;
yacySeed nextSeed;
boolean available;
@ -256,7 +256,7 @@ public class yacyDHTAction implements yacyPeerAction {
return null;
}
public Object next() {
public yacySeed next() {
yacySeed next = nextSeed;
nextSeed = nextInternal();
return next;
@ -269,20 +269,20 @@ public class yacyDHTAction implements yacyPeerAction {
}
public synchronized yacySeed getGlobalCrawlSeed(String urlHash) {
Iterator e = getAcceptRemoteCrawlSeeds(urlHash, true);
Iterator<yacySeed> e = getAcceptRemoteCrawlSeeds(urlHash, true);
yacySeed seed;
if (e.hasNext()) seed = (yacySeed) e.next(); else seed = null;
if (e.hasNext()) seed = e.next(); else seed = null;
e = null;
return seed;
}
public synchronized yacySeed getPublicClusterCrawlSeed(String urlHash, TreeMap clusterhashes) {
public synchronized yacySeed getPublicClusterCrawlSeed(String urlHash, TreeMap<String, String> clusterhashes) {
// clusterhashes is a String(hash)/String(IP) - mapping
kelondroCloneableIterator i = new kelondroRotateIterator(new kelondroCloneableMapIterator(clusterhashes, urlHash), null);
kelondroCloneableIterator<String> i = new kelondroRotateIterator<String>(new kelondroCloneableMapIterator<String>(clusterhashes, urlHash), null);
String hash;
int count = clusterhashes.size(); // counter to ensure termination
while ((i.hasNext()) && (count-- > 0)) {
hash = (String) i.next();
hash = i.next();
yacySeed seed = seedDB.getConnected(hash);
if (seed == null) continue;
seed.setAlternativeAddress((String) clusterhashes.get(hash));
@ -347,7 +347,7 @@ public class yacyDHTAction implements yacyPeerAction {
return ((double) (kelondroBase64Order.enhancedCoder.cardinal(from.getBytes()) - kelondroBase64Order.enhancedCoder.cardinal(to.getBytes()))) / ((double) Long.MAX_VALUE);
}
public synchronized ArrayList /* of yacySeed */ getDHTTargets(serverLog log, int primaryPeerCount, int reservePeerCount, String firstKey, String lastKey, double maxDist) {
public synchronized ArrayList<yacySeed> getDHTTargets(serverLog log, int primaryPeerCount, int reservePeerCount, String firstKey, String lastKey, double maxDist) {
// find a list of DHT-peers
assert firstKey != null;
assert lastKey != null;
@ -359,14 +359,14 @@ public class yacyDHTAction implements yacyPeerAction {
!(kelondroBase64Order.enhancedCoder.cardinal(firstKey.getBytes()) < kelondroBase64Order.enhancedCoder.cardinal(yacyCore.seedDB.mySeed.hash.getBytes()) &&
kelondroBase64Order.enhancedCoder.cardinal(lastKey.getBytes()) > kelondroBase64Order.enhancedCoder.cardinal(yacyCore.seedDB.mySeed.hash.getBytes()));
*/
ArrayList seeds = new ArrayList();
ArrayList<yacySeed> seeds = new ArrayList<yacySeed>();
yacySeed seed;
//double ownDistance = Math.min(yacyDHTAction.dhtDistance(yacyCore.seedDB.mySeed.hash, firstKey), yacyDHTAction.dhtDistance(yacyCore.seedDB.mySeed.hash, lastKey));
//double maxDistance = Math.min(ownDistance, maxDist);
double firstdist, lastdist;
Iterator e = this.getAcceptRemoteIndexSeeds(lastKey);
TreeSet doublecheck = new TreeSet(kelondroBase64Order.enhancedCoder);
Iterator<yacySeed> e = this.getAcceptRemoteIndexSeeds(lastKey);
TreeSet<String> doublecheck = new TreeSet<String>(kelondroBase64Order.enhancedComparator);
int maxloop = Math.min(100, yacyCore.seedDB.sizeConnected()); // to ensure termination
if (log != null) log.logInfo("Collecting DHT target peers for first_hash = " + firstKey + ", last_hash = " + lastKey);
while ((e.hasNext()) && (seeds.size() < (primaryPeerCount + reservePeerCount)) && (maxloop-- > 0)) {

32
source/de/anomic/yacy/yacySeedDB.java
Unescape View File

@ -62,16 +62,16 @@ import java.util.TreeMap;
import de.anomic.http.httpHeader;
import de.anomic.http.httpc;
import de.anomic.http.httpd;
import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroDyn;
import de.anomic.kelondro.kelondroException;
import de.anomic.kelondro.kelondroMScoreCluster;
import de.anomic.kelondro.kelondroMapObjects;
import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.plasma.plasmaHTCache;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverCore;
import de.anomic.server.serverDomains;
import de.anomic.server.serverDate;
import de.anomic.server.serverDomains;
import de.anomic.server.serverFileUtils;
import de.anomic.server.serverSwitch;
import de.anomic.server.logging.serverLog;
@ -256,17 +256,17 @@ public final class yacySeedDB {
}
public Iterator seedsSortedConnected(boolean up, String field) {
public Iterator<yacySeed> seedsSortedConnected(boolean up, String field) {
// enumerates seed-type objects: all seeds sequentially ordered by field
return new seedEnum(up, field, seedActiveDB);
}
public Iterator seedsSortedDisconnected(boolean up, String field) {
public Iterator<yacySeed> seedsSortedDisconnected(boolean up, String field) {
// enumerates seed-type objects: all seeds sequentially ordered by field
return new seedEnum(up, field, seedPassiveDB);
}
public Iterator seedsSortedPotential(boolean up, String field) {
public Iterator<yacySeed> seedsSortedPotential(boolean up, String field) {
// enumerates seed-type objects: all seeds sequentially ordered by field
return new seedEnum(up, field, seedPotentialDB);
}
@ -318,28 +318,28 @@ public final class yacySeedDB {
return clustermap;
}
public Iterator seedsConnected(boolean up, boolean rot, String firstHash, float minVersion) {
public Iterator<yacySeed> seedsConnected(boolean up, boolean rot, String firstHash, float minVersion) {
// enumerates seed-type objects: all seeds sequentially without order
return new seedEnum(up, rot, (firstHash == null) ? null : firstHash.getBytes(), null, seedActiveDB, minVersion);
}
public Iterator seedsDisconnected(boolean up, boolean rot, String firstHash, float minVersion) {
public Iterator<yacySeed> seedsDisconnected(boolean up, boolean rot, String firstHash, float minVersion) {
// enumerates seed-type objects: all seeds sequentially without order
return new seedEnum(up, rot, (firstHash == null) ? null : firstHash.getBytes(), null, seedPassiveDB, minVersion);
}
public Iterator seedsPotential(boolean up, boolean rot, String firstHash, float minVersion) {
public Iterator<yacySeed> seedsPotential(boolean up, boolean rot, String firstHash, float minVersion) {
// enumerates seed-type objects: all seeds sequentially without order
return new seedEnum(up, rot, (firstHash == null) ? null : firstHash.getBytes(), null, seedPotentialDB, minVersion);
}
public yacySeed anySeedVersion(float minVersion) {
// return just any seed that has a specific minimum version number
Iterator e = seedsConnected(true, true, yacySeed.randomHash(), minVersion);
Iterator<yacySeed> e = seedsConnected(true, true, yacySeed.randomHash(), minVersion);
return (yacySeed) e.next();
}
public HashMap seedsByAge(boolean up, int count) {
public HashMap<String, yacySeed> seedsByAge(boolean up, int count) {
// returns a peerhash/yacySeed relation
// to get most recent peers, set up = true; for oldest peers, set up = false
@ -349,12 +349,12 @@ public final class yacySeedDB {
kelondroMScoreCluster<String> seedScore = new kelondroMScoreCluster<String>();
yacySeed ys;
long absage;
Iterator s = seedsConnected(true, false, null, (float) 0.0);
Iterator<yacySeed> s = seedsConnected(true, false, null, (float) 0.0);
int searchcount = 1000;
if (searchcount > sizeConnected()) searchcount = sizeConnected();
try {
while ((s.hasNext()) && (searchcount-- > 0)) {
ys = (yacySeed) s.next();
ys = s.next();
if ((ys != null) && (ys.get(yacySeed.LASTSEEN, "").length() > 10)) try {
absage = Math.abs(System.currentTimeMillis() + serverDate.dayMillis - ys.getLastSeenUTC());
seedScore.addScore(ys.hash, (int) absage); // the higher absage, the older is the peer
@ -362,8 +362,8 @@ public final class yacySeedDB {
}
// result is now in the score object; create a result vector
HashMap result = new HashMap();
Iterator it = seedScore.scores(up);
HashMap<String, yacySeed> result = new HashMap<String, yacySeed>();
Iterator<String> it = seedScore.scores(up);
int c = 0;
while ((c < count) && (it.hasNext())) {
c++;
@ -907,7 +907,7 @@ public final class yacySeedDB {
}
}
class seedEnum implements Iterator {
class seedEnum implements Iterator<yacySeed> {
kelondroMapObjects.mapIterator it;
yacySeed nextSeed;
@ -967,7 +967,7 @@ public final class yacySeedDB {
return new yacySeed(hash, dna);
}
public Object next() {
public yacySeed next() {
yacySeed seed = nextSeed;
try {while (true) {
nextSeed = internalNext();

Write Preview
Loading…
Cancel
Save