From a5054c038d4565cfb24f38c5fee336a72e5e0cbf Mon Sep 17 00:00:00 2001 From: orbiter Date: Fri, 11 Jan 2008 00:12:01 +0000 Subject: [PATCH] - added large number of generics - redesign of ordering structures in kelondro (old did not work with strict generics) - 50% IO reduction during read access on kelondroFlex (ommiting of read on index table) git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4320 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- htroot/yacy/hello.java | 4 +- htroot/yacy/profile.java | 4 +- htroot/yacy/search.java | 45 ++-- htroot/yacy/transferRWI.java | 2 +- htroot/yacysearch.java | 14 +- htroot/yacysearchitem.java | 20 +- source/dbtest.java | 18 +- source/de/anomic/index/indexCachedRI.java | 1 - .../de/anomic/index/indexContainerOrder.java | 85 ++----- .../de/anomic/index/indexRWIEntryOrder.java | 24 +- .../kelondro/kelondroAbstractOrder.java | 67 +---- .../kelondro/kelondroAbstractRecords.java | 8 +- .../anomic/kelondro/kelondroBase64Order.java | 11 +- .../de/anomic/kelondro/kelondroBinSearch.java | 2 +- .../de/anomic/kelondro/kelondroByteArray.java | 4 +- .../de/anomic/kelondro/kelondroByteOrder.java | 50 ++++ .../anomic/kelondro/kelondroBytesIntMap.java | 2 +- .../kelondroCloneableMapIterator.java | 1 - .../kelondro/kelondroCollectionIndex.java | 12 +- source/de/anomic/kelondro/kelondroDyn.java | 3 +- .../kelondro/kelondroFlexSplitTable.java | 14 +- .../de/anomic/kelondro/kelondroFlexTable.java | 56 ++--- .../kelondro/kelondroFlexWidthArray.java | 19 +- .../anomic/kelondro/kelondroIntBytesMap.java | 10 +- .../kelondro/kelondroMScoreCluster.java | 2 - .../de/anomic/kelondro/kelondroMSetTools.java | 237 +++++++----------- .../anomic/kelondro/kelondroMapObjects.java | 1 - .../de/anomic/kelondro/kelondroMapTable.java | 6 +- .../kelondro/kelondroMergeIterator.java | 13 +- .../anomic/kelondro/kelondroNaturalOrder.java | 12 +- source/de/anomic/kelondro/kelondroOrder.java | 27 +- .../de/anomic/kelondro/kelondroRAMIndex.java | 14 +- .../kelondro/kelondroRotateIterator.java | 1 - source/de/anomic/kelondro/kelondroRow.java | 39 ++- .../kelondro/kelondroRowCollection.java | 2 +- source/de/anomic/kelondro/kelondroRowSet.java | 4 +- .../de/anomic/kelondro/kelondroSQLTable.java | 4 +- .../anomic/kelondro/kelondroSplittedTree.java | 9 +- source/de/anomic/kelondro/kelondroTree.java | 28 +-- .../de/anomic/kelondro/kelondroXMLTables.java | 1 - source/de/anomic/plasma/plasmaCondenser.java | 4 +- source/de/anomic/plasma/plasmaCrawlLURL.java | 52 ++-- source/de/anomic/plasma/plasmaDHTChunk.java | 25 +- .../plasma/plasmaRankingRCIEvaluation.java | 2 +- .../de/anomic/plasma/plasmaSearchEvent.java | 17 +- .../de/anomic/plasma/plasmaSearchQuery.java | 10 +- .../de/anomic/plasma/plasmaSwitchboard.java | 22 +- source/de/anomic/plasma/plasmaWordIndex.java | 65 +++-- source/de/anomic/yacy/yacyDHTAction.java | 48 ++-- source/de/anomic/yacy/yacySeedDB.java | 32 +-- 50 files changed, 521 insertions(+), 632 deletions(-) create mode 100644 source/de/anomic/kelondro/kelondroByteOrder.java diff --git a/htroot/yacy/hello.java b/htroot/yacy/hello.java index 1c01e3651..89c614b4b 100644 --- a/htroot/yacy/hello.java +++ b/htroot/yacy/hello.java @@ -198,7 +198,7 @@ public final class hello { if (count > 100) { count = 100; } // latest seeds - final Map ySeeds = yacyCore.seedDB.seedsByAge(true, count); // peerhash/yacySeed relation + final Map ySeeds = yacyCore.seedDB.seedsByAge(true, count); // peerhash/yacySeed relation // attach also my own seed seeds.append("seed0=").append(yacyCore.seedDB.mySeed().genSeedStr(key)).append(serverCore.CRLF_STRING); @@ -207,7 +207,7 @@ public final class hello { // attach other seeds if (ySeeds != null) { seeds.ensureCapacity((ySeeds.size() + 1) * 768); - Iterator si = ySeeds.values().iterator(); + Iterator si = ySeeds.values().iterator(); yacySeed s; while (si.hasNext()) { s = (yacySeed) si.next(); diff --git a/htroot/yacy/profile.java b/htroot/yacy/profile.java index 737bb22a8..6003b8701 100644 --- a/htroot/yacy/profile.java +++ b/htroot/yacy/profile.java @@ -92,9 +92,9 @@ public final class profile { if (fileIn != null) try { fileIn.close(); fileIn = null; } catch (Exception e) {} } - Iterator it = ((Map)profile).keySet().iterator(); + Iterator it = ((Map)profile).keySet().iterator(); while (it.hasNext()) { - key=(String)it.next(); + key = it.next(); value=profile.getProperty(key, "").replaceAll("\r","").replaceAll("\n","\\\\n"); if( !(key.equals("")) && !(value.equals("")) ){ prop.put("list_"+count+"_key", key); diff --git a/htroot/yacy/search.java b/htroot/yacy/search.java index 19bb9b00d..26e6b0e8d 100644 --- a/htroot/yacy/search.java +++ b/htroot/yacy/search.java @@ -45,6 +45,7 @@ import de.anomic.plasma.plasmaSearchEvent; import de.anomic.plasma.plasmaSearchQuery; import de.anomic.plasma.plasmaSearchRankingProfile; import de.anomic.plasma.plasmaSwitchboard; +import de.anomic.plasma.plasmaSearchEvent.ResultEntry; import de.anomic.server.serverCore; import de.anomic.server.serverObjects; import de.anomic.server.serverProfiling; @@ -120,7 +121,7 @@ public final class search { // tell all threads to do nothing for a specific time sb.intermissionAllThreads(2 * duetime); - TreeSet abstractSet = ((abstracts.length() == 0) || (abstracts.equals("auto"))) ? null : plasmaSearchQuery.hashes2Set(abstracts); + TreeSet abstractSet = ((abstracts.length() == 0) || (abstracts.equals("auto"))) ? null : plasmaSearchQuery.hashes2Set(abstracts); // store accessing peer if (yacyCore.seedDB == null) { @@ -130,8 +131,8 @@ public final class search { } // prepare search - final TreeSet queryhashes = plasmaSearchQuery.hashes2Set(query); - final TreeSet excludehashes = (exclude.length() == 0) ? new TreeSet(kelondroBase64Order.enhancedCoder) : plasmaSearchQuery.hashes2Set(exclude); + final TreeSet queryhashes = plasmaSearchQuery.hashes2Set(query); + final TreeSet excludehashes = (exclude.length() == 0) ? new TreeSet(kelondroBase64Order.enhancedComparator) : plasmaSearchQuery.hashes2Set(exclude); final long timestamp = System.currentTimeMillis(); // prepare a search profile @@ -142,25 +143,25 @@ public final class search { int indexabstractContainercount = 0; int joincount = 0; plasmaSearchQuery theQuery = null; - ArrayList accu = null; + ArrayList accu = null; long urlRetrievalAllTime = 0, snippetComputationAllTime = 0; if ((query.length() == 0) && (abstractSet != null)) { // this is _not_ a normal search, only a request for index abstracts - theQuery = new plasmaSearchQuery(null, abstractSet, new TreeSet(kelondroBase64Order.enhancedCoder), rankingProfile, maxdist, prefer, plasmaSearchQuery.contentdomParser(contentdom), false, count, 0, duetime, filter, plasmaSearchQuery.SEARCHDOM_LOCAL, null, -1, null, false); + theQuery = new plasmaSearchQuery(null, abstractSet, new TreeSet(kelondroBase64Order.enhancedComparator), rankingProfile, maxdist, prefer, plasmaSearchQuery.contentdomParser(contentdom), false, count, 0, duetime, filter, plasmaSearchQuery.SEARCHDOM_LOCAL, null, -1, null, false); theQuery.domType = plasmaSearchQuery.SEARCHDOM_LOCAL; yacyCore.log.logInfo("INIT HASH SEARCH (abstracts only): " + plasmaSearchQuery.anonymizedQueryHashes(theQuery.queryHashes) + " - " + theQuery.displayResults() + " links"); long timer = System.currentTimeMillis(); - Map[] containers = sb.wordIndex.localSearchContainers(theQuery, plasmaSearchQuery.hashes2Set(urls)); + Map[] containers = sb.wordIndex.localSearchContainers(theQuery, plasmaSearchQuery.hashes2Set(urls)); serverProfiling.update("SEARCH", new plasmaProfiling.searchEvent(theQuery.id(true), plasmaSearchEvent.COLLECTION, containers[0].size(), System.currentTimeMillis() - timer)); if (containers != null) { - Iterator ci = containers[0].entrySet().iterator(); - Map.Entry entry; + Iterator> ci = containers[0].entrySet().iterator(); + Map.Entry entry; String wordhash; while (ci.hasNext()) { - entry = (Map.Entry) ci.next(); - wordhash = (String) entry.getKey(); - indexContainer container = (indexContainer) entry.getValue(); + entry = ci.next(); + wordhash = entry.getKey(); + indexContainer container = entry.getValue(); indexabstractContainercount += container.size(); indexabstract.append("indexabstract." + wordhash + "=").append(indexContainer.compressIndex(container, null, 1000).toString()).append(serverCore.CRLF_STRING); } @@ -188,18 +189,18 @@ public final class search { } else { // attach information about index abstracts StringBuffer indexcount = new StringBuffer(); - Map.Entry entry; - Iterator i = theSearch.IACount.entrySet().iterator(); + Map.Entry entry; + Iterator> i = theSearch.IACount.entrySet().iterator(); while (i.hasNext()) { - entry = (Map.Entry) i.next(); + entry = i.next(); indexcount.append("indexcount.").append((String) entry.getKey()).append('=').append(((Integer) entry.getValue()).toString()).append(serverCore.CRLF_STRING); } if (abstractSet != null) { // if a specific index-abstract is demanded, attach it here - i = abstractSet.iterator(); + Iterator j = abstractSet.iterator(); String wordhash; - while (i.hasNext()) { - wordhash = (String) i.next(); + while (j.hasNext()) { + wordhash = (String) j.next(); indexabstractContainercount += ((Integer) theSearch.IACount.get(wordhash)).intValue(); indexabstract.append("indexabstract." + wordhash + "=").append((String) theSearch.IAResults.get(wordhash)).append(serverCore.CRLF_STRING); } @@ -238,9 +239,9 @@ public final class search { // prepare reference hints long timer = System.currentTimeMillis(); - Set ws = theSearch.references(10); + Set ws = theSearch.references(10); StringBuffer refstr = new StringBuffer(); - Iterator j = ws.iterator(); + Iterator j = ws.iterator(); while (j.hasNext()) { refstr.append(",").append((String) j.next()); } @@ -282,15 +283,15 @@ public final class search { // prepare search statistics Long trackerHandle = new Long(System.currentTimeMillis()); - HashMap searchProfile = theQuery.resultProfile(joincount, System.currentTimeMillis() - timestamp, urlRetrievalAllTime, snippetComputationAllTime); + HashMap searchProfile = theQuery.resultProfile(joincount, System.currentTimeMillis() - timestamp, urlRetrievalAllTime, snippetComputationAllTime); String client = (String) header.get("CLIENTIP"); searchProfile.put("host", client); yacySeed remotepeer = yacyCore.seedDB.lookupByIP(natLib.getInetAddress(client), true, false, false); searchProfile.put("peername", (remotepeer == null) ? "unknown" : remotepeer.getName()); searchProfile.put("time", trackerHandle); sb.remoteSearches.add(searchProfile); - TreeSet handles = (TreeSet) sb.remoteSearchTracker.get(client); - if (handles == null) handles = new TreeSet(); + TreeSet handles = sb.remoteSearchTracker.get(client); + if (handles == null) handles = new TreeSet(); handles.add(trackerHandle); sb.remoteSearchTracker.put(client, handles); diff --git a/htroot/yacy/transferRWI.java b/htroot/yacy/transferRWI.java index c9e9c2bae..8e4d50173 100644 --- a/htroot/yacy/transferRWI.java +++ b/htroot/yacy/transferRWI.java @@ -123,7 +123,7 @@ public final class transferRWI { final long startProcess = System.currentTimeMillis(); // decode request - final List v = nxTools.strings(indexes, null); + final List v = nxTools.strings(indexes, null); // free memory indexes = null; diff --git a/htroot/yacysearch.java b/htroot/yacysearch.java index b529d5bb8..9674cfea6 100644 --- a/htroot/yacysearch.java +++ b/htroot/yacysearch.java @@ -200,7 +200,7 @@ public class yacysearch { } // filter out stopwords - final TreeSet filtered = kelondroMSetTools.joinConstructive(query[0], plasmaSwitchboard.stopwords); + final TreeSet filtered = kelondroMSetTools.joinConstructive(query[0], plasmaSwitchboard.stopwords); if (filtered.size() > 0) { kelondroMSetTools.excludeDestructive(query[0], plasmaSwitchboard.stopwords); } @@ -217,7 +217,7 @@ public class yacysearch { sb.wordIndex.removeWordReferences(query[0], delHash); // make new news message with negative voting - HashMap map = new HashMap(); + HashMap map = new HashMap(); map.put("urlhash", delHash); map.put("vote", "negative"); map.put("refid", ""); @@ -238,7 +238,7 @@ public class yacysearch { document = plasmaSnippetCache.retrieveDocument(comp.url(), true, 5000, true); if (document != null) { // create a news message - HashMap map = new HashMap(); + HashMap map = new HashMap(); map.put("url", comp.url().toNormalform(false, true).replace(',', '|')); map.put("title", comp.title().replace(',', ' ')); map.put("description", ((document == null) ? comp.title() : document.getTitle()).replace(',', ' ')); @@ -255,7 +255,7 @@ public class yacysearch { final boolean globalsearch = (global) && (yacyonline); // do the search - TreeSet queryHashes = plasmaCondenser.words2hashes(query[0]); + TreeSet queryHashes = plasmaCondenser.words2hashes(query[0]); plasmaSearchQuery theQuery = new plasmaSearchQuery( querystring, queryHashes, @@ -311,14 +311,14 @@ public class yacysearch { // prepare search statistics Long trackerHandle = new Long(System.currentTimeMillis()); - HashMap searchProfile = theQuery.resultProfile(theSearch.getLocalCount() + theSearch.getGlobalCount(), System.currentTimeMillis() - timestamp, theSearch.getURLRetrievalTime(), theSearch.getSnippetComputationTime()); + HashMap searchProfile = theQuery.resultProfile(theSearch.getLocalCount() + theSearch.getGlobalCount(), System.currentTimeMillis() - timestamp, theSearch.getURLRetrievalTime(), theSearch.getSnippetComputationTime()); searchProfile.put("querystring", theQuery.queryString); searchProfile.put("time", trackerHandle); searchProfile.put("host", client); searchProfile.put("offset", new Integer(0)); sb.localSearches.add(searchProfile); - TreeSet handles = (TreeSet) sb.localSearchTracker.get(client); - if (handles == null) handles = new TreeSet(); + TreeSet handles = sb.localSearchTracker.get(client); + if (handles == null) handles = new TreeSet(); handles.add(trackerHandle); sb.localSearchTracker.put(client, handles); diff --git a/htroot/yacysearchitem.java b/htroot/yacysearchitem.java index 9446bf460..b01724e06 100644 --- a/htroot/yacysearchitem.java +++ b/htroot/yacysearchitem.java @@ -93,21 +93,21 @@ public class yacysearchitem { if (bottomline) { // attach the bottom line with search references (topwords) - final Set references = theSearch.references(20); + final Set references = theSearch.references(20); if (references.size() > 0) { // get the topwords - final TreeSet topwords = new TreeSet(kelondroNaturalOrder.naturalOrder); + final TreeSet topwords = new TreeSet(kelondroNaturalOrder.naturalComparator); String tmp = ""; - Iterator i = references.iterator(); + Iterator i = references.iterator(); while (i.hasNext()) { - tmp = (String) i.next(); + tmp = i.next(); if (tmp.matches("[a-z]+")) { topwords.add(tmp); } } // filter out the badwords - final TreeSet filteredtopwords = kelondroMSetTools.joinConstructive(topwords, plasmaSwitchboard.badwords); + final TreeSet filteredtopwords = kelondroMSetTools.joinConstructive(topwords, plasmaSwitchboard.badwords); if (filteredtopwords.size() > 0) { kelondroMSetTools.excludeDestructive(topwords, plasmaSwitchboard.badwords); } @@ -122,7 +122,7 @@ public class yacysearchitem { if (rss) { String word; int hintcount = 0; - final Iterator iter = topwords.iterator(); + final Iterator iter = topwords.iterator(); while (iter.hasNext()) { word = (String) iter.next(); if (word != null) { @@ -137,7 +137,7 @@ public class yacysearchitem { } else { String word; int hintcount = 0; - final Iterator iter = topwords.iterator(); + final Iterator iter = topwords.iterator(); while (iter.hasNext()) { word = (String) iter.next(); if (word != null) { @@ -209,7 +209,7 @@ public class yacysearchitem { prop.put("content_ybr", plasmaSearchRankingProcess.ybr(result.hash())); prop.putNum("content_size", result.filesize()); - TreeSet[] query = theQuery.queryWords(); + TreeSet[] query = theQuery.queryWords(); yacyURL wordURL = null; try { prop.putHTML("content_words", URLEncoder.encode(query[0].toString(),"UTF-8")); @@ -227,7 +227,7 @@ public class yacysearchitem { // image search; shows thumbnails // iterate over all images in the result prop.put("content", theQuery.contentdom + 1); // switch on specific content - ArrayList /* of plasmaSnippetCache.MediaSnippet */ images = result.mediaSnippets(); + ArrayList images = result.mediaSnippets(); if (images != null) { plasmaSnippetCache.MediaSnippet ms; yacyURL url; @@ -253,7 +253,7 @@ public class yacysearchitem { (theQuery.contentdom == plasmaSearchQuery.CONTENTDOM_APP)) { // any other media content prop.put("content", theQuery.contentdom + 1); // switch on specific content - ArrayList /* of plasmaSnippetCache.MediaSnippet */ media = result.mediaSnippets(); + ArrayList media = result.mediaSnippets(); if (item == 0) col = true; if (media != null) { plasmaSnippetCache.MediaSnippet ms; diff --git a/source/dbtest.java b/source/dbtest.java index 54ff8f927..b59557e48 100644 --- a/source/dbtest.java +++ b/source/dbtest.java @@ -7,13 +7,13 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Date; import java.util.HashSet; -import java.util.Iterator; import java.util.Random; import javax.imageio.ImageIO; import de.anomic.kelondro.kelondroBase64Order; import de.anomic.kelondro.kelondroCache; +import de.anomic.kelondro.kelondroCloneableIterator; import de.anomic.kelondro.kelondroFlexSplitTable; import de.anomic.kelondro.kelondroFlexTable; import de.anomic.kelondro.kelondroIndex; @@ -357,14 +357,14 @@ public class dbtest { } if (command.equals("list")) { - Iterator i = null; + kelondroCloneableIterator i = null; if (table instanceof kelondroSplittedTree) i = ((kelondroSplittedTree) table).rows(true, null); if (table instanceof kelondroTree) i = ((kelondroTree) table).rows(true, null); if (table instanceof kelondroSQLTable) i = ((kelondroSQLTable) table).rows(true, null); - byte[][] row; + kelondroRow.Entry row; while (i.hasNext()) { - row = (byte[][]) i.next(); - for (int j = 0; j < row.length; j++) System.out.print(new String(row[j]) + ","); + row = i.next(); + for (int j = 0; j < row.columns(); j++) System.out.print(row.getColString(j, null) + ","); System.out.println(); } } @@ -380,8 +380,8 @@ public class dbtest { long r; Long R; int p, rc=0; - ArrayList ra = new ArrayList(); - HashSet jcontrol = new HashSet(); + ArrayList ra = new ArrayList(); + HashSet jcontrol = new HashSet(); kelondroIntBytesMap kcontrol = new kelondroIntBytesMap(1, 0); for (int i = 0; i < writeCount; i++) { r = Math.abs(random.nextLong() % 1000); @@ -420,8 +420,8 @@ public class dbtest { long r; Long R; int p, rc=0; - ArrayList ra = new ArrayList(); - HashSet jcontrol = new HashSet(); + ArrayList ra = new ArrayList(); + HashSet jcontrol = new HashSet(); kelondroIntBytesMap kcontrol = new kelondroIntBytesMap(1, 0); for (int i = 0; i < writeCount; i++) { //if (i == 30) random = new Random(randomstart); diff --git a/source/de/anomic/index/indexCachedRI.java b/source/de/anomic/index/indexCachedRI.java index 083983da9..bdd22b43f 100644 --- a/source/de/anomic/index/indexCachedRI.java +++ b/source/de/anomic/index/indexCachedRI.java @@ -255,7 +255,6 @@ public class indexCachedRI implements indexRI { return wordContainers(startHash, false, rot); } - @SuppressWarnings("unchecked") public kelondroCloneableIterator wordContainers(String startHash, boolean ramOnly, boolean rot) { kelondroCloneableIterator i; if (ramOnly) { diff --git a/source/de/anomic/index/indexContainerOrder.java b/source/de/anomic/index/indexContainerOrder.java index 12222f3b9..97bfdfabf 100644 --- a/source/de/anomic/index/indexContainerOrder.java +++ b/source/de/anomic/index/indexContainerOrder.java @@ -26,25 +26,19 @@ package de.anomic.index; -import de.anomic.kelondro.kelondroNode; +import de.anomic.kelondro.kelondroAbstractOrder; import de.anomic.kelondro.kelondroOrder; -import de.anomic.kelondro.kelondroRow; -import de.anomic.kelondro.kelondroRow.Entry; -public class indexContainerOrder implements kelondroOrder { +public class indexContainerOrder extends kelondroAbstractOrder implements kelondroOrder { - private kelondroOrder embeddedOrder; + private kelondroOrder embeddedOrder; - public indexContainerOrder(kelondroOrder embedOrder) { + public indexContainerOrder(kelondroOrder embedOrder) { this.embeddedOrder = embedOrder; } - public boolean wellformed(byte[] a) { - return embeddedOrder.wellformed(a); - } - - public boolean wellformed(byte[] a, int astart, int alength) { - return embeddedOrder.wellformed(a, astart, alength); + public boolean wellformed(indexContainer a) { + return embeddedOrder.wellformed(a.getWordHash().getBytes()); } public void direction(boolean ascending) { @@ -59,16 +53,13 @@ public class indexContainerOrder implements kelondroOrder { return this.embeddedOrder.compare(a.getWordHash().getBytes(), b.getWordHash().getBytes()); } - public byte[] zero() { - return this.embeddedOrder.zero(); + public void rotate(indexContainer zero) { + this.embeddedOrder.rotate(zero.getWordHash().getBytes()); + this.zero = new indexContainer(new String(this.embeddedOrder.zero()), zero); } - public void rotate(byte[] zero) { - this.embeddedOrder.rotate(zero); - } - - public Object clone() { - return new indexContainerOrder((kelondroOrder) this.embeddedOrder.clone()); + public kelondroOrder clone() { + return new indexContainerOrder(this.embeddedOrder.clone()); } public String signature() { @@ -78,58 +69,14 @@ public class indexContainerOrder implements kelondroOrder { public long cardinal(byte[] key) { return this.embeddedOrder.cardinal(key); } - - public int compare(Object a, Object b) { - if ((a instanceof byte[]) && (b instanceof byte[])) { - return this.embeddedOrder.compare((byte[]) a, (byte[]) b); - } - if ((a instanceof Integer) && (b instanceof Integer)) { - return ((Integer) a).compareTo((Integer) b); - } - if ((a instanceof String) && (b instanceof String)) { - return this.embeddedOrder.compare(((String) a).getBytes(), ((String) b).getBytes()); - } - if ((a instanceof kelondroNode) && (b instanceof kelondroNode)) { - return this.embeddedOrder.compare((kelondroNode) a, (kelondroNode) b); - } - if ((a instanceof kelondroRow.Entry) && (b instanceof kelondroRow.Entry)) { - return this.embeddedOrder.compare((kelondroRow.Entry) a, (kelondroRow.Entry) b); - } - if ((a instanceof indexContainer) && (b instanceof indexContainer)) { - return this.embeddedOrder.compare((indexContainer) a, (indexContainer) b); - } - Class ac = a.getClass(); - Class bc = b.getClass(); - if (ac.equals(bc)) { - throw new UnsupportedOperationException("compare not implemented for this object type: " + ac); - } else { - throw new UnsupportedOperationException("compare must be applied to same object classes; here a = " + ac + ", b = " + bc); - } - } - public int compare(String a, String b) { - return this.embeddedOrder.compare(a, b); - } - - public int compare(kelondroNode a, kelondroNode b) { - return this.embeddedOrder.compare(a, b); - } - - public int compare(Entry a, Entry b) { - return this.embeddedOrder.compare(a, b); - } - - public int compare(byte[] a, byte[] b) { - return this.embeddedOrder.compare(a, b); - } - - public int compare(byte[] a, int aoffset, int alength, byte[] b, int boffset, int blength) { - return this.embeddedOrder.compare(a, aoffset, alength, b, boffset, blength); - } - - public boolean equals(kelondroOrder otherOrder) { + public boolean equals(kelondroOrder otherOrder) { if (!(otherOrder instanceof indexContainerOrder)) return false; return this.embeddedOrder.equals(((indexContainerOrder) otherOrder).embeddedOrder); } + public long cardinal(indexContainer key) { + return this.embeddedOrder.cardinal(key.getWordHash().getBytes()); + } + } diff --git a/source/de/anomic/index/indexRWIEntryOrder.java b/source/de/anomic/index/indexRWIEntryOrder.java index 49af1beda..3176fa522 100644 --- a/source/de/anomic/index/indexRWIEntryOrder.java +++ b/source/de/anomic/index/indexRWIEntryOrder.java @@ -39,7 +39,7 @@ import de.anomic.plasma.plasmaSearchRankingProcess; import de.anomic.plasma.plasmaSearchRankingProfile; import de.anomic.yacy.yacyURL; -public class indexRWIEntryOrder extends kelondroAbstractOrder implements kelondroOrder { +public class indexRWIEntryOrder extends kelondroAbstractOrder implements kelondroOrder { private indexRWIVarEntry min, max; private plasmaSearchRankingProfile ranking; private kelondroMScoreCluster doms; @@ -102,7 +102,7 @@ public class indexRWIEntryOrder extends kelondroAbstractOrder implements kelondr if (this.doms.size() > 0) this.maxdomcount = this.doms.getMaxScore(); } - public Object clone() { + public kelondroOrder clone() { return null; } @@ -152,27 +152,17 @@ public class indexRWIEntryOrder extends kelondroAbstractOrder implements kelondr return Long.MAX_VALUE - r; // returns a reversed number: the lower the number the better the ranking. This is used for simple sorting with a TreeMap } - public int compare(byte[] a, byte[] b) { - long ca = cardinal(new indexRWIRowEntry(a)); - long cb = cardinal(new indexRWIRowEntry(b)); + public int compare(indexRWIEntry a, indexRWIEntry b) { + long ca = cardinal(a); + long cb = cardinal(b); return (ca > cb) ? 1 : (ca < cb) ? -1 : 0; } - - public int compare(byte[] a, int aoffset, int alength, byte[] b, int boffset, int blength) { - long ca = cardinal(new indexRWIRowEntry(a, aoffset, false)); - long cb = cardinal(new indexRWIRowEntry(b, boffset, false)); - return (ca > cb) ? 1 : (ca < cb) ? -1 : 0; - } - + public String signature() { return "rx"; } - public boolean wellformed(byte[] a) { - return true; - } - - public boolean wellformed(byte[] a, int astart, int alength) { + public boolean wellformed(indexRWIEntry a) { return true; } diff --git a/source/de/anomic/kelondro/kelondroAbstractOrder.java b/source/de/anomic/kelondro/kelondroAbstractOrder.java index 1bdaa086a..af14c32e8 100644 --- a/source/de/anomic/kelondro/kelondroAbstractOrder.java +++ b/source/de/anomic/kelondro/kelondroAbstractOrder.java @@ -45,83 +45,36 @@ package de.anomic.kelondro; -import de.anomic.index.indexContainer; -public abstract class kelondroAbstractOrder implements kelondroOrder { +public abstract class kelondroAbstractOrder implements kelondroOrder { - protected byte[] zero = null; + protected A zero = null; protected boolean asc = true; - public abstract Object clone(); + abstract public kelondroOrder clone(); + public A zero() { + return zero; + } + public void direction(boolean ascending) { asc = ascending; } - public long partition(byte[] key, int forks) { + public long partition(A key, int forks) { final long d = (Long.MAX_VALUE / forks) + ((Long.MAX_VALUE % forks) + 1) / forks; return cardinal(key) / d; } - public int compare(Object a, Object b) { - if ((a instanceof byte[]) && (b instanceof byte[])) { - return compare((byte[]) a, (byte[]) b); - } - if ((a instanceof Integer) && (b instanceof Integer)) { - return compare((Integer) a, (Integer) b); - } - if ((a instanceof String) && (b instanceof String)) { - return compare(((String) a).getBytes(), ((String) b).getBytes()); - } - if ((a instanceof kelondroNode) && (b instanceof kelondroNode)) { - return compare((kelondroNode) a, (kelondroNode) b); - } - if ((a instanceof kelondroRow.Entry) && (b instanceof kelondroRow.Entry)) { - return compare((kelondroRow.Entry) a, (kelondroRow.Entry) b); - } - Class ac = a.getClass(); - Class bc = b.getClass(); - if (ac.equals(bc)) { - throw new UnsupportedOperationException("compare not implemented for this object type: " + ac); - } else { - throw new UnsupportedOperationException("compare must be applied to same object classes; here a = " + ac + ", b = " + bc); - } - } - - public int compare(kelondroNode a, kelondroNode b) { - return compare(a.getKey(), b.getKey()); - } - - public int compare(String a, String b) { - return compare(a.getBytes(), b.getBytes()); - } - - public int compare(kelondroRow.Entry a, kelondroRow.Entry b) { - return a.compareTo(b); - } - - public int compare(indexContainer a, indexContainer b) { - return compare(a.getWordHash(), b.getWordHash()); - } - - public int compare(Integer a, Integer b) { - return a.compareTo(b); - } - - public byte[] zero() { - return zero; - } - - public void rotate(byte[] newzero) { + public void rotate(A newzero) { this.zero = newzero; } - public boolean equals(kelondroOrder otherOrder) { + public boolean equals(kelondroOrder otherOrder) { if (otherOrder == null) return false; String thisSig = this.signature(); String otherSig = otherOrder.signature(); if ((thisSig == null) || (otherSig == null)) return false; return thisSig.equals(otherSig); } - } diff --git a/source/de/anomic/kelondro/kelondroAbstractRecords.java b/source/de/anomic/kelondro/kelondroAbstractRecords.java index 0cec224d6..45a0e6a00 100644 --- a/source/de/anomic/kelondro/kelondroAbstractRecords.java +++ b/source/de/anomic/kelondro/kelondroAbstractRecords.java @@ -370,7 +370,7 @@ public abstract class kelondroAbstractRecords implements kelondroRecords { } assignRowdef(rowdef); if (fileExisted) { - kelondroOrder oldOrder = readOrderType(); + kelondroByteOrder oldOrder = readOrderType(); if ((oldOrder != null) && (!(oldOrder.equals(rowdef.objectOrder)))) { writeOrderType(); // write new order type //throw new IOException("wrong object order upon initialization. new order is " + rowdef.objectOrder.toString() + ", old order was " + oldOrder.toString()); @@ -580,7 +580,7 @@ public abstract class kelondroAbstractRecords implements kelondroRecords { } catch (IOException e) {} } - private kelondroOrder readOrderType() { + private kelondroByteOrder readOrderType() { try { byte[] d = getDescription(); String s = new String(d).substring(0, 2); @@ -590,8 +590,8 @@ public abstract class kelondroAbstractRecords implements kelondroRecords { } } - public static kelondroOrder orderBySignature(String signature) { - kelondroOrder oo = null; + public static kelondroByteOrder orderBySignature(String signature) { + kelondroByteOrder oo = null; if (oo == null) oo = kelondroNaturalOrder.bySignature(signature); if (oo == null) oo = kelondroBase64Order.bySignature(signature); if (oo == null) oo = new kelondroNaturalOrder(true); diff --git a/source/de/anomic/kelondro/kelondroBase64Order.java b/source/de/anomic/kelondro/kelondroBase64Order.java index 355e6deab..1617220d3 100644 --- a/source/de/anomic/kelondro/kelondroBase64Order.java +++ b/source/de/anomic/kelondro/kelondroBase64Order.java @@ -51,7 +51,7 @@ import java.util.Comparator; import de.anomic.server.logging.serverLog; -public class kelondroBase64Order extends kelondroAbstractOrder implements kelondroOrder, kelondroCoding, Comparator { +public class kelondroBase64Order extends kelondroAbstractOrder implements kelondroByteOrder, kelondroCoding, Comparator { protected static final char[] alpha_standard = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/".toCharArray(); protected static final char[] alpha_enhanced = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_".toCharArray(); @@ -73,6 +73,8 @@ public class kelondroBase64Order extends kelondroAbstractOrder implements kelond public static final kelondroBase64Order standardCoder = new kelondroBase64Order(true, true); public static final kelondroBase64Order enhancedCoder = new kelondroBase64Order(true, false); + public static final Comparator standardComparator = new kelondroByteOrder.StringOrder(standardCoder); + public static final Comparator enhancedComparator = new kelondroByteOrder.StringOrder(enhancedCoder); private boolean rfc1113compliant; private final char[] alpha; @@ -95,9 +97,9 @@ public class kelondroBase64Order extends kelondroAbstractOrder implements kelond return z; } - public Object clone() { + public kelondroOrder clone() { kelondroBase64Order o = new kelondroBase64Order(this.asc, this.rfc1113compliant); - o.rotate(this.zero); + o.rotate(zero); return o; } @@ -115,7 +117,7 @@ public class kelondroBase64Order extends kelondroAbstractOrder implements kelond return true; } - public final static kelondroOrder bySignature(String signature) { + public final static kelondroByteOrder bySignature(String signature) { if (signature.equals("Bd")) return new kelondroBase64Order(false, false); if (signature.equals("bd")) return new kelondroBase64Order(false, true); if (signature.equals("Bu")) return new kelondroBase64Order(true, false); @@ -396,5 +398,4 @@ public class kelondroBase64Order extends kelondroAbstractOrder implements kelond System.out.println(b64.decodeString(s[1])); } } - } diff --git a/source/de/anomic/kelondro/kelondroBinSearch.java b/source/de/anomic/kelondro/kelondroBinSearch.java index 84c636c57..922e292da 100644 --- a/source/de/anomic/kelondro/kelondroBinSearch.java +++ b/source/de/anomic/kelondro/kelondroBinSearch.java @@ -51,7 +51,7 @@ public class kelondroBinSearch { private byte[] chunks; private int chunksize; private int count; - private kelondroOrder objectOrder = new kelondroNaturalOrder(true); + private kelondroByteOrder objectOrder = new kelondroNaturalOrder(true); public kelondroBinSearch(byte[] chunks, int chunksize) { this.chunks = chunks; diff --git a/source/de/anomic/kelondro/kelondroByteArray.java b/source/de/anomic/kelondro/kelondroByteArray.java index 31beae0fe..314d9777f 100644 --- a/source/de/anomic/kelondro/kelondroByteArray.java +++ b/source/de/anomic/kelondro/kelondroByteArray.java @@ -192,11 +192,11 @@ public class kelondroByteArray { this.offset = 0; } - public int compareTo(kelondroByteArray b, kelondroOrder order) { + public int compareTo(kelondroByteArray b, kelondroByteOrder order) { return order.compare(this.buffer, this.offset, this.length, b.buffer, b.offset, b.length); } - public int compareTo(int aoffset, int alength, kelondroByteArray b, int boffset, int blength, kelondroOrder order) { + public int compareTo(int aoffset, int alength, kelondroByteArray b, int boffset, int blength, kelondroByteOrder order) { return order.compare(this.buffer, this.offset + aoffset, alength, b.buffer, b.offset + boffset, blength); } } diff --git a/source/de/anomic/kelondro/kelondroByteOrder.java b/source/de/anomic/kelondro/kelondroByteOrder.java new file mode 100644 index 000000000..99b3d3c1b --- /dev/null +++ b/source/de/anomic/kelondro/kelondroByteOrder.java @@ -0,0 +1,50 @@ +// kelondroByteOrder.java +// ----------------------------- +// (C) 2008 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany +// first published 10.01.2008 on http://yacy.net +// +// This is a part of YaCy, a peer-to-peer based web search engine +// +// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $ +// $LastChangedRevision: 1986 $ +// $LastChangedBy: orbiter $ +// +// LICENSE +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +package de.anomic.kelondro; + +import java.util.Comparator; + +public interface kelondroByteOrder extends kelondroOrder { + + public boolean wellformed(byte[] a, int start, int len); + + public int compare(byte[] a, int astart, int alen, byte[] b, int bstart, int blen); + + public static class StringOrder implements Comparator { + + public kelondroByteOrder baseOrder; + public StringOrder(kelondroByteOrder base) { + this.baseOrder = base; + } + + public int compare(String s1, String s2) { + return baseOrder.compare(s1.getBytes(), s2.getBytes()); + } + + } +} diff --git a/source/de/anomic/kelondro/kelondroBytesIntMap.java b/source/de/anomic/kelondro/kelondroBytesIntMap.java index 614326959..ab04db770 100644 --- a/source/de/anomic/kelondro/kelondroBytesIntMap.java +++ b/source/de/anomic/kelondro/kelondroBytesIntMap.java @@ -38,7 +38,7 @@ public class kelondroBytesIntMap { this.rowdef = ki.row(); } - public kelondroBytesIntMap(int keylength, kelondroOrder objectOrder, int space) { + public kelondroBytesIntMap(int keylength, kelondroByteOrder objectOrder, int space) { this.rowdef = new kelondroRow(new kelondroColumn[]{new kelondroColumn("key", kelondroColumn.celltype_binary, kelondroColumn.encoder_bytes, keylength, "key"), new kelondroColumn("int c-4 {b256}")}, objectOrder, 0); this.index = new kelondroRAMIndex(rowdef, space); } diff --git a/source/de/anomic/kelondro/kelondroCloneableMapIterator.java b/source/de/anomic/kelondro/kelondroCloneableMapIterator.java index 1ee939bfe..0d4721194 100644 --- a/source/de/anomic/kelondro/kelondroCloneableMapIterator.java +++ b/source/de/anomic/kelondro/kelondroCloneableMapIterator.java @@ -54,7 +54,6 @@ public class kelondroCloneableMapIterator implements kelondroCloneableIterato this.last = null; } - @SuppressWarnings("unchecked") public kelondroCloneableMapIterator clone(Object modifier) { return new kelondroCloneableMapIterator(map, modifier); } diff --git a/source/de/anomic/kelondro/kelondroCollectionIndex.java b/source/de/anomic/kelondro/kelondroCollectionIndex.java index 5bada8888..773b7dc1a 100644 --- a/source/de/anomic/kelondro/kelondroCollectionIndex.java +++ b/source/de/anomic/kelondro/kelondroCollectionIndex.java @@ -74,7 +74,7 @@ public class kelondroCollectionIndex { private static final int idx_col_lastread = 6; // a time stamp, update time in days since 1.1.2000 private static final int idx_col_lastwrote = 7; // a time stamp, update time in days since 1.1.2000 - private static kelondroRow indexRow(int keylength, kelondroOrder payloadOrder) { + private static kelondroRow indexRow(int keylength, kelondroByteOrder payloadOrder) { return new kelondroRow( "byte[] key-" + keylength + "," + "int chunksize-4 {b256}," + @@ -111,7 +111,7 @@ public class kelondroCollectionIndex { return new File(path, filenameStub + "." + lf + "." + cs + ".properties"); } - public kelondroCollectionIndex(File path, String filenameStub, int keyLength, kelondroOrder indexOrder, + public kelondroCollectionIndex(File path, String filenameStub, int keyLength, kelondroByteOrder indexOrder, long preloadTime, int loadfactor, int maxpartitions, kelondroRow rowdef) throws IOException { // the buffersize is number of bytes that are only used if the kelondroFlexTable is backed up with a kelondroTree this.path = path; @@ -161,7 +161,7 @@ public class kelondroCollectionIndex { } } - private void openAllArrayFiles(boolean indexGeneration, kelondroOrder indexOrder) throws IOException { + private void openAllArrayFiles(boolean indexGeneration, kelondroByteOrder indexOrder) throws IOException { String[] list = this.path.list(); kelondroFixedWidthArray array; @@ -222,7 +222,7 @@ public class kelondroCollectionIndex { } } - private kelondroIndex openIndexFile(File path, String filenameStub, kelondroOrder indexOrder, + private kelondroIndex openIndexFile(File path, String filenameStub, kelondroByteOrder indexOrder, long preloadTime, int loadfactor, kelondroRow rowdef, int initialSpace) throws IOException { // open/create index table kelondroIndex theindex = new kelondroCache(new kelondroFlexTable(path, filenameStub + ".index", preloadTime, indexRow(keylength, indexOrder), initialSpace, true), true, false); @@ -246,7 +246,7 @@ public class kelondroCollectionIndex { return theindex; } - private kelondroFixedWidthArray openArrayFile(int partitionNumber, int serialNumber, kelondroOrder indexOrder, boolean create) throws IOException { + private kelondroFixedWidthArray openArrayFile(int partitionNumber, int serialNumber, kelondroByteOrder indexOrder, boolean create) throws IOException { File f = arrayFile(path, filenameStub, loadfactor, payloadrow.objectsize, partitionNumber, serialNumber); int load = arrayCapacity(partitionNumber); kelondroRow rowdef = new kelondroRow( @@ -261,7 +261,7 @@ public class kelondroCollectionIndex { return a; } - private kelondroFixedWidthArray getArray(int partitionNumber, int serialNumber, kelondroOrder indexOrder, int chunksize) { + private kelondroFixedWidthArray getArray(int partitionNumber, int serialNumber, kelondroByteOrder indexOrder, int chunksize) { String accessKey = partitionNumber + "-" + chunksize; kelondroFixedWidthArray array = (kelondroFixedWidthArray) arrays.get(accessKey); if (array != null) return array; diff --git a/source/de/anomic/kelondro/kelondroDyn.java b/source/de/anomic/kelondro/kelondroDyn.java index 4a0c6fad8..7c9306242 100644 --- a/source/de/anomic/kelondro/kelondroDyn.java +++ b/source/de/anomic/kelondro/kelondroDyn.java @@ -71,7 +71,7 @@ public class kelondroDyn { private kelondroRow rowdef; public kelondroDyn(File file, boolean useNodeCache, boolean useObjectCache, long preloadTime, int key, - int nodesize, char fillChar, kelondroOrder objectOrder, boolean usetree, boolean writebuffer, boolean resetOnFail) { + int nodesize, char fillChar, kelondroByteOrder objectOrder, boolean usetree, boolean writebuffer, boolean resetOnFail) { // creates or opens a dynamic tree rowdef = new kelondroRow("byte[] key-" + (key + counterlen) + ", byte[] node-" + nodesize, objectOrder, 0); kelondroIndex fbi; @@ -160,7 +160,6 @@ public class kelondroDyn { nextKey = n(); } - @SuppressWarnings("unchecked") public dynKeyIterator clone(Object modifier) { return new dynKeyIterator(ri.clone(modifier)); } diff --git a/source/de/anomic/kelondro/kelondroFlexSplitTable.java b/source/de/anomic/kelondro/kelondroFlexSplitTable.java index 02978942e..403d49665 100644 --- a/source/de/anomic/kelondro/kelondroFlexSplitTable.java +++ b/source/de/anomic/kelondro/kelondroFlexSplitTable.java @@ -45,11 +45,13 @@ public class kelondroFlexSplitTable implements kelondroIndex { private kelondroRow rowdef; private File path; private String tablename; + private kelondroOrder entryOrder; public kelondroFlexSplitTable(File path, String tablename, long preloadTime, kelondroRow rowdef, boolean resetOnFail) { this.path = path; this.tablename = tablename; this.rowdef = rowdef; + this.entryOrder = new kelondroRow.EntryComparator(rowdef.objectOrder); init(preloadTime, resetOnFail); } @@ -192,7 +194,7 @@ public class kelondroFlexSplitTable implements kelondroIndex { return (kelondroRow.Entry) keeper[1]; } - public synchronized void putMultiple(List rows) throws IOException { + public synchronized void putMultiple(List rows) throws IOException { throw new UnsupportedOperationException("not yet implemented"); } @@ -287,8 +289,8 @@ public class kelondroFlexSplitTable implements kelondroIndex { } } - public synchronized kelondroCloneableIterator keys(boolean up, byte[] firstKey) throws IOException { - HashSet> set = new HashSet>(); + public synchronized kelondroCloneableIterator keys(boolean up, byte[] firstKey) throws IOException { + HashSet> set = new HashSet>(); Iterator i = tables.values().iterator(); while (i.hasNext()) { set.add(i.next().keys(up, firstKey)); @@ -296,13 +298,13 @@ public class kelondroFlexSplitTable implements kelondroIndex { return kelondroMergeIterator.cascade(set, rowdef.objectOrder, kelondroMergeIterator.simpleMerge, up); } - public synchronized kelondroCloneableIterator rows(boolean up, byte[] firstKey) throws IOException { - HashSet> set = new HashSet>(); + public synchronized kelondroCloneableIterator rows(boolean up, byte[] firstKey) throws IOException { + HashSet> set = new HashSet>(); Iterator i = tables.values().iterator(); while (i.hasNext()) { set.add(i.next().rows(up, firstKey)); } - return kelondroMergeIterator.cascade(set, rowdef.objectOrder, kelondroMergeIterator.simpleMerge, up); + return kelondroMergeIterator.cascade(set, entryOrder, kelondroMergeIterator.simpleMerge, up); } public final int cacheObjectChunkSize() { diff --git a/source/de/anomic/kelondro/kelondroFlexTable.java b/source/de/anomic/kelondro/kelondroFlexTable.java index 5070a263b..ce8218ea1 100644 --- a/source/de/anomic/kelondro/kelondroFlexTable.java +++ b/source/de/anomic/kelondro/kelondroFlexTable.java @@ -80,10 +80,10 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr System.out.println("*** Delete File index " + indexfile); indexfile.delete(); } - + // fill the index System.out.print("*** Loading RAM index for " + size() + " entries from "+ newpath); - index = initializeRamIndex(minimumSpace); + index = initializeRamIndex(minimumSpace); System.out.println(" -done-"); System.out.println(index.size() @@ -153,52 +153,30 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr } System.out.print(" -ordering- "); System.out.flush(); - //int sbu = ri.size(); - //if (ri.size() != sbu) serverLog.logSevere("kelondroFlexTable.initializeRamIndex: " + tablename, "; size before uniq = " + sbu + ", after uniq = " + ri.size()); return ri; } - private kelondroIndex initializeTreeIndex(File indexfile, long preloadTime, kelondroOrder objectOrder) throws IOException { - kelondroIndex treeindex = new kelondroCache(new kelondroTree(indexfile, true, preloadTime, treeIndexRow(rowdef.primaryKeyLength, objectOrder), 2, 80), true, false); - Iterator content = super.col[0].contentNodes(-1); - kelondroNode node; - kelondroRow.Entry indexentry; - int i, c = 0, all = super.col[0].size(); - long start = System.currentTimeMillis(); - long last = start; - while (content.hasNext()) { - node = content.next(); - i = node.handle().hashCode(); - indexentry = treeindex.row().newEntry(); - indexentry.setCol(0, node.getValueRow()); - indexentry.setCol(1, i); - treeindex.addUnique(indexentry); - c++; - if (System.currentTimeMillis() - last > 30000) { - System.out.println(".. generated " + c + "/" + all + " entries, " + ((System.currentTimeMillis() - start) / c * (all - c) / 60000) + " minutes remaining"); - System.out.flush(); - last = System.currentTimeMillis(); - } - } - return treeindex; - } - - private static final kelondroRow treeIndexRow(int keywidth, kelondroOrder objectOrder) { - return new kelondroRow("byte[] key-" + keywidth + ", int reference-4 {b256}", objectOrder, 0); - } - public synchronized kelondroRow.Entry get(byte[] key) throws IOException { if (index == null) return null; // case may happen during shutdown int pos = index.geti(key); assert this.size() == index.size() : "content.size() = " + this.size() + ", index.size() = " + index.size(); if (pos < 0) return null; - // i may be greater than this.size(), because this table may have deleted entries + // pos may be greater than this.size(), because this table may have deleted entries // the deleted entries are subtracted from the 'real' tablesize, // so the size may be smaller than an index to a row entry - kelondroRow.Entry result = super.get(pos); - assert result != null; - assert rowdef.objectOrder.compare(result.getPrimaryKeyBytes(), key) == 0 : "key and row does not match; key = " + serverLog.arrayList(key, 0, key.length) + " row.key = " + serverLog.arrayList(result.getPrimaryKeyBytes(), 0, rowdef.primaryKeyLength); - return result; + /*if (kelondroAbstractRecords.debugmode) { + kelondroRow.Entry result = super.get(pos); + assert result != null; + assert rowdef.objectOrder.compare(result.getPrimaryKeyBytes(), key) == 0 : "key and row does not match; key = " + serverLog.arrayList(key, 0, key.length) + " row.key = " + serverLog.arrayList(result.getPrimaryKeyBytes(), 0, rowdef.primaryKeyLength); + return result; + } else {*/ + // assume that the column for the primary key is 0, + // and the column 0 is stored in a file only for that column + // then we don't need to lookup from that file, because we already know the value (it's the key) + kelondroRow.Entry result = super.getOmitCol0(pos, key); + assert result != null; + return result; + //} } public synchronized void putMultiple(List rows) throws IOException { @@ -298,7 +276,7 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr assert this.size() == index.size() : "content.size() = " + this.size() + ", index.size() = " + index.size(); return null; } - kelondroRow.Entry r = super.get(i); + kelondroRow.Entry r = super.getOmitCol0(i, key); if (r == null) { serverLog.logSevere("kelondroFlexTable", "remove(): index failure; the index pointed to a cell which is empty. content.size() = " + this.size() + ", index.size() = " + ((index == null) ? 0 : index.size())); // patch bug ***** FIND CAUSE! (see also: put) diff --git a/source/de/anomic/kelondro/kelondroFlexWidthArray.java b/source/de/anomic/kelondro/kelondroFlexWidthArray.java index 18a6033ed..feec7767f 100644 --- a/source/de/anomic/kelondro/kelondroFlexWidthArray.java +++ b/source/de/anomic/kelondro/kelondroFlexWidthArray.java @@ -266,7 +266,7 @@ public class kelondroFlexWidthArray implements kelondroArray { return index; } - protected synchronized TreeMap addMultiple(List rows) throws IOException { + protected synchronized TreeMap addMultiple(List rows) throws IOException { // result is a Integer/byte[] relation // of newly added rows (index, key) TreeMap indexref = new TreeMap(); @@ -307,6 +307,7 @@ public class kelondroFlexWidthArray implements kelondroArray { public synchronized kelondroRow.Entry get(int index) throws IOException { kelondroRow.Entry e = col[0].getIfValid(index); + //assert e != null; if (e == null) return null; // probably a deleted entry kelondroRow.Entry p = rowdef.newEntry(); p.setCol(0, e.getColBytes(0)); @@ -320,6 +321,22 @@ public class kelondroFlexWidthArray implements kelondroArray { } return p; } + + public synchronized kelondroRow.Entry getOmitCol0(int index, byte[] col0) throws IOException { + assert col[0].row().columns() == 1; + kelondroRow.Entry p = rowdef.newEntry(); + kelondroRow.Entry e; + p.setCol(0, col0); + int r = 1; + while (r < rowdef.columns()) { + e = col[r].get(index); + for (int i = 0; i < col[r].row().columns(); i++) { + p.setCol(r + i, e.getColBytes(i)); + } + r = r + col[r].row().columns(); + } + return p; + } public synchronized void remove(int index) throws IOException { int r = 0; diff --git a/source/de/anomic/kelondro/kelondroIntBytesMap.java b/source/de/anomic/kelondro/kelondroIntBytesMap.java index a4f8d0d52..bdc4e1e11 100644 --- a/source/de/anomic/kelondro/kelondroIntBytesMap.java +++ b/source/de/anomic/kelondro/kelondroIntBytesMap.java @@ -40,11 +40,13 @@ public class kelondroIntBytesMap { private kelondroRow rowdef; private kelondroRowSet index0, index1; + private kelondroOrder entryOrder; public kelondroIntBytesMap(int payloadSize, int initSize) { - rowdef = new kelondroRow("Cardinal key-4 {b256}, byte[] payload-" + payloadSize, kelondroNaturalOrder.naturalOrder, 0); - index0 = new kelondroRowSet(rowdef, initSize); - index1 = null; + this.rowdef = new kelondroRow("Cardinal key-4 {b256}, byte[] payload-" + payloadSize, kelondroNaturalOrder.naturalOrder, 0); + this.index0 = new kelondroRowSet(rowdef, initSize); + this.index1 = null; + this.entryOrder = new kelondroRow.EntryComparator(rowdef.objectOrder); } public long memoryNeededForGrow() { @@ -193,7 +195,7 @@ public class kelondroIntBytesMap { return new kelondroMergeIterator( index0.rows(true, null), index1.rows(true, null), - kelondroRow.entryComparator, + entryOrder, kelondroMergeIterator.simpleMerge, true); } diff --git a/source/de/anomic/kelondro/kelondroMScoreCluster.java b/source/de/anomic/kelondro/kelondroMScoreCluster.java index cd38ef3ce..1708a7c9b 100644 --- a/source/de/anomic/kelondro/kelondroMScoreCluster.java +++ b/source/de/anomic/kelondro/kelondroMScoreCluster.java @@ -288,7 +288,6 @@ public final class kelondroMScoreCluster { return getScores(maxCount, up, Integer.MIN_VALUE, Integer.MAX_VALUE); } - @SuppressWarnings("unchecked") public synchronized E[] getScores(int maxCount, boolean up, int minScore, int maxScore) { if (maxCount > refkeyDB.size()) maxCount = refkeyDB.size(); E[] s = (E[]) new Object[maxCount]; @@ -325,7 +324,6 @@ public final class kelondroMScoreCluster { E n; int min, max; - @SuppressWarnings("unchecked") public komplexScoreIterator(boolean up, int minScore, int maxScore) { this.up = up; this.min = minScore; diff --git a/source/de/anomic/kelondro/kelondroMSetTools.java b/source/de/anomic/kelondro/kelondroMSetTools.java index 7a6a761a5..a486ea289 100644 --- a/source/de/anomic/kelondro/kelondroMSetTools.java +++ b/source/de/anomic/kelondro/kelondroMSetTools.java @@ -61,19 +61,13 @@ public class kelondroMSetTools { // ------------------------------------------------------------------------------------------------ // helper methods - - private static int compare(Object a, Object b, Comparator c) { - if (c != null) return c.compare(a,b); - if ((a instanceof String) && (b instanceof String)) return ((String) a).compareTo((String) b); - throw new ClassCastException(); - } - + public static int log2a(int x) { // this computes 1 + log2 // it is the number of bits in x, not the logarithmus by 2 - int l = 0; - while (x > 0) {x = x >>> 1; l++;} - return l; + int l = 0; + while (x > 0) {x = x >>> 1; l++;} + return l; } // ------------------------------------------------------------------------------------------------ @@ -178,7 +172,7 @@ public class kelondroMSetTools { Map.Entry mentry1 = mi1.next(); Map.Entry mentry2 = mi2.next(); while (true) { - c = compare(mentry1.getKey(), mentry2.getKey(), comp); + c = comp.compare(mentry1.getKey(), mentry2.getKey()); if (c < 0) { if (mi1.hasNext()) mentry1 = mi1.next(); else break; } else if (c > 0) { @@ -198,65 +192,65 @@ public class kelondroMSetTools { } // now the same for set-set - public static TreeSet joinConstructive(TreeSet set1, TreeSet set2) { - // comparators must be equal + public static TreeSet joinConstructive(TreeSet set1, TreeSet set2) { + // comparators must be equal if ((set1 == null) || (set2 == null)) return null; - if (set1.comparator() != set2.comparator()) return null; - if ((set1.size() == 0) || (set2.size() == 0)) return new TreeSet(set1.comparator()); + if (set1.comparator() != set2.comparator()) return null; + if ((set1.size() == 0) || (set2.size() == 0)) return new TreeSet(set1.comparator()); - // decide which method to use - int high = ((set1.size() > set2.size()) ? set1.size() : set2.size()); - int low = ((set1.size() > set2.size()) ? set2.size() : set1.size()); - int stepsEnum = 10 * (high + low - 1); - int stepsTest = 12 * log2a(high) * low; + // decide which method to use + int high = ((set1.size() > set2.size()) ? set1.size() : set2.size()); + int low = ((set1.size() > set2.size()) ? set2.size() : set1.size()); + int stepsEnum = 10 * (high + low - 1); + int stepsTest = 12 * log2a(high) * low; - // start most efficient method - if (stepsEnum > stepsTest) { - if (set1.size() < set2.size()) return joinConstructiveByTest(set1, set2); - return joinConstructiveByTest(set2, set1); - } - return joinConstructiveByEnumeration(set1, set2); + // start most efficient method + if (stepsEnum > stepsTest) { + if (set1.size() < set2.size()) return joinConstructiveByTest(set1, set2); + return joinConstructiveByTest(set2, set1); + } + return joinConstructiveByEnumeration(set1, set2); } - private static TreeSet joinConstructiveByTest(TreeSet small, TreeSet large) { - Iterator mi = small.iterator(); - TreeSet result = new TreeSet(small.comparator()); - Object o; - while (mi.hasNext()) { - o = mi.next(); - if (large.contains(o)) result.add(o); - } - return result; + private static TreeSet joinConstructiveByTest(TreeSet small, TreeSet large) { + Iterator mi = small.iterator(); + TreeSet result = new TreeSet(small.comparator()); + A o; + while (mi.hasNext()) { + o = mi.next(); + if (large.contains(o)) result.add(o); + } + return result; } - private static TreeSet joinConstructiveByEnumeration(TreeSet set1, TreeSet set2) { - // implement pairvise enumeration - Comparator comp = set1.comparator(); - Iterator mi = set1.iterator(); - Iterator si = set2.iterator(); - TreeSet result = new TreeSet(set1.comparator()); - int c; - if ((mi.hasNext()) && (si.hasNext())) { - Object mobj = mi.next(); - Object sobj = si.next(); - while (true) { - c = compare(mobj, sobj, comp); - if (c < 0) { - if (mi.hasNext()) mobj = mi.next(); else break; - } else if (c > 0) { - if (si.hasNext()) sobj = si.next(); else break; - } else { - result.add(mobj); - if (mi.hasNext()) mobj = mi.next(); else break; - if (si.hasNext()) sobj = si.next(); else break; - } - } - } - return result; + private static TreeSet joinConstructiveByEnumeration(TreeSet set1, TreeSet set2) { + // implement pairvise enumeration + Comparator comp = set1.comparator(); + Iterator mi = set1.iterator(); + Iterator si = set2.iterator(); + TreeSet result = new TreeSet(set1.comparator()); + int c; + if ((mi.hasNext()) && (si.hasNext())) { + A mobj = mi.next(); + A sobj = si.next(); + while (true) { + c = comp.compare(mobj, sobj); + if (c < 0) { + if (mi.hasNext()) mobj = mi.next(); else break; + } else if (c > 0) { + if (si.hasNext()) sobj = si.next(); else break; + } else { + result.add(mobj); + if (mi.hasNext()) mobj = mi.next(); else break; + if (si.hasNext()) sobj = si.next(); else break; + } + } + } + return result; } // now the same for set-set - public static boolean anymatch(TreeSet set1, TreeSet set2) { + public static boolean anymatch(TreeSet set1, TreeSet set2) { // comparators must be equal if ((set1 == null) || (set2 == null)) return false; if (set1.comparator() != set2.comparator()) return false; @@ -276,9 +270,9 @@ public class kelondroMSetTools { return anymatchByEnumeration(set1, set2); } - private static boolean anymatchByTest(TreeSet small, TreeSet large) { - Iterator mi = small.iterator(); - Object o; + private static boolean anymatchByTest(TreeSet small, TreeSet large) { + Iterator mi = small.iterator(); + A o; while (mi.hasNext()) { o = mi.next(); if (large.contains(o)) return true; @@ -286,17 +280,17 @@ public class kelondroMSetTools { return false; } - private static boolean anymatchByEnumeration(TreeSet set1, TreeSet set2) { + private static boolean anymatchByEnumeration(TreeSet set1, TreeSet set2) { // implement pairvise enumeration - Comparator comp = set1.comparator(); - Iterator mi = set1.iterator(); - Iterator si = set2.iterator(); + Comparator comp = set1.comparator(); + Iterator mi = set1.iterator(); + Iterator si = set2.iterator(); int c; if ((mi.hasNext()) && (si.hasNext())) { - Object mobj = mi.next(); - Object sobj = si.next(); + A mobj = mi.next(); + A sobj = si.next(); while (true) { - c = compare(mobj, sobj, comp); + c = comp.compare(mobj, sobj); if (c < 0) { if (mi.hasNext()) mobj = mi.next(); else break; } else if (c > 0) { @@ -312,7 +306,7 @@ public class kelondroMSetTools { // ------------------------------------------------------------------------------------------------ // exclude - public static TreeMap excludeConstructive(TreeMap map, TreeSet set) { + public static TreeMap excludeConstructive(TreeMap map, TreeSet set) { // comparators must be equal if (map == null) return null; if (set == null) return map; @@ -322,58 +316,18 @@ public class kelondroMSetTools { // return excludeConstructiveByEnumeration(map, set); } - private static TreeMap excludeConstructiveByTestMapInSet(TreeMap map, Set set) { - Iterator mi = map.keySet().iterator(); - TreeMap result = new TreeMap(map.comparator()); - Object o; + private static TreeMap excludeConstructiveByTestMapInSet(TreeMap map, TreeSet set) { + Iterator mi = map.keySet().iterator(); + TreeMap result = new TreeMap(map.comparator()); + A o; while (mi.hasNext()) { o = mi.next(); if (!(set.contains(o))) result.put(o, map.get(o)); } return result; } - - /* - private static TreeMap excludeConstructiveByEnumeration(TreeMap map, TreeSet set) { - // returns map without the elements in set - // enumerates objects - Comparator comp = map.comparator(); - Iterator mi = map.keySet().iterator(); - Iterator si = set.iterator(); - TreeMap result = new TreeMap(map.comparator()); - int c; - if ((mi.hasNext()) && (si.hasNext())) { - Object mobj = mi.next(); - Object sobj = si.next(); - while (true) { - c = compare(mobj, sobj, comp); - if (c < 0) { - result.put(mobj, map.get(mobj)); - if (mi.hasNext()) mobj = mi.next(); - else break; - } else if (c > 0) { - if (si.hasNext()) sobj = si.next(); - else break; - } else { - if (mi.hasNext()) mobj = mi.next(); - else break; - if (si.hasNext()) sobj = si.next(); - else { - // final flush - result.put(mobj, map.get(mobj)); - while (mi.hasNext()) { - mobj = mi.next(); - result.put(mobj, map.get(mobj)); - } - break; - } - } - } - } - return result; - } - */ - public static void excludeDestructive(TreeMap map, TreeSet set) { + + public static void excludeDestructive(TreeMap map, TreeSet set) { // comparators must be equal if (map == null) return; if (set == null) return; @@ -386,18 +340,18 @@ public class kelondroMSetTools { excludeDestructiveByTestSetInMap(map, set); } - private static void excludeDestructiveByTestMapInSet(TreeMap map, TreeSet set) { - Iterator mi = map.keySet().iterator(); + private static void excludeDestructiveByTestMapInSet(TreeMap map, TreeSet set) { + Iterator mi = map.keySet().iterator(); while (mi.hasNext()) if (set.contains(mi.next())) mi.remove(); } - private static void excludeDestructiveByTestSetInMap(TreeMap map, TreeSet set) { - Iterator si = set.iterator(); + private static void excludeDestructiveByTestSetInMap(TreeMap map, TreeSet set) { + Iterator si = set.iterator(); while (si.hasNext()) map.remove(si.next()); } // and the same again with set-set - public static void excludeDestructive(TreeSet set1, TreeSet set2) { + public static void excludeDestructive(TreeSet set1, TreeSet set2) { // comparators must be equal if (set1 == null) return; if (set2 == null) return; @@ -410,20 +364,20 @@ public class kelondroMSetTools { excludeDestructiveByTestLargeInSmall(set1, set2); } - private static void excludeDestructiveByTestSmallInLarge(TreeSet small, TreeSet large) { - Iterator mi = small.iterator(); + private static void excludeDestructiveByTestSmallInLarge(TreeSet small, TreeSet large) { + Iterator mi = small.iterator(); while (mi.hasNext()) if (large.contains(mi.next())) mi.remove(); } - private static void excludeDestructiveByTestLargeInSmall(TreeSet large, TreeSet small) { - Iterator si = small.iterator(); + private static void excludeDestructiveByTestLargeInSmall(TreeSet large, TreeSet small) { + Iterator si = small.iterator(); while (si.hasNext()) large.remove(si.next()); } // ------------------------------------------------------------------------------------------------ - public static TreeMap loadMap(String filename, String sep) { - TreeMap map = new TreeMap(); + public static TreeMap loadMap(String filename, String sep) { + TreeMap map = new TreeMap(); BufferedReader br = null; try { br = new BufferedReader(new InputStreamReader(new FileInputStream(filename))); @@ -441,8 +395,8 @@ public class kelondroMSetTools { return map; } - public static TreeMap /* > */ loadMapMultiValsPerKey(String filename, String sep) { - TreeMap map = new TreeMap(); + public static TreeMap> loadMapMultiValsPerKey(String filename, String sep) { + TreeMap> map = new TreeMap>(); BufferedReader br = null; try { br = new BufferedReader(new InputStreamReader(new FileInputStream(filename))); @@ -453,9 +407,8 @@ public class kelondroMSetTools { if ((line.length() > 0) && (!(line.startsWith("#"))) && ((pos = line.indexOf(sep)) > 0)) { key = line.substring(0, pos).trim().toLowerCase(); value = line.substring(pos + sep.length()).trim(); - if (!map.containsKey(key)) - map.put(key, new ArrayList()); - ((ArrayList)map.get(key)).add(value); + if (!map.containsKey(key)) map.put(key, new ArrayList()); + map.get(key).add(value); } } } catch (IOException e) { @@ -465,8 +418,8 @@ public class kelondroMSetTools { return map; } - public static TreeSet loadList(File file, Comparator c) { - TreeSet list = new TreeSet(c); + public static TreeSet loadList(File file, Comparator c) { + TreeSet list = new TreeSet(c); if (!(file.exists())) return list; BufferedReader br = null; @@ -485,12 +438,12 @@ public class kelondroMSetTools { return list; } - public static String setToString(Set set, char separator) { - Iterator i = set.iterator(); + public static String setToString(Set set, char separator) { + Iterator i = set.iterator(); StringBuffer sb = new StringBuffer(set.size() * 7); - if (i.hasNext()) sb.append(i.next().toString()); + if (i.hasNext()) sb.append(i.next()); while (i.hasNext()) { - sb.append(separator).append(i.next().toString()); + sb.append(separator).append(i.next()); } return new String(sb); } @@ -499,8 +452,8 @@ public class kelondroMSetTools { public static void main(String[] args) { - TreeMap m = new TreeMap(); - TreeMap s = new TreeMap(); + TreeMap m = new TreeMap(); + TreeMap s = new TreeMap(); m.put("a", "a"); m.put("x", "x"); m.put("f", "f"); @@ -530,7 +483,7 @@ public class kelondroMSetTools { System.out.println("Join=" + joinConstructiveByTest(m, s, true)); System.out.println("Join=" + joinConstructiveByTest(m, s, true)); System.out.println("Join=" + joinConstructive(m, s, true)); - System.out.println("Exclude=" + excludeConstructiveByTestMapInSet(m, s.keySet())); + //System.out.println("Exclude=" + excludeConstructiveByTestMapInSet(m, s.keySet())); /* for (int low = 0; low < 10; low++) diff --git a/source/de/anomic/kelondro/kelondroMapObjects.java b/source/de/anomic/kelondro/kelondroMapObjects.java index 341cf8a81..9065ee38b 100644 --- a/source/de/anomic/kelondro/kelondroMapObjects.java +++ b/source/de/anomic/kelondro/kelondroMapObjects.java @@ -45,7 +45,6 @@ public class kelondroMapObjects extends kelondroObjects { this(dyn, cachesize, null, null, null, null, null); } - @SuppressWarnings("unchecked") public kelondroMapObjects(kelondroDyn dyn, int cachesize, String[] sortfields, String[] longaccfields, String[] doubleaccfields, Method externalInitializer, Object externalHandler) { super(dyn, cachesize); diff --git a/source/de/anomic/kelondro/kelondroMapTable.java b/source/de/anomic/kelondro/kelondroMapTable.java index 114431b27..96bb88487 100644 --- a/source/de/anomic/kelondro/kelondroMapTable.java +++ b/source/de/anomic/kelondro/kelondroMapTable.java @@ -64,19 +64,19 @@ public class kelondroMapTable { } public void declareMaps( - String tablename, int keysize, kelondroOrder objectOrder, int nodesize, int cacheslots, + String tablename, int keysize, kelondroByteOrder objectOrder, int nodesize, int cacheslots, char fillChar, boolean resetOnFail) { declareMaps(tablename, keysize, objectOrder, nodesize, cacheslots, null, null, null, fillChar, resetOnFail); } public void declareMaps( - String tablename, int keysize, kelondroOrder objectOrder, int nodesize, int cacheslots, + String tablename, int keysize, kelondroByteOrder objectOrder, int nodesize, int cacheslots, String[] sortfields, String[] longaccfields, String[] doubleaccfields, char fillChar, boolean resetOnFail) { declareMaps(tablename, keysize, objectOrder, nodesize, cacheslots, sortfields, longaccfields, doubleaccfields, fillChar, 0, resetOnFail); } public void declareMaps( - String tablename, int keysize, kelondroOrder objectOrder, int nodesize, int cacheslots, + String tablename, int keysize, kelondroByteOrder objectOrder, int nodesize, int cacheslots, String[] sortfields, String[] longaccfields, String[] doubleaccfields, char fillChar, long preloadTime, boolean resetOnFail) { if (mTables.containsKey(tablename)) throw new RuntimeException("kelondroTables.declareMap: table '" + tablename + "' declared twice."); diff --git a/source/de/anomic/kelondro/kelondroMergeIterator.java b/source/de/anomic/kelondro/kelondroMergeIterator.java index c69f0f50d..8d6f0e470 100644 --- a/source/de/anomic/kelondro/kelondroMergeIterator.java +++ b/source/de/anomic/kelondro/kelondroMergeIterator.java @@ -67,7 +67,6 @@ public class kelondroMergeIterator implements kelondroCloneableIterator { nextb(); } - @SuppressWarnings("unchecked") public kelondroMergeIterator clone(Object modifier) { return new kelondroMergeIterator(a.clone(modifier), b.clone(modifier), comp, merger, up); } @@ -100,7 +99,6 @@ public class kelondroMergeIterator implements kelondroCloneableIterator { return (na != null) || (nb != null); } - @SuppressWarnings("unchecked") public E next() { E s; if (na == null) { @@ -148,21 +146,20 @@ public class kelondroMergeIterator implements kelondroCloneableIterator { throw new java.lang.UnsupportedOperationException("merge does not support remove"); } - public static kelondroCloneableIterator cascade(Set> /*of*/ iterators, Comparator c, Method merger, boolean up) { + public static kelondroCloneableIterator cascade(Set> /*of*/ iterators, kelondroOrder c, Method merger, boolean up) { // this extends the ability to combine two iterators // to the abiliy of combining a set of iterators if (iterators == null) return null; if (iterators.size() == 0) return null; - return cascade(iterators.iterator(), c, merger, up); + return cascade((Set>) iterators.iterator(), c, merger, up); } - @SuppressWarnings("unchecked") - private static kelondroCloneableIterator cascade(Iterator /*of*/ iiterators, Comparator c, Method merger, boolean up) { + private static kelondroCloneableIterator cascade(Iterator /*of*/ iiterators, kelondroOrder c, Method merger, boolean up) { if (iiterators == null) return null; if (!(iiterators.hasNext())) return null; - kelondroCloneableIterator one = (kelondroCloneableIterator) iiterators.next(); + kelondroCloneableIterator one = (kelondroCloneableIterator) iiterators.next(); if (!(iiterators.hasNext())) return one; - return new kelondroMergeIterator(one, cascade(iiterators, c, merger, up), c, merger, up); + return new kelondroMergeIterator(one, cascade(iiterators, c, merger, up), c, merger, up); } public static Method simpleMerge = null; diff --git a/source/de/anomic/kelondro/kelondroNaturalOrder.java b/source/de/anomic/kelondro/kelondroNaturalOrder.java index a4fc38c9e..3e0da4597 100644 --- a/source/de/anomic/kelondro/kelondroNaturalOrder.java +++ b/source/de/anomic/kelondro/kelondroNaturalOrder.java @@ -47,10 +47,10 @@ package de.anomic.kelondro; import java.util.Comparator; -public final class kelondroNaturalOrder extends kelondroAbstractOrder implements kelondroOrder, Comparator, Cloneable { - - public static final kelondroOrder naturalOrder = new kelondroNaturalOrder(true); +public final class kelondroNaturalOrder extends kelondroAbstractOrder implements kelondroByteOrder, Comparator, Cloneable { + public static final kelondroByteOrder naturalOrder = new kelondroNaturalOrder(true); + public static final Comparator naturalComparator = new kelondroByteOrder.StringOrder(naturalOrder); public kelondroNaturalOrder(boolean ascending) { this.asc = ascending; this.zero = null; @@ -64,13 +64,13 @@ public final class kelondroNaturalOrder extends kelondroAbstractOrder implements return true; } - public final Object clone() { + public final kelondroOrder clone() { kelondroNaturalOrder o = new kelondroNaturalOrder(this.asc); o.rotate(this.zero); return o; } - public final static kelondroOrder bySignature(String signature) { + public final static kelondroByteOrder bySignature(String signature) { if (signature.equals("nd")) return new kelondroNaturalOrder(false); if (signature.equals("nu")) return new kelondroNaturalOrder(true); return null; @@ -189,7 +189,7 @@ public final class kelondroNaturalOrder extends kelondroAbstractOrder implements byte[] t = new byte[12]; for (int i = 0; i < 12; i++) t[i] = (byte) 255; t[0] = (byte) 127; - kelondroOrder o = new kelondroNaturalOrder(true); + kelondroOrder o = new kelondroNaturalOrder(true); System.out.println(o.partition(t, 16)); } diff --git a/source/de/anomic/kelondro/kelondroOrder.java b/source/de/anomic/kelondro/kelondroOrder.java index 3806e2423..0f9e0a9ca 100644 --- a/source/de/anomic/kelondro/kelondroOrder.java +++ b/source/de/anomic/kelondro/kelondroOrder.java @@ -47,34 +47,25 @@ package de.anomic.kelondro; import java.util.Comparator; -import de.anomic.index.indexContainer; +public interface kelondroOrder extends Comparator { -public interface kelondroOrder extends Comparator { - - public boolean wellformed(byte[] a); // returns true if and only if a has only characters that belong to the implemented order - public boolean wellformed(byte[] a, int astart, int alength); + public boolean wellformed(A a); // returns true if and only if a has only characters that belong to the implemented order - public Object clone(); + public kelondroOrder clone(); public void direction(boolean ascending); // the ordering direction can be changed at any time public String signature(); // returns a signature String so that different orderings have different signatures - public long partition(byte[] key, int forkes); + public long partition(A key, int forkes); - public long cardinal(byte[] key); // returns a cardinal number in the range of 0 .. Long.MAX_VALUE + public long cardinal(A key); // returns a cardinal number in the range of 0 .. Long.MAX_VALUE - public int compare(byte[] a, byte[] b); - public int compare(String a, String b); - public int compare(kelondroNode a, kelondroNode b); - public int compare(indexContainer a, indexContainer b); - public int compare(kelondroRow.Entry a, kelondroRow.Entry b); - - public int compare(byte[] a, int aoffset, int alength, byte[] b, int boffset, int blength); + public int compare(A a, A b); - public byte[] zero(); // returns the zero point of the Ordering; null if not defined + public A zero(); // returns the zero point of the Ordering; null if not defined - public void rotate(byte[] zero); // defines that the ordering rotates, and sets the zero point for the rotation + public void rotate(A zero); // defines that the ordering rotates, and sets the zero point for the rotation - public boolean equals(kelondroOrder o); // used to compare different order objects; they may define the same ordering + public boolean equals(kelondroOrder o); // used to compare different order objects; they may define the same ordering } diff --git a/source/de/anomic/kelondro/kelondroRAMIndex.java b/source/de/anomic/kelondro/kelondroRAMIndex.java index a13c32bc8..6bc9cd571 100644 --- a/source/de/anomic/kelondro/kelondroRAMIndex.java +++ b/source/de/anomic/kelondro/kelondroRAMIndex.java @@ -35,9 +35,11 @@ public class kelondroRAMIndex implements kelondroIndex { private kelondroRow rowdef; private kelondroRowSet index0, index1; + private kelondroRow.EntryComparator entryComparator; public kelondroRAMIndex(kelondroRow rowdef, int initialspace) { this.rowdef = rowdef; + this.entryComparator = new kelondroRow.EntryComparator(rowdef.objectOrder); reset(initialspace); } @@ -153,8 +155,7 @@ public class kelondroRAMIndex implements kelondroIndex { return index0.size() + index1.size(); } - @SuppressWarnings("unchecked") - public synchronized kelondroCloneableIterator keys(boolean up, byte[] firstKey) throws IOException { + public synchronized kelondroCloneableIterator keys(boolean up, byte[] firstKey) throws IOException { // returns the key-iterator of the underlying kelondroIndex if (index1 == null) { // finish initialization phase @@ -169,7 +170,7 @@ public class kelondroRAMIndex implements kelondroIndex { return index1.keys(up, firstKey); } //assert consistencyAnalysis0() : "consistency problem: " + consistencyAnalysis(); - return new kelondroMergeIterator( + return new kelondroMergeIterator( index0.keys(up, firstKey), index1.keys(up, firstKey), rowdef.objectOrder, @@ -177,8 +178,7 @@ public class kelondroRAMIndex implements kelondroIndex { true); } - @SuppressWarnings("unchecked") - public synchronized kelondroCloneableIterator rows(boolean up, byte[] firstKey) throws IOException { + public synchronized kelondroCloneableIterator rows(boolean up, byte[] firstKey) throws IOException { // returns the row-iterator of the underlying kelondroIndex if (index1 == null) { // finish initialization phase @@ -193,10 +193,10 @@ public class kelondroRAMIndex implements kelondroIndex { return index1.rows(up, firstKey); } //assert consistencyAnalysis0() : "consistency problem: " + consistencyAnalysis(); - return new kelondroMergeIterator( + return new kelondroMergeIterator( index0.rows(up, firstKey), index1.rows(up, firstKey), - rowdef.objectOrder, + entryComparator, kelondroMergeIterator.simpleMerge, true); } diff --git a/source/de/anomic/kelondro/kelondroRotateIterator.java b/source/de/anomic/kelondro/kelondroRotateIterator.java index ef64f3d19..de0e0dc2c 100644 --- a/source/de/anomic/kelondro/kelondroRotateIterator.java +++ b/source/de/anomic/kelondro/kelondroRotateIterator.java @@ -38,7 +38,6 @@ public class kelondroRotateIterator implements kelondroCloneableIterator { this.clone = (kelondroCloneableIterator) a.clone(modifier); } - @SuppressWarnings("unchecked") public kelondroRotateIterator clone(Object modifier) { return new kelondroRotateIterator(a, modifier); } diff --git a/source/de/anomic/kelondro/kelondroRow.java b/source/de/anomic/kelondro/kelondroRow.java index 77f4544cb..016977485 100644 --- a/source/de/anomic/kelondro/kelondroRow.java +++ b/source/de/anomic/kelondro/kelondroRow.java @@ -41,16 +41,16 @@ public final class kelondroRow { protected kelondroColumn[] row; protected int[] colstart; - protected kelondroOrder objectOrder; + protected kelondroByteOrder objectOrder; public int objectsize; public int primaryKeyIndex, primaryKeyLength; protected Map nickref = null; // a mapping from nicknames to Object[2]{kelondroColumn, Integer(colstart)} - public kelondroRow(kelondroColumn[] row, kelondroOrder objectOrder, int primaryKey) { + public kelondroRow(kelondroColumn[] row, kelondroByteOrder objectOrder, int primaryKey) { assert objectOrder != null; + this.objectOrder = objectOrder; this.row = row; assert (objectOrder != null); - this.objectOrder = objectOrder; this.colstart = new int[row.length]; this.objectsize = 0; for (int i = 0; i < row.length; i++) { @@ -61,7 +61,7 @@ public final class kelondroRow { this.primaryKeyLength = (primaryKey < 0) ? this.objectsize : row[primaryKeyIndex].cellwidth; } - public kelondroRow(String structure, kelondroOrder objectOrder, int primaryKey) { + public kelondroRow(String structure, kelondroByteOrder objectOrder, int primaryKey) { assert (objectOrder != null); this.objectOrder = objectOrder; // define row with row syntax @@ -94,14 +94,14 @@ public final class kelondroRow { this.primaryKeyLength = (primaryKey < 0) ? this.objectsize : row[primaryKeyIndex].cellwidth; } - public final void setOrdering(kelondroOrder objectOrder, int primaryKey) { + public final void setOrdering(kelondroByteOrder objectOrder, int primaryKey) { assert (objectOrder != null); this.objectOrder = objectOrder; this.primaryKeyIndex = primaryKey; this.primaryKeyLength = (primaryKey < 0) ? this.objectsize : row[primaryKeyIndex].cellwidth; } - public final kelondroOrder getOrdering() { + public final kelondroOrder getOrdering() { return this.objectOrder; } @@ -191,14 +191,33 @@ public final class kelondroRow { return new EntryIndex(rowinstance, index); } - public static final Comparator entryComparator = new EntryComparator(); - - public static class EntryComparator implements Comparator { + public static class EntryComparator extends kelondroAbstractOrder implements kelondroOrder, Comparator { + kelondroByteOrder base; + public EntryComparator(kelondroByteOrder baseOrder) { + this.base = baseOrder; + } + public int compare(Entry a, Entry b) { return a.compareTo(b); } - + + public kelondroOrder clone() { + return new EntryComparator(base); + } + + public long cardinal(Entry key) { + return base.cardinal(key.getPrimaryKeyBytes()); + } + + public String signature() { + return base.signature(); + } + + public boolean wellformed(Entry a) { + return base.wellformed(a.getPrimaryKeyBytes()); + } + } public class Entry implements Comparable { diff --git a/source/de/anomic/kelondro/kelondroRowCollection.java b/source/de/anomic/kelondro/kelondroRowCollection.java index 1a71bccdb..93b06a691 100644 --- a/source/de/anomic/kelondro/kelondroRowCollection.java +++ b/source/de/anomic/kelondro/kelondroRowCollection.java @@ -98,7 +98,7 @@ public class kelondroRowCollection { this.lastTimeRead = (exportedCollection.getColLong(exp_last_read) + 10957) * day; this.lastTimeWrote = (exportedCollection.getColLong(exp_last_wrote) + 10957) * day; String sortOrderKey = exportedCollection.getColString(exp_order_type, null); - kelondroOrder oldOrder = null; + kelondroByteOrder oldOrder = null; if ((sortOrderKey == null) || (sortOrderKey.equals("__"))) { oldOrder = null; } else { diff --git a/source/de/anomic/kelondro/kelondroRowSet.java b/source/de/anomic/kelondro/kelondroRowSet.java index 7ddef98bf..81e481270 100644 --- a/source/de/anomic/kelondro/kelondroRowSet.java +++ b/source/de/anomic/kelondro/kelondroRowSet.java @@ -61,7 +61,7 @@ public class kelondroRowSet extends kelondroRowCollection implements kelondroInd this.profile = new kelondroProfile(); } - public void setOrdering(kelondroOrder newOrder, int newColumn) { + public void setOrdering(kelondroByteOrder newOrder, int newColumn) { assert newOrder != null; if ((rowdef.objectOrder == null) || (!(rowdef.objectOrder.signature().equals(newOrder.signature()))) || @@ -275,7 +275,6 @@ public class kelondroRowSet extends kelondroRowCollection implements kelondroInd } } - @SuppressWarnings("unchecked") public keyIterator clone(Object second) { return new keyIterator(up, (byte[]) second); } @@ -331,7 +330,6 @@ public class kelondroRowSet extends kelondroRowCollection implements kelondroInd } } - @SuppressWarnings("unchecked") public rowIterator clone(Object second) { return new rowIterator(up, (byte[]) second); } diff --git a/source/de/anomic/kelondro/kelondroSQLTable.java b/source/de/anomic/kelondro/kelondroSQLTable.java index fafc8355a..e3f5c73db 100644 --- a/source/de/anomic/kelondro/kelondroSQLTable.java +++ b/source/de/anomic/kelondro/kelondroSQLTable.java @@ -61,7 +61,7 @@ public class kelondroSQLTable implements kelondroIndex { private final String db_pwd_str = "yacy"; private Connection theDBConnection = null; - private final kelondroOrder order = new kelondroNaturalOrder(true); + private final kelondroByteOrder order = new kelondroNaturalOrder(true); private kelondroRow rowdef; public kelondroSQLTable(String dbType, kelondroRow rowdef) throws Exception { @@ -262,7 +262,7 @@ public class kelondroSQLTable implements kelondroIndex { return 0; } - public kelondroOrder order() { + public kelondroByteOrder order() { return this.order; } diff --git a/source/de/anomic/kelondro/kelondroSplittedTree.java b/source/de/anomic/kelondro/kelondroSplittedTree.java index 9dbdf98f8..31a568c47 100644 --- a/source/de/anomic/kelondro/kelondroSplittedTree.java +++ b/source/de/anomic/kelondro/kelondroSplittedTree.java @@ -55,7 +55,7 @@ import java.util.List; public class kelondroSplittedTree implements kelondroIndex { private kelondroTree[] ktfs; - private kelondroOrder order; + private kelondroByteOrder order; private int ff; private String filename; @@ -69,7 +69,7 @@ public class kelondroSplittedTree implements kelondroIndex { return new File(path, filenameStub + "." + ff + "." + co + "." + ns + ".ktc"); } - public kelondroSplittedTree(File pathToFiles, String filenameStub, kelondroOrder objectOrder, + public kelondroSplittedTree(File pathToFiles, String filenameStub, kelondroByteOrder objectOrder, long preloadTime, int forkfactor, kelondroRow rowdef, int txtProps, int txtPropsWidth) { try { @@ -118,7 +118,6 @@ public class kelondroSplittedTree implements kelondroIndex { return ktfs[partition(key)].get(key); } - @SuppressWarnings("unchecked") public synchronized void putMultiple(List rows) throws IOException { Iterator i = rows.iterator(); kelondroRow.Entry row; @@ -188,7 +187,6 @@ public class kelondroSplittedTree implements kelondroIndex { ktfsI = ktfs[c].rows(up, firstKey); // FIXME: this works only correct with firstKey == null } - @SuppressWarnings("unchecked") public rowIterator clone(Object secondKey) { try { return new rowIterator(up, (byte[]) secondKey); @@ -255,7 +253,6 @@ public class kelondroSplittedTree implements kelondroIndex { ktfsI = ktfs[c].keys(up, firstKey); // FIXME: this works only correct with firstKey == null } - @SuppressWarnings("unchecked") public keyIterator clone(Object secondKey) { try { return new keyIterator(up, (byte[]) secondKey); @@ -305,7 +302,7 @@ public class kelondroSplittedTree implements kelondroIndex { } - public kelondroOrder order() { + public kelondroByteOrder order() { return this.order; } diff --git a/source/de/anomic/kelondro/kelondroTree.java b/source/de/anomic/kelondro/kelondroTree.java index 0816a27ab..5c80291a8 100644 --- a/source/de/anomic/kelondro/kelondroTree.java +++ b/source/de/anomic/kelondro/kelondroTree.java @@ -50,6 +50,7 @@ import java.io.File; import java.io.FileReader; import java.io.IOException; import java.io.RandomAccessFile; +import java.util.Comparator; import java.util.Date; import java.util.HashSet; import java.util.Iterator; @@ -86,11 +87,11 @@ public class kelondroTree extends kelondroCachedRecords implements kelondroIndex protected static final int root = 0; // pointer for FHandles-array: pointer to root node // class variables - private final Search writeSearchObj = new Search(); - protected kelondroOrder loopDetectionOrder = new kelondroNaturalOrder(true); - protected int readAheadChunkSize = 100; - protected long lastIteratorCount = readAheadChunkSize; - + private final Search writeSearchObj = new Search(); + protected Comparator loopDetectionOrder; + protected int readAheadChunkSize = 100; + protected long lastIteratorCount = readAheadChunkSize; + public kelondroTree(File file, boolean useNodeCache, long preloadTime, kelondroRow rowdef) throws IOException { this(file, useNodeCache, preloadTime, rowdef, rowdef.columns() /* txtProps */, 80 /* txtPropWidth */); } @@ -105,7 +106,7 @@ public class kelondroTree extends kelondroCachedRecords implements kelondroIndex setHandle(root, null); // define the root value } super.setLogger(log); - + this.loopDetectionOrder = new kelondroByteOrder.StringOrder(rowdef.objectOrder); } public static final kelondroTree open(File file, boolean useNodeCache, long preloadTime, kelondroRow rowdef) { @@ -135,7 +136,7 @@ public class kelondroTree extends kelondroCachedRecords implements kelondroIndex this(ra, filename, useNodeCache, preloadTime, rowdef, new kelondroNaturalOrder(true), rowdef.columns() /* txtProps */, 80 /* txtPropWidth */, exitOnFail); } - public kelondroTree(kelondroRA ra, String filename, boolean useNodeCache, long preloadTime, kelondroRow rowdef, kelondroOrder objectOrder, int txtProps, int txtPropsWidth, boolean exitOnFail) { + public kelondroTree(kelondroRA ra, String filename, boolean useNodeCache, long preloadTime, kelondroRow rowdef, kelondroByteOrder objectOrder, int txtProps, int txtPropsWidth, boolean exitOnFail) { // this creates a new tree within a kelondroRA super(ra, filename, useNodeCache, preloadTime, thisOHBytes, thisOHHandles, rowdef, @@ -148,6 +149,7 @@ public class kelondroTree extends kelondroCachedRecords implements kelondroIndex throw new RuntimeException("cannot set root handle / " + e.getMessage()); } super.setLogger(log); + this.loopDetectionOrder = new kelondroByteOrder.StringOrder(rowdef.objectOrder); } public kelondroTree(kelondroRA ra, String filename, boolean useNodeCache, long preloadTime) throws IOException { @@ -272,7 +274,7 @@ public class kelondroTree extends kelondroCachedRecords implements kelondroIndex return; } // System.out.println("Comparing key = '" + new String(key) + "' with '" + otherkey + "':"); // debug - c = row().objectOrder.compare(new String(key), keybuffer); + c = row().objectOrder.compare(key, keybuffer.getBytes()); // System.out.println(c); // debug if (c == 0) { found = true; @@ -988,10 +990,10 @@ public class kelondroTree extends kelondroCachedRecords implements kelondroIndex public TreeMap rowMap(boolean up, byte[] firstKey, boolean including, int count) throws IOException { // returns an ordered map of keys/row relations; key objects are of type String, value objects are of type byte[][] - kelondroOrder setOrder = (kelondroOrder) row().objectOrder.clone(); + kelondroByteOrder setOrder = (kelondroByteOrder) row().objectOrder.clone(); setOrder.direction(up); setOrder.rotate(firstKey); - TreeMap rows = new TreeMap(setOrder); + TreeMap rows = new TreeMap(this.loopDetectionOrder); CacheNode n; String key; synchronized (this) { @@ -1008,10 +1010,10 @@ public class kelondroTree extends kelondroCachedRecords implements kelondroIndex public TreeSet keySet(boolean up, boolean rotating, byte[] firstKey, boolean including, int count) throws IOException { // returns an ordered set of keys; objects are of type String - kelondroOrder setOrder = (kelondroOrder) row().objectOrder.clone(); + kelondroByteOrder setOrder = (kelondroByteOrder) row().objectOrder.clone(); setOrder.direction(up); setOrder.rotate(firstKey); - TreeSet set = new TreeSet(setOrder); + TreeSet set = new TreeSet(this.loopDetectionOrder); kelondroNode n; synchronized (this) { Iterator i = (firstKey == null) ? new nodeIterator(up, rotating) : new nodeIterator(up, rotating, firstKey, including); @@ -1054,7 +1056,6 @@ public class kelondroTree extends kelondroCachedRecords implements kelondroIndex lastIteratorCount = 0; } - @SuppressWarnings("unchecked") public rowIterator clone(Object secondStart) { try { return new rowIterator(inc, (byte[]) secondStart, guessedCountLimit); @@ -1128,7 +1129,6 @@ public class kelondroTree extends kelondroCachedRecords implements kelondroIndex lastIteratorCount = 0; } - @SuppressWarnings("unchecked") public keyIterator clone(Object secondStart) { try { return new keyIterator(inc, (byte[]) secondStart, guessedCountLimit); diff --git a/source/de/anomic/kelondro/kelondroXMLTables.java b/source/de/anomic/kelondro/kelondroXMLTables.java index b8e549002..1d452ed64 100644 --- a/source/de/anomic/kelondro/kelondroXMLTables.java +++ b/source/de/anomic/kelondro/kelondroXMLTables.java @@ -68,7 +68,6 @@ public class kelondroXMLTables { this.tables = new Hashtable>(); } - @SuppressWarnings("unchecked") public kelondroXMLTables(File file) throws IOException { this.propFile = file; this.timestamp = System.currentTimeMillis(); diff --git a/source/de/anomic/plasma/plasmaCondenser.java b/source/de/anomic/plasma/plasmaCondenser.java index 274de3d5d..e2400be1d 100644 --- a/source/de/anomic/plasma/plasmaCondenser.java +++ b/source/de/anomic/plasma/plasmaCondenser.java @@ -267,7 +267,7 @@ public final class plasmaCondenser { } public static final Set words2hashSet(String[] words) { - TreeSet hashes = new TreeSet(kelondroBase64Order.enhancedCoder); + TreeSet hashes = new TreeSet(kelondroBase64Order.enhancedComparator); for (int i = 0; i < words.length; i++) hashes.add(word2hash(words[i])); return hashes; } @@ -280,7 +280,7 @@ public final class plasmaCondenser { public static final TreeSet words2hashes(Set words) { Iterator i = words.iterator(); - TreeSet hashes = new TreeSet(kelondroBase64Order.enhancedCoder); + TreeSet hashes = new TreeSet(kelondroBase64Order.enhancedComparator); while (i.hasNext()) hashes.add(word2hash(i.next())); return hashes; } diff --git a/source/de/anomic/plasma/plasmaCrawlLURL.java b/source/de/anomic/plasma/plasmaCrawlLURL.java index 71c44b9c8..70ae23db6 100644 --- a/source/de/anomic/plasma/plasmaCrawlLURL.java +++ b/source/de/anomic/plasma/plasmaCrawlLURL.java @@ -86,12 +86,12 @@ public final class plasmaCrawlLURL { // result stacks; // these have all entries of form // strings: urlHash + initiatorHash + ExecutorHash - private final LinkedList externResultStack; // 1 - remote index: retrieved by other peer - private final LinkedList searchResultStack; // 2 - partly remote/local index: result of search queries - private final LinkedList transfResultStack; // 3 - partly remote/local index: result of index transfer - private final LinkedList proxyResultStack; // 4 - local index: result of proxy fetch/prefetch - private final LinkedList lcrawlResultStack; // 5 - local index: result of local crawling - private final LinkedList gcrawlResultStack; // 6 - local index: triggered external + private final LinkedList externResultStack; // 1 - remote index: retrieved by other peer + private final LinkedList searchResultStack; // 2 - partly remote/local index: result of search queries + private final LinkedList transfResultStack; // 3 - partly remote/local index: result of index transfer + private final LinkedList proxyResultStack; // 4 - local index: result of proxy fetch/prefetch + private final LinkedList lcrawlResultStack; // 5 - local index: result of local crawling + private final LinkedList gcrawlResultStack; // 6 - local index: triggered external // the class object private kelondroIndex urlIndexFile; @@ -102,12 +102,12 @@ public final class plasmaCrawlLURL { urlIndexFile = new kelondroFlexSplitTable(new File(indexPath, "PUBLIC/TEXT"), "urls", preloadTime, indexURLEntry.rowdef, false); // init result stacks - externResultStack = new LinkedList(); - searchResultStack = new LinkedList(); - transfResultStack = new LinkedList(); - proxyResultStack = new LinkedList(); - lcrawlResultStack = new LinkedList(); - gcrawlResultStack = new LinkedList(); + externResultStack = new LinkedList(); + searchResultStack = new LinkedList(); + transfResultStack = new LinkedList(); + proxyResultStack = new LinkedList(); + lcrawlResultStack = new LinkedList(); + gcrawlResultStack = new LinkedList(); } public int size() { @@ -312,14 +312,14 @@ public final class plasmaCrawlLURL { } } - public kelondroCloneableIterator entries(boolean up, String firstHash) throws IOException { + public kelondroCloneableIterator entries(boolean up, String firstHash) throws IOException { // enumerates entry elements return new kiter(up, firstHash); } - public class kiter implements kelondroCloneableIterator { + public class kiter implements kelondroCloneableIterator { // enumerates entry elements - private Iterator iter; + private Iterator iter; private boolean error; boolean up; @@ -343,10 +343,10 @@ public final class plasmaCrawlLURL { return this.iter.hasNext(); } - public final Object next() { + public final indexURLEntry next() { kelondroRow.Entry e = null; if (this.iter == null) { return null; } - if (this.iter.hasNext()) { e = (kelondroRow.Entry) this.iter.next(); } + if (this.iter.hasNext()) { e = this.iter.next(); } if (e == null) { return null; } return new indexURLEntry(e, null, 0); } @@ -364,9 +364,9 @@ public final class plasmaCrawlLURL { */ public void urldbcleanup() { serverLog log = new serverLog("URLDBCLEANUP"); - HashSet damagedURLS = new HashSet(); + HashSet damagedURLS = new HashSet(); try { - Iterator eiter = entries(true, null); + Iterator eiter = entries(true, null); int iteratorCount = 0; while (eiter.hasNext()) try { eiter.next(); @@ -382,7 +382,7 @@ public final class plasmaCrawlLURL { try { Thread.sleep(1000); } catch (InterruptedException e) { } log.logInfo("URLs vorher: " + size() + " Entries loaded during Iteratorloop: " + iteratorCount + " kaputte URLs: " + damagedURLS.size()); - Iterator eiter2 = damagedURLS.iterator(); + Iterator eiter2 = damagedURLS.iterator(); String urlHash; while (eiter2.hasNext()) { urlHash = (String) eiter2.next(); @@ -455,7 +455,7 @@ public final class plasmaCrawlLURL { public void run() { try { serverLog.logInfo("URLDBCLEANER", "UrldbCleaner-Thread startet"); - final Iterator eiter = entries(true, null); + final Iterator eiter = entries(true, null); while (eiter.hasNext() && run) { synchronized (this) { if (this.pause) { @@ -468,7 +468,7 @@ public final class plasmaCrawlLURL { } } } - final indexURLEntry entry = (indexURLEntry) eiter.next(); + final indexURLEntry entry = eiter.next(); if (entry == null) { serverLog.logFine("URLDBCLEANER", "entry == null"); } else if (entry.hash() == null) { @@ -605,12 +605,12 @@ public final class plasmaCrawlLURL { pw.println("http://yacy.net"); } - Iterator i = entries(true, null); // iterates indexURLEntry objects + Iterator i = entries(true, null); // iterates indexURLEntry objects indexURLEntry entry; indexURLEntry.Components comp; String url; loop: while (i.hasNext()) { - entry = (indexURLEntry) i.next(); + entry = i.next(); comp = entry.comp(); url = comp.url().toNormalform(true, false); if (!url.matches(filter)) continue; @@ -683,9 +683,9 @@ public final class plasmaCrawlLURL { if (args[0].equals("-l")) try { // arg 1 is path to URLCache final plasmaCrawlLURL urls = new plasmaCrawlLURL(new File(args[2]), 0); - final Iterator enu = urls.entries(true, null); + final Iterator enu = urls.entries(true, null); while (enu.hasNext()) { - System.out.println(((indexURLEntry) enu.next()).toString()); + System.out.println(enu.next().toString()); } } catch (Exception e) { e.printStackTrace(); diff --git a/source/de/anomic/plasma/plasmaDHTChunk.java b/source/de/anomic/plasma/plasmaDHTChunk.java index 2e2abe194..9e6024b90 100644 --- a/source/de/anomic/plasma/plasmaDHTChunk.java +++ b/source/de/anomic/plasma/plasmaDHTChunk.java @@ -52,6 +52,7 @@ import java.util.Iterator; import de.anomic.index.indexContainer; import de.anomic.index.indexRWIEntry; +import de.anomic.index.indexRWIRowEntry; import de.anomic.index.indexURLEntry; import de.anomic.kelondro.kelondroBase64Order; import de.anomic.kelondro.kelondroException; @@ -78,7 +79,7 @@ public class plasmaDHTChunk { private int status = chunkStatus_UNDEFINED; private String startPointHash = "AAAAAAAAAAAA"; private indexContainer[] indexContainers = null; - private HashMap urlCache; // String (url-hash) / plasmaCrawlLURL.Entry + private HashMap urlCache; // String (url-hash) / plasmaCrawlLURL.Entry private int idxCount; private long selectionStartTime = 0; @@ -114,7 +115,7 @@ public class plasmaDHTChunk { return c; } - public HashMap urlCacheMap() { + public HashMap urlCacheMap() { return urlCache; } @@ -207,17 +208,17 @@ public class plasmaDHTChunk { private int selectTransferContainersResource(String hash, boolean ram, int maxcount, int maxtime) throws InterruptedException { // if (maxcount > 500) { maxcount = 500; } // flooding & OOM reduce // the hash is a start hash from where the indexes are picked - final ArrayList tmpContainers = new ArrayList(maxcount); + final ArrayList tmpContainers = new ArrayList(maxcount); try { - final Iterator indexContainerIterator = wordIndex.indexContainerSet(hash, ram, true, maxcount).iterator(); + final Iterator indexContainerIterator = wordIndex.indexContainerSet(hash, ram, true, maxcount).iterator(); indexContainer container; - Iterator urlIter; + Iterator urlIter; indexRWIEntry iEntry; indexURLEntry lurl; int refcount = 0; int wholesize; - urlCache = new HashMap(); + urlCache = new HashMap(); final double maximumDistance = ((double) peerRedundancy * 2) / ((double) yacyCore.seedDB.sizeConnected()); final long timeout = (maxtime < 0) ? Long.MAX_VALUE : System.currentTimeMillis() + maxtime; while ( @@ -226,7 +227,7 @@ public class plasmaDHTChunk { ((container = (indexContainer) indexContainerIterator.next()) != null) && (container.size() > 0) && ((tmpContainers.size() == 0) || - (yacyDHTAction.dhtDistance(container.getWordHash(), ((indexContainer) tmpContainers.get(0)).getWordHash()) < maximumDistance)) && + (yacyDHTAction.dhtDistance(container.getWordHash(), tmpContainers.get(0).getWordHash()) < maximumDistance)) && (System.currentTimeMillis() < timeout) ) { // check for interruption @@ -288,16 +289,16 @@ public class plasmaDHTChunk { } catch (kelondroException e) { log.logSevere("selectTransferIndexes database corrupted: " + e.getMessage(), e); indexContainers = new indexContainer[0]; - urlCache = new HashMap(); + urlCache = new HashMap(); this.status = chunkStatus_FAILED; return 0; } } public synchronized String deleteTransferIndexes() { - Iterator urlIter; + Iterator urlIter; indexRWIEntry iEntry; - HashSet urlHashes; + HashSet urlHashes; String count = "0"; for (int i = 0; i < this.indexContainers.length; i++) { @@ -307,10 +308,10 @@ public class plasmaDHTChunk { continue; } int c = this.indexContainers[i].size(); - urlHashes = new HashSet(this.indexContainers[i].size()); + urlHashes = new HashSet(this.indexContainers[i].size()); urlIter = this.indexContainers[i].entries(); while (urlIter.hasNext()) { - iEntry = (indexRWIEntry) urlIter.next(); + iEntry = urlIter.next(); urlHashes.add(iEntry.urlHash()); } String wordHash = indexContainers[i].getWordHash(); diff --git a/source/de/anomic/plasma/plasmaRankingRCIEvaluation.java b/source/de/anomic/plasma/plasmaRankingRCIEvaluation.java index c5ae283f5..9b526fc9f 100644 --- a/source/de/anomic/plasma/plasmaRankingRCIEvaluation.java +++ b/source/de/anomic/plasma/plasmaRankingRCIEvaluation.java @@ -155,7 +155,7 @@ public class plasmaRankingRCIEvaluation { public static TreeSet[] genRankingTable(kelondroAttrSeq rci, int[] partition) { TreeSet[] ranked = new TreeSet[partition.length]; - for (int i = 0; i < partition.length; i++) ranked[i] = new TreeSet(kelondroBase64Order.enhancedCoder); + for (int i = 0; i < partition.length; i++) ranked[i] = new TreeSet(kelondroBase64Order.enhancedComparator); Iterator i = rci.keys(); String key; kelondroAttrSeq.Entry entry; diff --git a/source/de/anomic/plasma/plasmaSearchEvent.java b/source/de/anomic/plasma/plasmaSearchEvent.java index 1d0f94165..df50b07cb 100644 --- a/source/de/anomic/plasma/plasmaSearchEvent.java +++ b/source/de/anomic/plasma/plasmaSearchEvent.java @@ -72,11 +72,12 @@ public final class plasmaSearchEvent { private Thread localSearchThread; private TreeMap preselectedPeerHashes; //private Object[] references; - public TreeMap IAResults, IACount; + public TreeMap IAResults; + public TreeMap IACount; public String IAmaxcounthash, IAneardhthash; private int localcount; private resultWorker[] workerThreads; - private ArrayList resultList; // list of this.Entry objects + private ArrayList resultList; //private int resultListLock; // a pointer that shows that all elements below this pointer are fixed and may not be changed again private HashMap failedURLs; // a mapping from a urlhash to a fail reason string TreeSet snippetFetchWordHashes; // a set of word hashes that are used to match with the snippets @@ -605,7 +606,7 @@ public final class plasmaSearchEvent { } } - public ArrayList completeResults(long waitingtime) { + public ArrayList completeResults(long waitingtime) { long timeout = System.currentTimeMillis() + waitingtime; while ((this.resultList.size() < query.neededResults()) && (anyWorkerAlive()) && (System.currentTimeMillis() < timeout)) { try {Thread.sleep(200);} catch (InterruptedException e) {} @@ -718,7 +719,7 @@ public final class plasmaSearchEvent { //assert e != null; } - public Set references(int count) { + public Set references(int count) { // returns a set of words that are computed as toplist return this.rankedCache.getReferences(count); } @@ -730,12 +731,14 @@ public final class plasmaSearchEvent { private String alternative_urlstring; private String alternative_urlname; private plasmaSnippetCache.TextSnippet textSnippet; - private ArrayList /* of plasmaSnippetCache.MediaSnippet */ mediaSnippets; + private ArrayList mediaSnippets; // statistic objects public long dbRetrievalTime, snippetComputationTime; - public ResultEntry(indexURLEntry urlentry, plasmaWordIndex wordIndex, plasmaSnippetCache.TextSnippet textSnippet, ArrayList mediaSnippets, + public ResultEntry(indexURLEntry urlentry, plasmaWordIndex wordIndex, + plasmaSnippetCache.TextSnippet textSnippet, + ArrayList mediaSnippets, long dbRetrievalTime, long snippetComputationTime) { this.urlentry = urlentry; this.urlcomps = urlentry.comp(); @@ -796,7 +799,7 @@ public final class plasmaSearchEvent { public plasmaSnippetCache.TextSnippet textSnippet() { return this.textSnippet; } - public ArrayList /* of plasmaSnippetCache.MediaSnippet */ mediaSnippets() { + public ArrayList mediaSnippets() { return this.mediaSnippets; } public Date modified() { diff --git a/source/de/anomic/plasma/plasmaSearchQuery.java b/source/de/anomic/plasma/plasmaSearchQuery.java index c15dbf46b..78d4b81e2 100644 --- a/source/de/anomic/plasma/plasmaSearchQuery.java +++ b/source/de/anomic/plasma/plasmaSearchQuery.java @@ -184,8 +184,8 @@ public plasmaSearchQuery( } public static TreeSet hashes2Set(String query) { - if (query == null) return new TreeSet(kelondroBase64Order.enhancedCoder); - final TreeSet keyhashes = new TreeSet(kelondroBase64Order.enhancedCoder); + if (query == null) return new TreeSet(kelondroBase64Order.enhancedComparator); + final TreeSet keyhashes = new TreeSet(kelondroBase64Order.enhancedComparator); for (int i = 0; i < (query.length() / yacySeedDB.commonHashLength); i++) { keyhashes.add(query.substring(i * yacySeedDB.commonHashLength, (i + 1) * yacySeedDB.commonHashLength)); } @@ -226,7 +226,7 @@ public plasmaSearchQuery( public static TreeSet[] cleanQuery(String querystring) { // returns two sets: a query set and a exclude set - if ((querystring == null) || (querystring.length() == 0)) return new TreeSet[]{new TreeSet(kelondroNaturalOrder.naturalOrder), new TreeSet(kelondroNaturalOrder.naturalOrder)}; + if ((querystring == null) || (querystring.length() == 0)) return new TreeSet[]{new TreeSet(kelondroNaturalOrder.naturalComparator), new TreeSet(kelondroNaturalOrder.naturalComparator)}; // convert Umlaute querystring = htmlFilterAbstractScraper.convertUmlaute(new serverCharBuffer(querystring.toCharArray())).toString(); @@ -240,8 +240,8 @@ public plasmaSearchQuery( } // the string is clean now, but we must generate a set out of it - final TreeSet query = new TreeSet(kelondroNaturalOrder.naturalOrder); - final TreeSet exclude = new TreeSet(kelondroNaturalOrder.naturalOrder); + final TreeSet query = new TreeSet(kelondroNaturalOrder.naturalComparator); + final TreeSet exclude = new TreeSet(kelondroNaturalOrder.naturalComparator); final String[] a = querystring.split(" "); for (int i = 0; i < a.length; i++) { if (a[i].startsWith("-")) { diff --git a/source/de/anomic/plasma/plasmaSwitchboard.java b/source/de/anomic/plasma/plasmaSwitchboard.java index 086ea210a..c463f8dd5 100644 --- a/source/de/anomic/plasma/plasmaSwitchboard.java +++ b/source/de/anomic/plasma/plasmaSwitchboard.java @@ -173,7 +173,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser // 3) result of index transfer, some of them are here (not possible here) // 4) proxy-load (initiator is "------------") // 5) local prefetch/crawling (initiator is own seedHash) - // 6) local fetching for global crawling (other known or unknwon initiator) + // 6) local fetching for global crawling (other known or unknown initiator) public static final int PROCESSCASE_0_UNKNOWN = 0; public static final int PROCESSCASE_1_GLOBAL_CRAWLING = 1; public static final int PROCESSCASE_2_SEARCH_QUERY_RESULT = 2; @@ -183,9 +183,9 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser public static final int PROCESSCASE_6_GLOBAL_CRAWLING = 6; // couloured list management - public static TreeSet badwords = null; - public static TreeSet blueList = null; - public static TreeSet stopwords = null; + public static TreeSet badwords = null; + public static TreeSet blueList = null; + public static TreeSet stopwords = null; public static plasmaURLPattern urlBlacklist; public static wikiParser wikiParser = null; @@ -199,7 +199,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser public File rankingPath; public File workPath; public File releasePath; - public HashMap rankingPermissions; + public HashMap rankingPermissions; public plasmaWordIndex wordIndex; public plasmaCrawlQueues crawlQueues; public plasmaSwitchboardQueue sbQueue; @@ -230,7 +230,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser public plasmaDHTFlush transferIdxThread = null; private plasmaDHTChunk dhtTransferChunk = null; public ArrayList> localSearches, remoteSearches; // array of search result properties as HashMaps - public HashMap localSearchTracker, remoteSearchTracker; // mappings from requesting host to a TreeSet of Long(access time) + public HashMap> localSearchTracker, remoteSearchTracker; // mappings from requesting host to a TreeSet of Long(access time) public long lastseedcheckuptime = -1; public long indexedPages = 0; public long lastindexedPages = 0; @@ -985,7 +985,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser // read only once upon first instantiation of this class String f = getConfig(LIST_BLUE, LIST_BLUE_DEFAULT); File plasmaBlueListFile = new File(f); - if (f != null) blueList = kelondroMSetTools.loadList(plasmaBlueListFile, kelondroNaturalOrder.naturalOrder); else blueList= new TreeSet(); + if (f != null) blueList = kelondroMSetTools.loadList(plasmaBlueListFile, kelondroNaturalOrder.naturalComparator); else blueList= new TreeSet(); this.log.logConfig("loaded blue-list from file " + plasmaBlueListFile.getName() + ", " + blueList.size() + " entries, " + ppRamString(plasmaBlueListFile.length()/1024)); @@ -1018,7 +1018,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser // load badwords (to filter the topwords) if (badwords == null) { File badwordsFile = new File(rootPath, LIST_BADWORDS_DEFAULT); - badwords = kelondroMSetTools.loadList(badwordsFile, kelondroNaturalOrder.naturalOrder); + badwords = kelondroMSetTools.loadList(badwordsFile, kelondroNaturalOrder.naturalComparator); this.log.logConfig("loaded badwords from file " + badwordsFile.getName() + ", " + badwords.size() + " entries, " + ppRamString(badwordsFile.length()/1024)); @@ -1027,7 +1027,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser // load stopwords if (stopwords == null) { File stopwordsFile = new File(rootPath, LIST_STOPWORDS_DEFAULT); - stopwords = kelondroMSetTools.loadList(stopwordsFile, kelondroNaturalOrder.naturalOrder); + stopwords = kelondroMSetTools.loadList(stopwordsFile, kelondroNaturalOrder.naturalComparator); this.log.logConfig("loaded stopwords from file " + stopwordsFile.getName() + ", " + stopwords.size() + " entries, " + ppRamString(stopwordsFile.length()/1024)); @@ -1205,8 +1205,8 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser this.incomingCookies = new HashMap(); // init search history trackers - this.localSearchTracker = new HashMap(); // String:TreeSet - IP:set of Long(accessTime) - this.remoteSearchTracker = new HashMap(); + this.localSearchTracker = new HashMap>(); // String:TreeSet - IP:set of Long(accessTime) + this.remoteSearchTracker = new HashMap>(); this.localSearches = new ArrayList>(); // contains search result properties as HashMaps this.remoteSearches = new ArrayList>(); diff --git a/source/de/anomic/plasma/plasmaWordIndex.java b/source/de/anomic/plasma/plasmaWordIndex.java index 4e27812d3..94d44ec48 100644 --- a/source/de/anomic/plasma/plasmaWordIndex.java +++ b/source/de/anomic/plasma/plasmaWordIndex.java @@ -46,6 +46,7 @@ import de.anomic.index.indexRWIEntry; import de.anomic.index.indexRWIRowEntry; import de.anomic.index.indexURLEntry; import de.anomic.kelondro.kelondroBase64Order; +import de.anomic.kelondro.kelondroByteOrder; import de.anomic.kelondro.kelondroCloneableIterator; import de.anomic.kelondro.kelondroMergeIterator; import de.anomic.kelondro.kelondroOrder; @@ -65,7 +66,7 @@ public final class plasmaWordIndex implements indexRI { public static final int lowcachedivisor = 320; public static final int maxCollectionPartition = 7; // should be 7 - private final kelondroOrder indexOrder = kelondroBase64Order.enhancedCoder; + private final kelondroByteOrder indexOrder = kelondroBase64Order.enhancedCoder; private final indexRAMRI dhtOutCache, dhtInCache; private final indexCollectionRI collections; // new database structure to replace AssortmentCluster and FileCluster public boolean busyCacheFlush; // shows if a cache flush is currently performed @@ -133,12 +134,8 @@ public final class plasmaWordIndex implements indexRI { long entryBytes = indexRWIRowEntry.urlEntryRow.objectsize; indexRAMRI cache = (in ? dhtInCache : dhtOutCache); synchronized (cache) { - Iterator it = cache.wordContainers(null, false); - indexContainer ic; - while ( it.hasNext() ) { - ic = (indexContainer)it.next(); - cacheBytes += ic.size() * entryBytes; - } + Iterator it = cache.wordContainers(null, false); + while (it.hasNext()) cacheBytes += it.next().size() * entryBytes; } return cacheBytes; @@ -216,7 +213,7 @@ public final class plasmaWordIndex implements indexRI { busyCacheFlush = true; String wordHash; - ArrayList containerList = new ArrayList(); + ArrayList containerList = new ArrayList(); count = Math.min(5000, Math.min(count, ram.size())); boolean collectMax = true; indexContainer c; @@ -293,15 +290,15 @@ public final class plasmaWordIndex implements indexRI { int urlComps = htmlFilterContentScraper.urlComps(url.toString()).length; // iterate over all words of context text - Iterator i = condenser.words().entrySet().iterator(); - Map.Entry wentry; + Iterator> i = condenser.words().entrySet().iterator(); + Map.Entry wentry; String word; indexRWIEntry ientry; plasmaCondenser.wordStatProp wprop; while (i.hasNext()) { - wentry = (Map.Entry) i.next(); - word = (String) wentry.getKey(); - wprop = (plasmaCondenser.wordStatProp) wentry.getValue(); + wentry = i.next(); + word = wentry.getKey(); + wprop = wentry.getValue(); assert (wprop.flags != null); ientry = new indexRWIRowEntry(url.hash(), urlLength, urlComps, (document == null) ? urlLength : document.getTitle().length(), @@ -333,7 +330,7 @@ public final class plasmaWordIndex implements indexRI { return false; } - public indexContainer getContainer(String wordHash, Set urlselection) { + public indexContainer getContainer(String wordHash, Set urlselection) { if ((wordHash == null) || (wordHash.length() != yacySeedDB.commonHashLength)) { // wrong input return null; @@ -461,7 +458,7 @@ public final class plasmaWordIndex implements indexRI { public int removeEntryMultiple(Set wordHashes, String urlHash) { // remove the same url hashes for multiple words // this is mainly used when correcting a index after a search - Iterator i = wordHashes.iterator(); + Iterator i = wordHashes.iterator(); int count = 0; while (i.hasNext()) { if (removeEntry((String) i.next(), urlHash)) count++; @@ -500,7 +497,7 @@ public final class plasmaWordIndex implements indexRI { public void removeEntriesMultiple(Set wordHashes, Set urlHashes) { // remove the same url hashes for multiple words // this is mainly used when correcting a index after a search - Iterator i = wordHashes.iterator(); + Iterator i = wordHashes.iterator(); while (i.hasNext()) { removeEntries((String) i.next(), urlHashes); } @@ -509,7 +506,7 @@ public final class plasmaWordIndex implements indexRI { public int removeWordReferences(Set words, String urlhash) { // sequentially delete all word references // returns number of deletions - Iterator iter = words.iterator(); + Iterator iter = words.iterator(); int count = 0; while (iter.hasNext()) { // delete the URL reference in this word index @@ -528,13 +525,13 @@ public final class plasmaWordIndex implements indexRI { if (d > 0) return d; else return dhtOutCache.tryRemoveURLs(urlHash); } - public synchronized TreeSet indexContainerSet(String startHash, boolean ram, boolean rot, int count) { + public synchronized TreeSet indexContainerSet(String startHash, boolean ram, boolean rot, int count) { // creates a set of indexContainers // this does not use the dhtInCache - kelondroOrder containerOrder = new indexContainerOrder((kelondroOrder) indexOrder.clone()); - containerOrder.rotate(startHash.getBytes()); - TreeSet containers = new TreeSet(containerOrder); - Iterator i = wordContainers(startHash, ram, rot); + kelondroOrder containerOrder = new indexContainerOrder(indexOrder.clone()); + containerOrder.rotate(emptyContainer(startHash, 0)); + TreeSet containers = new TreeSet(containerOrder); + Iterator i = wordContainers(startHash, ram, rot); if (ram) count = Math.min(dhtOutCache.size(), count); indexContainer container; // this loop does not terminate using the i.hasNex() predicate when rot == true @@ -542,7 +539,7 @@ public final class plasmaWordIndex implements indexRI { // in this case a termination must be ensured with a counter // It must also be ensured that the counter is in/decreased every loop while ((count > 0) && (i.hasNext())) { - container = (indexContainer) i.next(); + container = i.next(); if ((container != null) && (container.size() > 0)) { containers.add(container); } @@ -551,22 +548,22 @@ public final class plasmaWordIndex implements indexRI { return containers; // this may return less containers as demanded } - public synchronized kelondroCloneableIterator wordContainers(String startHash, boolean ram, boolean rot) { - kelondroCloneableIterator i = wordContainers(startHash, ram); + public synchronized kelondroCloneableIterator wordContainers(String startHash, boolean ram, boolean rot) { + kelondroCloneableIterator i = wordContainers(startHash, ram); if (rot) { - return new kelondroRotateIterator(i, new String(kelondroBase64Order.zero(startHash.length()))); + return new kelondroRotateIterator(i, new String(kelondroBase64Order.zero(startHash.length()))); } else { return i; } } - public synchronized kelondroCloneableIterator wordContainers(String startWordHash, boolean ram) { - kelondroOrder containerOrder = new indexContainerOrder((kelondroOrder) indexOrder.clone()); - containerOrder.rotate(startWordHash.getBytes()); + public synchronized kelondroCloneableIterator wordContainers(String startWordHash, boolean ram) { + kelondroOrder containerOrder = new indexContainerOrder(indexOrder.clone()); + containerOrder.rotate(emptyContainer(startWordHash, 0)); if (ram) { return dhtOutCache.wordContainers(startWordHash, false); } else { - return new kelondroMergeIterator( + return new kelondroMergeIterator( dhtOutCache.wordContainers(startWordHash, false), collections.wordContainers(startWordHash, false), containerOrder, @@ -602,12 +599,12 @@ public final class plasmaWordIndex implements indexRI { indexContainer container = null; indexRWIEntry entry = null; yacyURL url = null; - HashSet urlHashs = new HashSet(); - Iterator indexContainerIterator = indexContainerSet(startHash, false, false, 100).iterator(); + HashSet urlHashs = new HashSet(); + Iterator indexContainerIterator = indexContainerSet(startHash, false, false, 100).iterator(); while (indexContainerIterator.hasNext() && run) { waiter(); container = (indexContainer) indexContainerIterator.next(); - Iterator containerIterator = container.entries(); + Iterator containerIterator = container.entries(); wordHashNow = container.getWordHash(); while (containerIterator.hasNext() && run) { waiter(); @@ -633,7 +630,7 @@ public final class plasmaWordIndex implements indexRI { } if (!containerIterator.hasNext()) { // We may not be finished yet, try to get the next chunk of wordHashes - TreeSet containers = indexContainerSet(container.getWordHash(), false, false, 100); + TreeSet containers = indexContainerSet(container.getWordHash(), false, false, 100); indexContainerIterator = containers.iterator(); // Make sure we don't get the same wordhash twice, but don't skip a word if ((indexContainerIterator.hasNext()) && (!container.getWordHash().equals(((indexContainer) indexContainerIterator.next()).getWordHash()))) { diff --git a/source/de/anomic/yacy/yacyDHTAction.java b/source/de/anomic/yacy/yacyDHTAction.java index 6703b1984..cfc3acf6d 100644 --- a/source/de/anomic/yacy/yacyDHTAction.java +++ b/source/de/anomic/yacy/yacyDHTAction.java @@ -66,7 +66,7 @@ public class yacyDHTAction implements yacyPeerAction { this.seedCrawlReady = new kelondroMScoreCluster(); // init crawl-ready table try { - Iterator en = seedDB.seedsConnected(true, false, null, (float) 0.0); + Iterator en = seedDB.seedsConnected(true, false, null, (float) 0.0); yacySeed ys; while (en.hasNext()) { ys = (yacySeed) en.next(); @@ -76,14 +76,14 @@ public class yacyDHTAction implements yacyPeerAction { } } - public Iterator getDHTSeeds(boolean up, String firstHash, float minVersion) { + public Iterator getDHTSeeds(boolean up, String firstHash, float minVersion) { // enumerates seed-type objects: all seeds with starting point in the middle, rotating at the end/beginning return new seedDHTEnum(up, firstHash, minVersion); } - class seedDHTEnum implements Iterator { + class seedDHTEnum implements Iterator { - Iterator e1, e2; + Iterator e1, e2; boolean up; int steps; float minVersion; @@ -100,11 +100,11 @@ public class yacyDHTAction implements yacyPeerAction { return (steps > 0) && ((e2 == null) || (e2.hasNext())); } - public Object next() { + public yacySeed next() { if (steps == 0) return null; steps--; if ((e1 != null) && (e1.hasNext())) { - Object n = e1.next(); + yacySeed n = e1.next(); if (!(e1.hasNext())) { e1 = null; e2 = seedDB.seedsConnected(up, false, null, minVersion); @@ -130,7 +130,7 @@ public class yacyDHTAction implements yacyPeerAction { class providesRemoteCrawlURLsEnum implements Iterator { - Iterator se; + Iterator se; yacySeed nextSeed; public providesRemoteCrawlURLsEnum() { @@ -171,16 +171,16 @@ public class yacyDHTAction implements yacyPeerAction { } - public Iterator getAcceptRemoteIndexSeeds(String starthash) { + public Iterator getAcceptRemoteIndexSeeds(String starthash) { // returns an enumeration of yacySeed-Objects // that have the AcceptRemoteIndex-Flag set // the seeds are enumerated in the right order according DHT return new acceptRemoteIndexSeedEnum(starthash); } - class acceptRemoteIndexSeedEnum implements Iterator { + class acceptRemoteIndexSeedEnum implements Iterator { - Iterator se; + Iterator se; yacySeed nextSeed; public acceptRemoteIndexSeedEnum(String starthash) { @@ -209,7 +209,7 @@ public class yacyDHTAction implements yacyPeerAction { return null; } - public Object next() { + public yacySeed next() { yacySeed next = nextSeed; nextSeed = nextInternal(); return next; @@ -221,13 +221,13 @@ public class yacyDHTAction implements yacyPeerAction { } - public Iterator getAcceptRemoteCrawlSeeds(String starthash, boolean available) { + public Iterator getAcceptRemoteCrawlSeeds(String starthash, boolean available) { return new acceptRemoteCrawlSeedEnum(starthash, available); } - class acceptRemoteCrawlSeedEnum implements Iterator { + class acceptRemoteCrawlSeedEnum implements Iterator { - Iterator se; + Iterator se; yacySeed nextSeed; boolean available; @@ -256,7 +256,7 @@ public class yacyDHTAction implements yacyPeerAction { return null; } - public Object next() { + public yacySeed next() { yacySeed next = nextSeed; nextSeed = nextInternal(); return next; @@ -269,20 +269,20 @@ public class yacyDHTAction implements yacyPeerAction { } public synchronized yacySeed getGlobalCrawlSeed(String urlHash) { - Iterator e = getAcceptRemoteCrawlSeeds(urlHash, true); + Iterator e = getAcceptRemoteCrawlSeeds(urlHash, true); yacySeed seed; - if (e.hasNext()) seed = (yacySeed) e.next(); else seed = null; + if (e.hasNext()) seed = e.next(); else seed = null; e = null; return seed; } - public synchronized yacySeed getPublicClusterCrawlSeed(String urlHash, TreeMap clusterhashes) { + public synchronized yacySeed getPublicClusterCrawlSeed(String urlHash, TreeMap clusterhashes) { // clusterhashes is a String(hash)/String(IP) - mapping - kelondroCloneableIterator i = new kelondroRotateIterator(new kelondroCloneableMapIterator(clusterhashes, urlHash), null); + kelondroCloneableIterator i = new kelondroRotateIterator(new kelondroCloneableMapIterator(clusterhashes, urlHash), null); String hash; int count = clusterhashes.size(); // counter to ensure termination while ((i.hasNext()) && (count-- > 0)) { - hash = (String) i.next(); + hash = i.next(); yacySeed seed = seedDB.getConnected(hash); if (seed == null) continue; seed.setAlternativeAddress((String) clusterhashes.get(hash)); @@ -347,7 +347,7 @@ public class yacyDHTAction implements yacyPeerAction { return ((double) (kelondroBase64Order.enhancedCoder.cardinal(from.getBytes()) - kelondroBase64Order.enhancedCoder.cardinal(to.getBytes()))) / ((double) Long.MAX_VALUE); } - public synchronized ArrayList /* of yacySeed */ getDHTTargets(serverLog log, int primaryPeerCount, int reservePeerCount, String firstKey, String lastKey, double maxDist) { + public synchronized ArrayList getDHTTargets(serverLog log, int primaryPeerCount, int reservePeerCount, String firstKey, String lastKey, double maxDist) { // find a list of DHT-peers assert firstKey != null; assert lastKey != null; @@ -359,14 +359,14 @@ public class yacyDHTAction implements yacyPeerAction { !(kelondroBase64Order.enhancedCoder.cardinal(firstKey.getBytes()) < kelondroBase64Order.enhancedCoder.cardinal(yacyCore.seedDB.mySeed.hash.getBytes()) && kelondroBase64Order.enhancedCoder.cardinal(lastKey.getBytes()) > kelondroBase64Order.enhancedCoder.cardinal(yacyCore.seedDB.mySeed.hash.getBytes())); */ - ArrayList seeds = new ArrayList(); + ArrayList seeds = new ArrayList(); yacySeed seed; //double ownDistance = Math.min(yacyDHTAction.dhtDistance(yacyCore.seedDB.mySeed.hash, firstKey), yacyDHTAction.dhtDistance(yacyCore.seedDB.mySeed.hash, lastKey)); //double maxDistance = Math.min(ownDistance, maxDist); double firstdist, lastdist; - Iterator e = this.getAcceptRemoteIndexSeeds(lastKey); - TreeSet doublecheck = new TreeSet(kelondroBase64Order.enhancedCoder); + Iterator e = this.getAcceptRemoteIndexSeeds(lastKey); + TreeSet doublecheck = new TreeSet(kelondroBase64Order.enhancedComparator); int maxloop = Math.min(100, yacyCore.seedDB.sizeConnected()); // to ensure termination if (log != null) log.logInfo("Collecting DHT target peers for first_hash = " + firstKey + ", last_hash = " + lastKey); while ((e.hasNext()) && (seeds.size() < (primaryPeerCount + reservePeerCount)) && (maxloop-- > 0)) { diff --git a/source/de/anomic/yacy/yacySeedDB.java b/source/de/anomic/yacy/yacySeedDB.java index dd297d4a0..6bd3ffc6a 100644 --- a/source/de/anomic/yacy/yacySeedDB.java +++ b/source/de/anomic/yacy/yacySeedDB.java @@ -62,16 +62,16 @@ import java.util.TreeMap; import de.anomic.http.httpHeader; import de.anomic.http.httpc; import de.anomic.http.httpd; +import de.anomic.kelondro.kelondroBase64Order; import de.anomic.kelondro.kelondroDyn; import de.anomic.kelondro.kelondroException; import de.anomic.kelondro.kelondroMScoreCluster; import de.anomic.kelondro.kelondroMapObjects; -import de.anomic.kelondro.kelondroBase64Order; import de.anomic.plasma.plasmaHTCache; import de.anomic.plasma.plasmaSwitchboard; import de.anomic.server.serverCore; -import de.anomic.server.serverDomains; import de.anomic.server.serverDate; +import de.anomic.server.serverDomains; import de.anomic.server.serverFileUtils; import de.anomic.server.serverSwitch; import de.anomic.server.logging.serverLog; @@ -256,17 +256,17 @@ public final class yacySeedDB { } - public Iterator seedsSortedConnected(boolean up, String field) { + public Iterator seedsSortedConnected(boolean up, String field) { // enumerates seed-type objects: all seeds sequentially ordered by field return new seedEnum(up, field, seedActiveDB); } - public Iterator seedsSortedDisconnected(boolean up, String field) { + public Iterator seedsSortedDisconnected(boolean up, String field) { // enumerates seed-type objects: all seeds sequentially ordered by field return new seedEnum(up, field, seedPassiveDB); } - public Iterator seedsSortedPotential(boolean up, String field) { + public Iterator seedsSortedPotential(boolean up, String field) { // enumerates seed-type objects: all seeds sequentially ordered by field return new seedEnum(up, field, seedPotentialDB); } @@ -318,28 +318,28 @@ public final class yacySeedDB { return clustermap; } - public Iterator seedsConnected(boolean up, boolean rot, String firstHash, float minVersion) { + public Iterator seedsConnected(boolean up, boolean rot, String firstHash, float minVersion) { // enumerates seed-type objects: all seeds sequentially without order return new seedEnum(up, rot, (firstHash == null) ? null : firstHash.getBytes(), null, seedActiveDB, minVersion); } - public Iterator seedsDisconnected(boolean up, boolean rot, String firstHash, float minVersion) { + public Iterator seedsDisconnected(boolean up, boolean rot, String firstHash, float minVersion) { // enumerates seed-type objects: all seeds sequentially without order return new seedEnum(up, rot, (firstHash == null) ? null : firstHash.getBytes(), null, seedPassiveDB, minVersion); } - public Iterator seedsPotential(boolean up, boolean rot, String firstHash, float minVersion) { + public Iterator seedsPotential(boolean up, boolean rot, String firstHash, float minVersion) { // enumerates seed-type objects: all seeds sequentially without order return new seedEnum(up, rot, (firstHash == null) ? null : firstHash.getBytes(), null, seedPotentialDB, minVersion); } public yacySeed anySeedVersion(float minVersion) { // return just any seed that has a specific minimum version number - Iterator e = seedsConnected(true, true, yacySeed.randomHash(), minVersion); + Iterator e = seedsConnected(true, true, yacySeed.randomHash(), minVersion); return (yacySeed) e.next(); } - public HashMap seedsByAge(boolean up, int count) { + public HashMap seedsByAge(boolean up, int count) { // returns a peerhash/yacySeed relation // to get most recent peers, set up = true; for oldest peers, set up = false @@ -349,12 +349,12 @@ public final class yacySeedDB { kelondroMScoreCluster seedScore = new kelondroMScoreCluster(); yacySeed ys; long absage; - Iterator s = seedsConnected(true, false, null, (float) 0.0); + Iterator s = seedsConnected(true, false, null, (float) 0.0); int searchcount = 1000; if (searchcount > sizeConnected()) searchcount = sizeConnected(); try { while ((s.hasNext()) && (searchcount-- > 0)) { - ys = (yacySeed) s.next(); + ys = s.next(); if ((ys != null) && (ys.get(yacySeed.LASTSEEN, "").length() > 10)) try { absage = Math.abs(System.currentTimeMillis() + serverDate.dayMillis - ys.getLastSeenUTC()); seedScore.addScore(ys.hash, (int) absage); // the higher absage, the older is the peer @@ -362,8 +362,8 @@ public final class yacySeedDB { } // result is now in the score object; create a result vector - HashMap result = new HashMap(); - Iterator it = seedScore.scores(up); + HashMap result = new HashMap(); + Iterator it = seedScore.scores(up); int c = 0; while ((c < count) && (it.hasNext())) { c++; @@ -907,7 +907,7 @@ public final class yacySeedDB { } } - class seedEnum implements Iterator { + class seedEnum implements Iterator { kelondroMapObjects.mapIterator it; yacySeed nextSeed; @@ -967,7 +967,7 @@ public final class yacySeedDB { return new yacySeed(hash, dna); } - public Object next() { + public yacySeed next() { yacySeed seed = nextSeed; try {while (true) { nextSeed = internalNext();