From 9c05e2a820cc40b16c4415bb9fa3c0a2179f9d95 Mon Sep 17 00:00:00 2001 From: orbiter Date: Mon, 29 Jan 2007 23:51:10 +0000 Subject: [PATCH] re-design ob kelondroMap - this class is replaced by an object that can hold any type of object - this object must be defined as a class that implements kelondroObjectsEntry - the kelodroMap is now implemented as kelondroMapObjects git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@3297 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- source/de/anomic/data/blogBoard.java | 22 +- source/de/anomic/data/bookmarksDB.java | 35 +- source/de/anomic/data/messageBoard.java | 44 +- source/de/anomic/data/userDB.java | 18 +- source/de/anomic/data/wikiBoard.java | 18 +- .../kelondro/kelondroCachedObjectMap.java | 6 +- ...londroMap.java => kelondroMapObjects.java} | 689 ++++++++---------- .../de/anomic/kelondro/kelondroMapTable.java | 38 +- .../de/anomic/kelondro/kelondroObjects.java | 215 ++++++ .../anomic/kelondro/kelondroObjectsEntry.java | 35 + .../kelondro/kelondroObjectsMapEntry.java | 70 ++ .../de/anomic/plasma/plasmaCrawlProfile.java | 18 +- .../anomic/plasma/plasmaCrawlRobotsTxt.java | 16 +- source/de/anomic/plasma/plasmaHTCache.java | 14 +- source/de/anomic/yacy/yacyDHTAction.java | 1 + source/de/anomic/yacy/yacySeedDB.java | 64 +- source/yacy.java | 6 +- 17 files changed, 739 insertions(+), 570 deletions(-) rename source/de/anomic/kelondro/{kelondroMap.java => kelondroMapObjects.java} (59%) create mode 100644 source/de/anomic/kelondro/kelondroObjects.java create mode 100644 source/de/anomic/kelondro/kelondroObjectsEntry.java create mode 100644 source/de/anomic/kelondro/kelondroObjectsMapEntry.java diff --git a/source/de/anomic/data/blogBoard.java b/source/de/anomic/data/blogBoard.java index 793ded137..392dd11ed 100644 --- a/source/de/anomic/data/blogBoard.java +++ b/source/de/anomic/data/blogBoard.java @@ -67,7 +67,7 @@ import org.xml.sax.SAXException; import de.anomic.kelondro.kelondroBase64Order; import de.anomic.kelondro.kelondroDyn; -import de.anomic.kelondro.kelondroMap; +import de.anomic.kelondro.kelondroMapObjects; public class blogBoard { @@ -78,12 +78,12 @@ public class blogBoard { private static TimeZone GMTTimeZone = TimeZone.getTimeZone("PST"); private static SimpleDateFormat SimpleFormatter = new SimpleDateFormat(dateFormat); - private kelondroMap datbase = null; + private kelondroMapObjects datbase = null; public blogBoard(File actpath, int bufferkb, long preloadTime) { new File(actpath.getParent()).mkdir(); if (datbase == null) { - datbase = new kelondroMap(kelondroDyn.open(actpath, bufferkb / 2 * 0x40, preloadTime, keyLength, recordSize, '_', true, false)); + datbase = new kelondroMapObjects(kelondroDyn.open(actpath, bufferkb / 2 * 0x40, preloadTime, keyLength, recordSize, '_', true, false), 500); } } @@ -238,16 +238,12 @@ public class blogBoard { return read(key, datbase); } - private entry read(String key, kelondroMap base) { - try { - key = normalize(key); - if (key.length() > keyLength) key = key.substring(0, keyLength); - Map record = base.get(key); - if (record == null) return newEntry(key, "".getBytes(), "anonymous".getBytes(), "127.0.0.1", new GregorianCalendar(GMTTimeZone).getTime(), "".getBytes()); - return new entry(key, record); - } catch (IOException e) { - return null; - } + private entry read(String key, kelondroMapObjects base) { + key = normalize(key); + if (key.length() > keyLength) key = key.substring(0, keyLength); + Map record = base.getMap(key); + if (record == null) return newEntry(key, "".getBytes(), "anonymous".getBytes(), "127.0.0.1", new GregorianCalendar(GMTTimeZone).getTime(), "".getBytes()); + return new entry(key, record); } public boolean importXML(String input) { diff --git a/source/de/anomic/data/bookmarksDB.java b/source/de/anomic/data/bookmarksDB.java index 006bdb92b..640f00a35 100644 --- a/source/de/anomic/data/bookmarksDB.java +++ b/source/de/anomic/data/bookmarksDB.java @@ -78,16 +78,16 @@ import de.anomic.kelondro.kelondroCachedObject; import de.anomic.kelondro.kelondroCachedObjectMap; import de.anomic.kelondro.kelondroDyn; import de.anomic.kelondro.kelondroException; -import de.anomic.kelondro.kelondroMap; +import de.anomic.kelondro.kelondroMapObjects; import de.anomic.net.URL; import de.anomic.server.serverFileUtils; import de.anomic.server.logging.serverLog; public class bookmarksDB { - kelondroMap tagsTable; + kelondroMapObjects tagsTable; //kelondroMap bookmarksTable; kelondroCachedObjectMap bookmarksTable; - kelondroMap datesTable; + kelondroMapObjects datesTable; HashMap tagCache; HashMap bookmarkCache; @@ -135,17 +135,17 @@ public class bookmarksDB { bookmarkCache=new HashMap(); bookmarksFile.getParentFile().mkdirs(); //this.bookmarksTable = new kelondroMap(kelondroDyn.open(bookmarksFile, bufferkb * 1024, preloadTime, 12, 256, '_', true, false)); - this.bookmarksTable = new kelondroCachedObjectMap(new kelondroMap(kelondroDyn.open(bookmarksFile, bufferkb * 1024, preloadTime, 12, 256, '_', true, false))); + this.bookmarksTable = new kelondroCachedObjectMap(new kelondroMapObjects(kelondroDyn.open(bookmarksFile, bufferkb * 1024, preloadTime, 12, 256, '_', true, false), 500)); // tags tagsFile.getParentFile().mkdirs(); boolean tagsFileExisted = tagsFile.exists(); - this.tagsTable = new kelondroMap(kelondroDyn.open(tagsFile, bufferkb * 1024, preloadTime, 12, 256, '_', true, false)); + this.tagsTable = new kelondroMapObjects(kelondroDyn.open(tagsFile, bufferkb * 1024, preloadTime, 12, 256, '_', true, false), 500); if (!tagsFileExisted) rebuildTags(); // dates boolean datesExisted = datesFile.exists(); - this.datesTable = new kelondroMap(kelondroDyn.open(datesFile, bufferkb * 1024, preloadTime, 20, 256, '_', true, false)); + this.datesTable = new kelondroMapObjects(kelondroDyn.open(datesFile, bufferkb * 1024, preloadTime, 20, 256, '_', true, false), 500); if (!datesExisted) rebuildDates(); } @@ -195,13 +195,11 @@ public class bookmarksDB { public Tag loadTag(String hash){ Map map; Tag ret=null; - try { - map = tagsTable.get(hash); - if(map!=null){ - ret=new Tag(hash, map); - tagCache.put(hash, ret); - } - } catch (IOException e) {} + map = tagsTable.getMap(hash); + if(map!=null){ + ret=new Tag(hash, map); + tagCache.put(hash, ret); + } return ret; } @@ -284,14 +282,9 @@ public class bookmarksDB { } public bookmarksDate getDate(String date){ Map map; - try { - map=datesTable.get(date); - if(map==null) return new bookmarksDate(date); - return new bookmarksDate(date, map); - } catch (IOException e) { - return null; - } - + map=datesTable.getMap(date); + if(map==null) return new bookmarksDate(date); + return new bookmarksDate(date, map); } public boolean renameTag(String oldName, String newName){ String tagHash=tagHash(oldName); diff --git a/source/de/anomic/data/messageBoard.java b/source/de/anomic/data/messageBoard.java index a60b3ec7c..0c06b704e 100644 --- a/source/de/anomic/data/messageBoard.java +++ b/source/de/anomic/data/messageBoard.java @@ -53,7 +53,7 @@ import java.util.TimeZone; import de.anomic.kelondro.kelondroBase64Order; import de.anomic.kelondro.kelondroDyn; -import de.anomic.kelondro.kelondroMap; +import de.anomic.kelondro.kelondroMapObjects; public class messageBoard { @@ -64,13 +64,13 @@ public class messageBoard { private static TimeZone GMTTimeZone = TimeZone.getTimeZone("PST"); private static SimpleDateFormat SimpleFormatter = new SimpleDateFormat(dateFormat); - private kelondroMap database = null; + private kelondroMapObjects database = null; private int sn = 0; public messageBoard(File path, int bufferkb, long preloadTime) { new File(path.getParent()).mkdir(); if (database == null) { - database = new kelondroMap(kelondroDyn.open(path, bufferkb * 0x400, preloadTime, categoryLength + dateFormat.length() + 2, recordSize, '_', true, false)); + database = new kelondroMapObjects(kelondroDyn.open(path, bufferkb * 0x400, preloadTime, categoryLength + dateFormat.length() + 2, recordSize, '_', true, false), 500); } sn = 0; } @@ -216,39 +216,25 @@ public class messageBoard { } public String write(entry message) { - // writes a message and returns key - try { - database.set(message.key, message.record); - return message.key; - } catch (IOException e) { - return null; - } + // writes a message and returns key + try { + database.set(message.key, message.record); + return message.key; + } catch (IOException e) { + return null; + } } public entry read(String key) { - try { - Map record = database.get(key); + Map record = database.getMap(key); return new entry(key, record); - } catch (IOException e) { - return null; - } - } - - /* - public boolean has(String key) { - try { - return database.has(key); - } catch (IOException e) { - return false; - } } - */ public void remove(String key) { - try { - database.remove(key); - } catch (IOException e) { - } + try { + database.remove(key); + } catch (IOException e) { + } } public Iterator keys(String category, boolean up) throws IOException { diff --git a/source/de/anomic/data/userDB.java b/source/de/anomic/data/userDB.java index 2b9ce249a..495bfec65 100644 --- a/source/de/anomic/data/userDB.java +++ b/source/de/anomic/data/userDB.java @@ -58,7 +58,7 @@ import de.anomic.http.httpHeader; import de.anomic.kelondro.kelondroBase64Order; import de.anomic.kelondro.kelondroDyn; import de.anomic.kelondro.kelondroException; -import de.anomic.kelondro.kelondroMap; +import de.anomic.kelondro.kelondroMapObjects; import de.anomic.server.serverCodings; public final class userDB { @@ -66,7 +66,7 @@ public final class userDB { public static final int USERNAME_MAX_LENGTH = 128; public static final int USERNAME_MIN_LENGTH = 4; - kelondroMap userTable; + kelondroMapObjects userTable; private final File userTableFile; private final int bufferkb; private long preloadTime; @@ -78,7 +78,7 @@ public final class userDB { this.bufferkb = bufferkb; this.preloadTime = preloadTime; userTableFile.getParentFile().mkdirs(); - this.userTable = new kelondroMap(kelondroDyn.open(userTableFile, bufferkb * 1024, preloadTime, 128, 256, '_', true, false)); + this.userTable = new kelondroMapObjects(kelondroDyn.open(userTableFile, bufferkb * 1024, preloadTime, 128, 256, '_', true, false), 10); } public int dbCacheNodeChunkSize() { @@ -96,7 +96,7 @@ public final class userDB { } catch (IOException e) {} if (!(userTableFile.delete())) throw new RuntimeException("cannot delete user database"); userTableFile.getParentFile().mkdirs(); - userTable = new kelondroMap(kelondroDyn.open(userTableFile, this.bufferkb, preloadTime, 256, 512, '_', true, false)); + userTable = new kelondroMapObjects(kelondroDyn.open(userTableFile, this.bufferkb, preloadTime, 256, 512, '_', true, false), 10); } public void close() { @@ -119,13 +119,9 @@ public final class userDB { if(userName.length()>128){ userName=userName.substring(0, 127); } - try { - Map record = userTable.get(userName); - if (record == null) return null; - return new Entry(userName, record); - } catch (IOException e) { - return null; - } + Map record = userTable.getMap(userName); + if (record == null) return null; + return new Entry(userName, record); } public Entry createEntry(String userName, HashMap userProps) throws IllegalArgumentException{ diff --git a/source/de/anomic/data/wikiBoard.java b/source/de/anomic/data/wikiBoard.java index d5ff52d03..29b17c0f0 100644 --- a/source/de/anomic/data/wikiBoard.java +++ b/source/de/anomic/data/wikiBoard.java @@ -53,7 +53,7 @@ import java.util.TimeZone; import de.anomic.kelondro.kelondroBase64Order; import de.anomic.kelondro.kelondroDyn; -import de.anomic.kelondro.kelondroMap; +import de.anomic.kelondro.kelondroMapObjects; import de.anomic.kelondro.kelondroRecords; public class wikiBoard { @@ -65,18 +65,18 @@ public class wikiBoard { private static TimeZone GMTTimeZone = TimeZone.getTimeZone("PST"); private static SimpleDateFormat SimpleFormatter = new SimpleDateFormat(dateFormat); - private kelondroMap datbase = null; - private kelondroMap bkpbase = null; + private kelondroMapObjects datbase = null; + private kelondroMapObjects bkpbase = null; private static HashMap authors = new HashMap(); public wikiBoard(File actpath, File bkppath, int bufferkb, long preloadTime) { new File(actpath.getParent()).mkdirs(); if (datbase == null) { - datbase = new kelondroMap(kelondroDyn.open(actpath, bufferkb / 2 * 0x400, preloadTime, keyLength, recordSize, '_', true, false)); + datbase = new kelondroMapObjects(kelondroDyn.open(actpath, bufferkb / 2 * 0x400, preloadTime, keyLength, recordSize, '_', true, false), 500); } new File(bkppath.getParent()).mkdirs(); if (bkpbase == null) { - bkpbase = new kelondroMap(kelondroDyn.open(bkppath, bufferkb / 2 * 0x400, preloadTime, keyLength + dateFormat.length(), recordSize, '_', true, false)); + bkpbase = new kelondroMapObjects(kelondroDyn.open(bkppath, bufferkb / 2 * 0x400, preloadTime, keyLength + dateFormat.length(), recordSize, '_', true, false), 500); } } @@ -305,11 +305,11 @@ public class wikiBoard { return read(key, datbase); } - private entry read(String key, kelondroMap base) { + private entry read(String key, kelondroMapObjects base) { try { - key = normalize(key); - if (key.length() > keyLength) key = key.substring(0, keyLength); - Map record = base.get(key); + key = normalize(key); + if (key.length() > keyLength) key = key.substring(0, keyLength); + Map record = base.getMap(key); if (record == null) return newEntry(key, "anonymous", "127.0.0.1", "New Page", "".getBytes()); return new entry(key, record); } catch (IOException e) { diff --git a/source/de/anomic/kelondro/kelondroCachedObjectMap.java b/source/de/anomic/kelondro/kelondroCachedObjectMap.java index a63101013..6ec64c39d 100644 --- a/source/de/anomic/kelondro/kelondroCachedObjectMap.java +++ b/source/de/anomic/kelondro/kelondroCachedObjectMap.java @@ -26,13 +26,11 @@ package de.anomic.kelondro; import java.io.IOException; import java.util.HashMap; import java.util.Iterator; -import java.util.Map; -import java.util.Set; public class kelondroCachedObjectMap { - private kelondroMap db; + private kelondroMapObjects db; private HashMap cache; - public kelondroCachedObjectMap(kelondroMap db){ + public kelondroCachedObjectMap(kelondroMapObjects db){ this.db=db; cache=new HashMap(); } diff --git a/source/de/anomic/kelondro/kelondroMap.java b/source/de/anomic/kelondro/kelondroMapObjects.java similarity index 59% rename from source/de/anomic/kelondro/kelondroMap.java rename to source/de/anomic/kelondro/kelondroMapObjects.java index 877cff3a0..93cafe2c8 100644 --- a/source/de/anomic/kelondro/kelondroMap.java +++ b/source/de/anomic/kelondro/kelondroMapObjects.java @@ -1,398 +1,291 @@ -// kelondroMap.java -// ----------------------- -// part of The Kelondro Database -// (C) by Michael Peter Christen; mc@anomic.de -// first published on http://www.anomic.de -// Frankfurt, Germany, 2004 -// -// $LastChangedDate$ -// $LastChangedRevision$ -// $LastChangedBy$ -// -// This program is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -// -// Using this software in any meaning (reading, learning, copying, compiling, -// running) means that you agree that the Author(s) is (are) not responsible -// for cost, loss of data or any harm that may be caused directly or indirectly -// by usage of this softare or this documentation. The usage of this software -// is on your own risk. The installation and usage (starting/running) of this -// software may allow other people or application to access your computer and -// any attached devices and is highly dependent on the configuration of the -// software which must be done by the user of the software; the author(s) is -// (are) also not responsible for proper configuration and usage of the -// software, even if provoked by documentation provided together with -// the software. -// -// Any changes to this file according to the GPL as documented in the file -// gpl.txt aside this file in the shipment you received can be done to the -// lines that follows this copyright notice here, but changes must not be -// done inside the copyright notive above. A re-distribution must contain -// the intact and unchanged copyright notice. -// Contributions and changes to the program code must be marked as such. - -package de.anomic.kelondro; - -import java.io.IOException; -import java.util.HashMap; -import java.util.Iterator; -import java.util.Map; - -public class kelondroMap { - - private static final int cachesize = 500; - - private kelondroDyn dyn; - private kelondroMScoreCluster cacheScore; - private HashMap cache; - private long startup; - private String[] sortfields, accfields; - private HashMap sortClusterMap; // a String-kelondroMScoreCluster - relation - private HashMap accMap; // to store accumulations of specific fields - private int elementCount; - - public kelondroMap(kelondroDyn dyn) { - this(dyn, null, null); - } - - public kelondroMap(kelondroDyn dyn, String[] sortfields, String[] accfields) { - this.dyn = dyn; - this.cache = new HashMap(); - this.cacheScore = new kelondroMScoreCluster(); - this.startup = System.currentTimeMillis(); - this.elementCount = 0; - - // create fast ordering clusters and acc fields - this.sortfields = sortfields; - this.accfields = accfields; - - kelondroMScoreCluster[] cluster = null; - if (sortfields == null) sortClusterMap = null; else { - sortClusterMap = new HashMap(); - cluster = new kelondroMScoreCluster[sortfields.length]; - for (int i = 0; i < sortfields.length; i++) { - cluster[i] = new kelondroMScoreCluster(); - } - } - - Long[] accumulator = null; - if (accfields == null) accMap = null; else { - accMap = new HashMap(); - accumulator = new Long[accfields.length]; - for (int i = 0; i < accfields.length; i++) { - accumulator[i] = new Long(0); - } - } - - // fill cluster and accumulator with values - if ((sortfields != null) || (accfields != null)) try { - kelondroDyn.dynKeyIterator it = dyn.dynKeys(true, false); - String key, value; - long valuel; - Map map; - while (it.hasNext()) { - key = (String) it.next(); - map = get(key); - if (map == null) break; - - if (sortfields != null) for (int i = 0; i < sortfields.length; i++) { - value = (String) map.get(sortfields[i]); - if (value != null) cluster[i].setScore(key, kelondroMScoreCluster.string2score(value)); - } - - if (accfields != null) for (int i = 0; i < accfields.length; i++) { - value = (String) map.get(accfields[i]); - if (value != null) try { - valuel = Long.parseLong(value); - accumulator[i] = new Long(accumulator[i].longValue() + valuel); - } catch (NumberFormatException e) {} - } - elementCount++; - } - } catch (IOException e) {} - - // fill cluster - if (sortfields != null) for (int i = 0; i < sortfields.length; i++) sortClusterMap.put(sortfields[i], cluster[i]); - - // fill acc map - if (accfields != null) for (int i = 0; i < accfields.length; i++) accMap.put(accfields[i], accumulator[i]); - } - - public int keySize() { - return dyn.row().width(0); - } - - public int cacheNodeChunkSize() { - return dyn.cacheNodeChunkSize(); - } - - public int cacheObjectChunkSize() { - return dyn.cacheObjectChunkSize(); - } - - public int[] cacheNodeStatus() { - return dyn.cacheNodeStatus(); - } - - public long[] cacheObjectStatus() { - return dyn.cacheObjectStatus(); - } - - public synchronized void set(String key, Map newMap) throws IOException { - assert (key != null); - assert (key.length() > 0); - assert (newMap != null); - // update elementCount - if ((sortfields != null) || (accfields != null)) { - final Map oldMap = get(key, false); - if (oldMap == null) { - // new element - elementCount++; - } else { - // element exists, update acc - if (accfields != null) updateAcc(oldMap, false); - } - } - // write entry - writeKra(key, newMap, ""); - - // check for space in cache - checkCacheSpace(); - - // write map to cache - cacheScore.setScore(key, (int) ((System.currentTimeMillis() - startup) / 1000)); - cache.put(key, newMap); - - // update sortCluster - if (sortClusterMap != null) updateSortCluster(key, newMap); - - // update accumulators with new values (add) - if (accfields != null) updateAcc(newMap, true); - } - - private synchronized void writeKra(final String key, final Map newMap, String comment) throws IOException { - // write map to kra - final kelondroRA kra = dyn.getRA(key); - kra.writeMap(newMap, comment); - kra.close(); - } - - private void updateAcc(Map map, boolean add) { - String value; - long valuel; - Long accumulator; - for (int i = 0; i < accfields.length; i++) { - value = (String) map.get(accfields[i]); - if (value != null) { - try { - valuel = Long.parseLong(value); - accumulator = (Long) accMap.get(accfields[i]); - if (add) { - accMap.put(accfields[i], new Long(accumulator.longValue() + valuel)); - } else { - accMap.put(accfields[i], new Long(accumulator.longValue() - valuel)); - } - } catch (NumberFormatException e) {} - } - } - } - - private void updateSortCluster(final String key, final Map map) { - String value; - kelondroMScoreCluster cluster; - for (int i = 0; i < sortfields.length; i++) { - value = (String) map.get(sortfields[i]); - if (value != null) { - cluster = (kelondroMScoreCluster) sortClusterMap.get(sortfields[i]); - cluster.setScore(key, kelondroMScoreCluster.string2score(value)); - sortClusterMap.put(sortfields[i], cluster); - } - } - } - - public synchronized void remove(String key) throws IOException { - // update elementCount - if (key == null) return; - if ((sortfields != null) || (accfields != null)) { - final Map map = get(key); - if (map != null) { - // update count - elementCount--; - - // update accumulators (subtract) - if (accfields != null) updateAcc(map, false); - - // remove from sortCluster - if (sortfields != null) deleteSortCluster(key); - } - } - // remove from cache - cacheScore.deleteScore(key); - cache.remove(key); - - // remove from file - dyn.remove(key); - } - - private void deleteSortCluster(final String key) { - if (key == null) return; - kelondroMScoreCluster cluster; - for (int i = 0; i < sortfields.length; i++) { - cluster = (kelondroMScoreCluster) sortClusterMap.get(sortfields[i]); - cluster.deleteScore(key); - sortClusterMap.put(sortfields[i], cluster); - } - } - - public synchronized Map get(final String key) throws IOException { - if (key == null) return null; - return get(key, true); - } - - private synchronized Map get(final String key, final boolean storeCache) throws IOException { - // load map from cache - Map map = (Map) cache.get(key); - if (map != null) return map; - - // load map from kra - if (!(dyn.existsDyn(key))) return null; - //final kelondroRA kra = new kelondroBufferedRA(dyn.getRA(key), dyn.cacheObjectChunkSize(), 0); - final kelondroRA kra = dyn.getRA(key); - map = kra.readMap(); - kra.close(); - - if (storeCache) { - // cache it also - checkCacheSpace(); - // write map to cache - cacheScore.setScore(key, (int) ((System.currentTimeMillis() - startup) / 1000)); - cache.put(key, map); - } - - // return value - return map; - } - - private synchronized void checkCacheSpace() { - // check for space in cache - if (cache.size() >= cachesize) { - // delete one entry - final String delkey = (String) cacheScore.getMinObject(); - cacheScore.deleteScore(delkey); - cache.remove(delkey); - } - } - - public synchronized kelondroDyn.dynKeyIterator keys(final boolean up, final boolean rotating) throws IOException { - // simple enumeration of key names without special ordering - return dyn.dynKeys(up, rotating); - } - - public synchronized kelondroDyn.dynKeyIterator keys(final boolean up, final boolean rotating, final byte[] firstKey) throws IOException { - // simple enumeration of key names without special ordering - return dyn.dynKeys(up, rotating, firstKey); - } - - public synchronized Iterator keys(final boolean up, /* sorted by */ String field) { - // sorted iteration using the sortClusters - if (sortClusterMap == null) return null; - final kelondroMScoreCluster cluster = (kelondroMScoreCluster) sortClusterMap.get(field); - if (cluster == null) return null; // sort field does not exist - //System.out.println("DEBUG: cluster for field " + field + ": " + cluster.toString()); - return cluster.scores(up); - } - - public synchronized mapIterator maps(final boolean up, final boolean rotating) throws IOException { - return new mapIterator(keys(up, rotating)); - } - - public synchronized mapIterator maps(final boolean up, final boolean rotating, final byte[] firstKey) throws IOException { - return new mapIterator(keys(up, rotating, firstKey)); - } - - public synchronized mapIterator maps(final boolean up, final String field) { - return new mapIterator(keys(up, field)); - } - - public synchronized long getAcc(final String field) { - final Long accumulator = (Long) accMap.get(field); - if (accumulator == null) return -1; - return accumulator.longValue(); - } - - public synchronized int size() { - if ((sortfields != null) || (accfields != null)) return elementCount; - try { - return dyn.sizeDyn(); - } catch (IOException e) { - return 0; - } - } - - public void close() throws IOException { - // finish queue - //writeWorker.terminate(true); - - // close cluster - if (sortClusterMap != null) { - for (int i = 0; i < sortfields.length; i++) sortClusterMap.remove(sortfields[i]); - sortClusterMap = null; - } - cache = null; - cacheScore = null; - - // close file - dyn.close(); - } - - public class mapIterator implements Iterator { - // enumerates Map-Type elements - // the key is also included in every map that is returned; it's key is 'key' - - Iterator keyIterator; - boolean finish; - - public mapIterator(Iterator keyIterator) { - this.keyIterator = keyIterator; - this.finish = false; - } - - public boolean hasNext() { - return (!(finish)) && (keyIterator.hasNext()); - } - - public Object next() { - final String nextKey = (String) keyIterator.next(); - if (nextKey == null) { - finish = true; - return null; - } - try { - final Map map = get(nextKey); - //assert (map != null) : "nextKey = " + nextKey; - if (map == null) throw new kelondroException("no more elements available"); - map.put("key", nextKey); - return map; - } catch (IOException e) { - finish = true; - return null; - } - } - - public void remove() { - throw new UnsupportedOperationException(); - } - } // class mapIterator - -} // class kelondroMap +// kelondroMapObjects.java +// ----------------------- +// (C) 29.01.2007 by Michael Peter Christen; mc@anomic.de, Frankfurt a. M., Germany +// first published 2004 as part of kelondroMap on http://www.anomic.de +// +// This is a part of YaCy, a peer-to-peer based web search engine +// +// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $ +// $LastChangedRevision: 1986 $ +// $LastChangedBy: orbiter $ +// +// LICENSE +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +package de.anomic.kelondro; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; + +public class kelondroMapObjects extends kelondroObjects { + + private String[] sortfields, accfields; + private HashMap sortClusterMap; // a String-kelondroMScoreCluster - relation + private HashMap accMap; // to store accumulations of specific fields + private int elementCount; + + public kelondroMapObjects(kelondroDyn dyn, int cachesize) { + this(dyn, cachesize, null, null); + } + + public kelondroMapObjects(kelondroDyn dyn, int cachesize, String[] sortfields, String[] accfields) { + super(dyn, cachesize); + + // create fast ordering clusters and acc fields + this.sortfields = sortfields; + this.accfields = accfields; + + kelondroMScoreCluster[] cluster = null; + if (sortfields == null) sortClusterMap = null; else { + sortClusterMap = new HashMap(); + cluster = new kelondroMScoreCluster[sortfields.length]; + for (int i = 0; i < sortfields.length; i++) { + cluster[i] = new kelondroMScoreCluster(); + } + } + + Long[] accumulator = null; + if (accfields == null) accMap = null; else { + accMap = new HashMap(); + accumulator = new Long[accfields.length]; + for (int i = 0; i < accfields.length; i++) { + accumulator[i] = new Long(0); + } + } + + // fill cluster and accumulator with values + if ((sortfields != null) || (accfields != null)) try { + kelondroDyn.dynKeyIterator it = dyn.dynKeys(true, false); + String key, value; + long valuel; + Map map; + while (it.hasNext()) { + key = (String) it.next(); + map = getMap(key); + if (map == null) break; + + if (sortfields != null) for (int i = 0; i < sortfields.length; i++) { + value = (String) map.get(sortfields[i]); + if (value != null) cluster[i].setScore(key, kelondroMScoreCluster.string2score(value)); + } + + if (accfields != null) for (int i = 0; i < accfields.length; i++) { + value = (String) map.get(accfields[i]); + if (value != null) try { + valuel = Long.parseLong(value); + accumulator[i] = new Long(accumulator[i].longValue() + valuel); + } catch (NumberFormatException e) {} + } + elementCount++; + } + } catch (IOException e) {} + + // fill cluster + if (sortfields != null) for (int i = 0; i < sortfields.length; i++) sortClusterMap.put(sortfields[i], cluster[i]); + + // fill acc map + if (accfields != null) for (int i = 0; i < accfields.length; i++) accMap.put(accfields[i], accumulator[i]); + } + + public synchronized void set(String key, Map newMap) throws IOException { + assert (key != null); + assert (key.length() > 0); + assert (newMap != null); + + // update elementCount + if ((sortfields != null) || (accfields != null)) { + final Map oldMap = getMap(key, false); + if (oldMap == null) { + // new element + elementCount++; + } else { + // element exists, update acc + if (accfields != null) updateAcc(oldMap, false); + } + } + + super.set(key, new kelondroObjectsMapEntry(newMap)); + + // update sortCluster + if (sortClusterMap != null) updateSortCluster(key, newMap); + + // update accumulators with new values (add) + if (accfields != null) updateAcc(newMap, true); + } + + private void updateAcc(Map map, boolean add) { + String value; + long valuel; + Long accumulator; + for (int i = 0; i < accfields.length; i++) { + value = (String) map.get(accfields[i]); + if (value != null) { + try { + valuel = Long.parseLong(value); + accumulator = (Long) accMap.get(accfields[i]); + if (add) { + accMap.put(accfields[i], new Long(accumulator.longValue() + valuel)); + } else { + accMap.put(accfields[i], new Long(accumulator.longValue() - valuel)); + } + } catch (NumberFormatException e) {} + } + } + } + + private void updateSortCluster(final String key, final Map map) { + String value; + kelondroMScoreCluster cluster; + for (int i = 0; i < sortfields.length; i++) { + value = (String) map.get(sortfields[i]); + if (value != null) { + cluster = (kelondroMScoreCluster) sortClusterMap.get(sortfields[i]); + cluster.setScore(key, kelondroMScoreCluster.string2score(value)); + sortClusterMap.put(sortfields[i], cluster); + } + } + } + + public synchronized void remove(String key) throws IOException { + if (key == null) return; + + // update elementCount + if ((sortfields != null) || (accfields != null)) { + final Map map = getMap(key); + if (map != null) { + // update count + elementCount--; + + // update accumulators (subtract) + if (accfields != null) updateAcc(map, false); + + // remove from sortCluster + if (sortfields != null) deleteSortCluster(key); + } + } + super.remove(key); + } + + public Map getMap(String key) { + try { + kelondroObjectsMapEntry mapEntry = (kelondroObjectsMapEntry) super.get(key); + if (mapEntry == null) return null; + return mapEntry.map(); + } catch (IOException e) { + e.printStackTrace(); + return null; + } + } + + protected Map getMap(String key, boolean cache) { + try { + kelondroObjectsMapEntry mapEntry = (kelondroObjectsMapEntry) super.get(key, cache); + if (mapEntry == null) return null; + return mapEntry.map(); + } catch (IOException e) { + e.printStackTrace(); + return null; + } + } + + private void deleteSortCluster(final String key) { + if (key == null) return; + kelondroMScoreCluster cluster; + for (int i = 0; i < sortfields.length; i++) { + cluster = (kelondroMScoreCluster) sortClusterMap.get(sortfields[i]); + cluster.deleteScore(key); + sortClusterMap.put(sortfields[i], cluster); + } + } + + public synchronized Iterator keys(final boolean up, /* sorted by */ String field) { + // sorted iteration using the sortClusters + if (sortClusterMap == null) return null; + final kelondroMScoreCluster cluster = (kelondroMScoreCluster) sortClusterMap.get(field); + if (cluster == null) return null; // sort field does not exist + //System.out.println("DEBUG: cluster for field " + field + ": " + cluster.toString()); + return cluster.scores(up); + } + + public synchronized mapIterator maps(final boolean up, final String field) { + return new mapIterator(keys(up, field)); + } + + public synchronized mapIterator maps(final boolean up, final boolean rotating) throws IOException { + return new mapIterator(keys(up, rotating)); + } + + public synchronized mapIterator maps(final boolean up, final boolean rotating, final byte[] firstKey) throws IOException { + return new mapIterator(keys(up, rotating, firstKey)); + } + + public synchronized long getAcc(final String field) { + final Long accumulator = (Long) accMap.get(field); + if (accumulator == null) return -1; + return accumulator.longValue(); + } + + public synchronized int size() { + if ((sortfields != null) || (accfields != null)) return elementCount; + return super.size(); + } + + public void close() throws IOException { + // close cluster + if (sortClusterMap != null) { + for (int i = 0; i < sortfields.length; i++) sortClusterMap.remove(sortfields[i]); + sortClusterMap = null; + } + + super.close(); + } + + public class mapIterator implements Iterator { + // enumerates Map-Type elements + // the key is also included in every map that is returned; it's key is 'key' + + Iterator keyIterator; + boolean finish; + + public mapIterator(Iterator keyIterator) { + this.keyIterator = keyIterator; + this.finish = false; + } + + public boolean hasNext() { + return (!(finish)) && (keyIterator != null) && (keyIterator.hasNext()); + } + + public Object next() { + final String nextKey = (String) keyIterator.next(); + if (nextKey == null) { + finish = true; + return null; + } + final Map map = getMap(nextKey); + //assert (map != null) : "nextKey = " + nextKey; + if (map == null) throw new kelondroException("no more elements available"); + map.put("key", nextKey); + return map; + } + + public void remove() { + throw new UnsupportedOperationException(); + } + } // class mapIterator +} diff --git a/source/de/anomic/kelondro/kelondroMapTable.java b/source/de/anomic/kelondro/kelondroMapTable.java index f1c4cb2b4..e98590f3b 100644 --- a/source/de/anomic/kelondro/kelondroMapTable.java +++ b/source/de/anomic/kelondro/kelondroMapTable.java @@ -62,19 +62,19 @@ public class kelondroMapTable { } public void declareMaps( - String tablename, int keysize, int nodesize, + String tablename, int keysize, int nodesize, int cacheslots, char fillChar) throws IOException { - declareMaps(tablename, keysize, nodesize, null, null, fillChar); + declareMaps(tablename, keysize, nodesize, cacheslots, null, null, fillChar); } public void declareMaps( - String tablename, int keysize, int nodesize, + String tablename, int keysize, int nodesize, int cacheslots, String[] sortfields, String[] accfields, char fillChar) throws IOException { - declareMaps(tablename, keysize, nodesize, sortfields, accfields, fillChar, 0x800, 0); + declareMaps(tablename, keysize, nodesize, cacheslots, sortfields, accfields, fillChar, 0x800, 0); } public void declareMaps( - String tablename, int keysize, int nodesize, + String tablename, int keysize, int nodesize, int cacheslots, String[] sortfields, String[] accfields, char fillChar, long buffersize /*bytes*/, long preloadTime) throws IOException { if (mTables.containsKey(tablename)) throw new RuntimeException("kelondroTables.declareMap: table '" + tablename + "' declared twice."); @@ -83,7 +83,7 @@ public class kelondroMapTable { kelondroDyn dyn; if (!(tablefile.exists())) tablefile.getParentFile().mkdirs(); dyn = new kelondroDyn(tablefile, buffersize, preloadTime, keysize, nodesize, fillChar, true, false); - kelondroMap map = new kelondroMap(dyn, sortfields, accfields); + kelondroMapObjects map = new kelondroMapObjects(dyn, cacheslots, sortfields, accfields); mTables.put(tablename, map); } @@ -96,7 +96,7 @@ public class kelondroMapTable { } public synchronized void update(String tablename, String key, Map map) throws IOException { - kelondroMap table = (kelondroMap) mTables.get(tablename); + kelondroMapObjects table = (kelondroMapObjects) mTables.get(tablename); if (table == null) throw new RuntimeException("kelondroTables.update: map table '" + tablename + "' does not exist."); if (key.length() > table.keySize()) key = key.substring(0, table.keySize()); table.set(key, map); @@ -111,10 +111,10 @@ public class kelondroMapTable { } public synchronized Map selectMap(String tablename, String key) throws IOException { - kelondroMap table = (kelondroMap) mTables.get(tablename); + kelondroMapObjects table = (kelondroMapObjects) mTables.get(tablename); if (table == null) throw new RuntimeException("kelondroTables.selectMap: map table '" + tablename + "' does not exist."); if (key.length() > table.keySize()) key = key.substring(0, table.keySize()); - return table.get(key); + return table.getMap(key); } public synchronized kelondroRow.Entry selectByte(String tablename, String key) throws IOException { @@ -123,20 +123,20 @@ public class kelondroMapTable { return tree.get(key.getBytes()); } - public synchronized kelondroMap.mapIterator /* of Map-Elements */ maps(String tablename, boolean up, boolean rotating) throws IOException { - kelondroMap table = (kelondroMap) mTables.get(tablename); + public synchronized kelondroMapObjects.mapIterator /* of Map-Elements */ maps(String tablename, boolean up, boolean rotating) throws IOException { + kelondroMapObjects table = (kelondroMapObjects) mTables.get(tablename); if (table == null) throw new RuntimeException("kelondroTables.maps: map table '" + tablename + "' does not exist."); return table.maps(up, rotating); } - public synchronized kelondroMap.mapIterator /* of Map-Elements */ maps(String tablename, boolean up, boolean rotating, byte[] firstKey) throws IOException { - kelondroMap table = (kelondroMap) mTables.get(tablename); + public synchronized kelondroMapObjects.mapIterator /* of Map-Elements */ maps(String tablename, boolean up, boolean rotating, byte[] firstKey) throws IOException { + kelondroMapObjects table = (kelondroMapObjects) mTables.get(tablename); if (table == null) throw new RuntimeException("kelondroTables.maps: map table '" + tablename + "' does not exist."); return table.maps(up, rotating, firstKey); } - public synchronized kelondroMap.mapIterator /* of Map-Elements */ maps(String tablename, boolean up, String field) { - kelondroMap table = (kelondroMap) mTables.get(tablename); + public synchronized kelondroMapObjects.mapIterator /* of Map-Elements */ maps(String tablename, boolean up, String field) { + kelondroMapObjects table = (kelondroMapObjects) mTables.get(tablename); if (table == null) throw new RuntimeException("kelondroTables.maps: map table '" + tablename + "' does not exist."); return table.maps(up, field); } @@ -150,7 +150,7 @@ public class kelondroMapTable { // if you need the long-values from a row-iteration, please use kelondroRecords.bytes2long to convert from byte[] to long public synchronized void delete(String tablename, String key) throws IOException { - kelondroMap table = (kelondroMap) mTables.get(tablename); + kelondroMapObjects table = (kelondroMapObjects) mTables.get(tablename); if (key.length() > table.keySize()) key = key.substring(0, table.keySize()); if (table != null) {table.remove(key); mTables.put(tablename, table); return;} @@ -161,13 +161,13 @@ public class kelondroMapTable { } public synchronized long accumulator(String tablename, String field) { - kelondroMap table = (kelondroMap) mTables.get(tablename); + kelondroMapObjects table = (kelondroMapObjects) mTables.get(tablename); if (table == null) throw new RuntimeException("kelondroTables.accumulator: map table '" + tablename + "' does not exist."); return table.getAcc(field); } public synchronized int size(String tablename) { - kelondroMap table = (kelondroMap) mTables.get(tablename); + kelondroMapObjects table = (kelondroMapObjects) mTables.get(tablename); if (table != null) return table.size(); kelondroIndex Tree = (kelondroIndex) tTables.get(tablename); @@ -178,7 +178,7 @@ public class kelondroMapTable { public void close() throws IOException { Iterator tablesIt = mTables.values().iterator(); - while (tablesIt.hasNext()) ((kelondroMap) tablesIt.next()).close(); + while (tablesIt.hasNext()) ((kelondroMapObjects) tablesIt.next()).close(); mTables = null; Iterator TreeIt = tTables.values().iterator(); diff --git a/source/de/anomic/kelondro/kelondroObjects.java b/source/de/anomic/kelondro/kelondroObjects.java new file mode 100644 index 000000000..11f9c7cfa --- /dev/null +++ b/source/de/anomic/kelondro/kelondroObjects.java @@ -0,0 +1,215 @@ +// kelondroObjects.java +// ----------------------- +// (C) 29.01.2007 by Michael Peter Christen; mc@anomic.de, Frankfurt a. M., Germany +// first published 2004 as kelondroMap on http://www.anomic.de +// +// This is a part of YaCy, a peer-to-peer based web search engine +// +// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $ +// $LastChangedRevision: 1986 $ +// $LastChangedBy: orbiter $ +// +// LICENSE +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +package de.anomic.kelondro; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Iterator; + +public class kelondroObjects { + + private kelondroDyn dyn; + private kelondroMScoreCluster cacheScore; + private HashMap cache; + private long startup; + private int cachesize; + + + public kelondroObjects(kelondroDyn dyn, int cachesize) { + this.dyn = dyn; + this.cache = new HashMap(); + this.cacheScore = new kelondroMScoreCluster(); + this.startup = System.currentTimeMillis(); + this.cachesize = cachesize; + } + + public int keySize() { + return dyn.row().width(0); + } + + public int cacheNodeChunkSize() { + return dyn.cacheNodeChunkSize(); + } + + public int cacheObjectChunkSize() { + return dyn.cacheObjectChunkSize(); + } + + public int[] cacheNodeStatus() { + return dyn.cacheNodeStatus(); + } + + public long[] cacheObjectStatus() { + return dyn.cacheObjectStatus(); + } + + public synchronized void set(String key, kelondroObjectsEntry newMap) throws IOException { + assert (key != null); + assert (key.length() > 0); + assert (newMap != null); + + // write entry + kelondroRA kra = dyn.getRA(key); + newMap.write(kra); + kra.close(); + + // check for space in cache + checkCacheSpace(); + + // write map to cache + cacheScore.setScore(key, (int) ((System.currentTimeMillis() - startup) / 1000)); + cache.put(key, newMap); + } + + public synchronized void remove(String key) throws IOException { + // update elementCount + if (key == null) return; + + // remove from cache + cacheScore.deleteScore(key); + cache.remove(key); + + // remove from file + dyn.remove(key); + } + + public synchronized kelondroObjectsEntry get(final String key) throws IOException { + if (key == null) return null; + return get(key, true); + } + + protected synchronized kelondroObjectsEntry get(final String key, final boolean storeCache) throws IOException { + // load map from cache + kelondroObjectsEntry map = (kelondroObjectsEntry) cache.get(key); + if (map != null) return map; + + // load map from kra + if (!(dyn.existsDyn(key))) return null; + + // read object + kelondroRA kra = dyn.getRA(key); + map = new kelondroObjectsMapEntry(kra); + kra.close(); + + if (storeCache) { + // cache it also + checkCacheSpace(); + // write map to cache + cacheScore.setScore(key, (int) ((System.currentTimeMillis() - startup) / 1000)); + cache.put(key, map); + } + + // return value + return map; + } + + private synchronized void checkCacheSpace() { + // check for space in cache + if (cache.size() >= cachesize) { + // delete one entry + final String delkey = (String) cacheScore.getMinObject(); + cacheScore.deleteScore(delkey); + cache.remove(delkey); + } + } + + public synchronized kelondroDyn.dynKeyIterator keys(final boolean up, final boolean rotating) throws IOException { + // simple enumeration of key names without special ordering + return dyn.dynKeys(up, rotating); + } + + public synchronized kelondroDyn.dynKeyIterator keys(final boolean up, final boolean rotating, final byte[] firstKey) throws IOException { + // simple enumeration of key names without special ordering + return dyn.dynKeys(up, rotating, firstKey); + } + + + public synchronized objectIterator entries(final boolean up, final boolean rotating) throws IOException { + return new objectIterator(keys(up, rotating)); + } + + public synchronized objectIterator entries(final boolean up, final boolean rotating, final byte[] firstKey) throws IOException { + return new objectIterator(keys(up, rotating, firstKey)); + } + + public synchronized int size() { + try { + return dyn.sizeDyn(); + } catch (IOException e) { + return 0; + } + } + + public void close() throws IOException { + // finish queue + //writeWorker.terminate(true); + + cache = null; + cacheScore = null; + + // close file + dyn.close(); + } + + public class objectIterator implements Iterator { + // enumerates Map-Type elements + // the key is also included in every map that is returned; it's key is 'key' + + Iterator keyIterator; + boolean finish; + + public objectIterator(Iterator keyIterator) { + this.keyIterator = keyIterator; + this.finish = false; + } + + public boolean hasNext() { + return (!(finish)) && (keyIterator.hasNext()); + } + + public Object next() { + final String nextKey = (String) keyIterator.next(); + if (nextKey == null) { + finish = true; + return null; + } + try { + final kelondroObjectsEntry obj = get(nextKey); + if (obj == null) throw new kelondroException("no more elements available"); + return obj; + } catch (IOException e) { + finish = true; + return null; + } + } + + public void remove() { + throw new UnsupportedOperationException(); + } + } // class mapIterator +} diff --git a/source/de/anomic/kelondro/kelondroObjectsEntry.java b/source/de/anomic/kelondro/kelondroObjectsEntry.java new file mode 100644 index 000000000..30f825f42 --- /dev/null +++ b/source/de/anomic/kelondro/kelondroObjectsEntry.java @@ -0,0 +1,35 @@ +// kelondroObjectsEntry.java +// ------------------------- +// (C) 29.01.2007 by Michael Peter Christen; mc@anomic.de, Frankfurt a. M., Germany +// first published 2004 as part of kelondroMap on http://www.anomic.de +// +// This is a part of YaCy, a peer-to-peer based web search engine +// +// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $ +// $LastChangedRevision: 1986 $ +// $LastChangedBy: orbiter $ +// +// LICENSE +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +package de.anomic.kelondro; + +public interface kelondroObjectsEntry { + + public void write(kelondroRA ra); + public void read(kelondroRA ra); + +} diff --git a/source/de/anomic/kelondro/kelondroObjectsMapEntry.java b/source/de/anomic/kelondro/kelondroObjectsMapEntry.java new file mode 100644 index 000000000..96880eaac --- /dev/null +++ b/source/de/anomic/kelondro/kelondroObjectsMapEntry.java @@ -0,0 +1,70 @@ +// kelondroObjectsMapEntry.java +// ---------------------------- +// (C) 29.01.2007 by Michael Peter Christen; mc@anomic.de, Frankfurt a. M., Germany +// first published 2004 as part of kelondroMap on http://www.anomic.de +// +// This is a part of YaCy, a peer-to-peer based web search engine +// +// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $ +// $LastChangedRevision: 1986 $ +// $LastChangedBy: orbiter $ +// +// LICENSE +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +package de.anomic.kelondro; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +public class kelondroObjectsMapEntry implements kelondroObjectsEntry { + + private Map entry; + + public kelondroObjectsMapEntry() { + this.entry = new HashMap(); + } + + public kelondroObjectsMapEntry(Map map) { + this.entry = map; + } + + public kelondroObjectsMapEntry(kelondroRA ra) { + this.read(ra); + } + + public void read(kelondroRA ra) { + try { + this.entry = ra.readMap(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + public void write(kelondroRA ra) { + try { + ra.writeMap(this.entry, ""); + } catch (IOException e) { + e.printStackTrace(); + } + } + + public Map map() { + return this.entry; + } + +} diff --git a/source/de/anomic/plasma/plasmaCrawlProfile.java b/source/de/anomic/plasma/plasmaCrawlProfile.java index 2ebd762a8..3897b0da8 100644 --- a/source/de/anomic/plasma/plasmaCrawlProfile.java +++ b/source/de/anomic/plasma/plasmaCrawlProfile.java @@ -51,12 +51,12 @@ import java.util.Map; import de.anomic.kelondro.kelondroBase64Order; import de.anomic.kelondro.kelondroDyn; import de.anomic.kelondro.kelondroException; -import de.anomic.kelondro.kelondroMap; +import de.anomic.kelondro.kelondroMapObjects; import de.anomic.server.serverCodings; public class plasmaCrawlProfile { - private kelondroMap profileTable; + private kelondroMapObjects profileTable; private HashMap domsCache; private File profileTableFile; private int bufferkb; @@ -70,7 +70,7 @@ public class plasmaCrawlProfile { this.preloadTime = preloadTime; profileTableFile.getParentFile().mkdirs(); kelondroDyn dyn = kelondroDyn.open(profileTableFile, bufferkb * 1024, preloadTime, crawlProfileHandleLength, 2000, '#', true, false); - profileTable = new kelondroMap(dyn); + profileTable = new kelondroMapObjects(dyn, 500); domsCache = new HashMap(); } @@ -96,7 +96,7 @@ public class plasmaCrawlProfile { if (!(profileTableFile.delete())) throw new RuntimeException("cannot delete crawl profile database"); profileTableFile.getParentFile().mkdirs(); kelondroDyn dyn = kelondroDyn.open(profileTableFile, bufferkb * 1024, preloadTime, crawlProfileHandleLength, 2000, '#', true, false); - profileTable = new kelondroMap(dyn); + profileTable = new kelondroMapObjects(dyn, 500); } public void close() { @@ -222,13 +222,9 @@ public class plasmaCrawlProfile { } public entry getEntry(String handle) { - try { - Map m = profileTable.get(handle); - if (m == null) return null; - return new entry(m); - } catch (IOException e) { - return null; - } + Map m = profileTable.getMap(handle); + if (m == null) return null; + return new entry(m); } public class DomProfile { diff --git a/source/de/anomic/plasma/plasmaCrawlRobotsTxt.java b/source/de/anomic/plasma/plasmaCrawlRobotsTxt.java index b0f14d732..4e95efa85 100644 --- a/source/de/anomic/plasma/plasmaCrawlRobotsTxt.java +++ b/source/de/anomic/plasma/plasmaCrawlRobotsTxt.java @@ -57,13 +57,13 @@ import java.util.Map; import de.anomic.kelondro.kelondroDyn; import de.anomic.kelondro.kelondroException; -import de.anomic.kelondro.kelondroMap; +import de.anomic.kelondro.kelondroMapObjects; public class plasmaCrawlRobotsTxt { public static final String ROBOTS_DB_PATH_SEPARATOR = ";"; - kelondroMap robotsTable; + kelondroMapObjects robotsTable; private final File robotsTableFile; private int bufferkb; private long preloadTime; @@ -73,7 +73,7 @@ public class plasmaCrawlRobotsTxt { this.bufferkb = bufferkb; this.preloadTime = preloadTime; robotsTableFile.getParentFile().mkdirs(); - robotsTable = new kelondroMap(kelondroDyn.open(robotsTableFile, bufferkb * 1024, preloadTime, 256, 512, '_', true, false)); + robotsTable = new kelondroMapObjects(kelondroDyn.open(robotsTableFile, bufferkb * 1024, preloadTime, 256, 512, '_', true, false), 100); } public int cacheNodeChunkSize() { @@ -99,7 +99,7 @@ public class plasmaCrawlRobotsTxt { } catch (IOException e) {} if (!(robotsTableFile.delete())) throw new RuntimeException("cannot delete robots.txt database"); robotsTableFile.getParentFile().mkdirs(); - robotsTable = new kelondroMap(kelondroDyn.open(robotsTableFile, this.bufferkb, preloadTime, 256, 512, '_', true, false)); + robotsTable = new kelondroMapObjects(kelondroDyn.open(robotsTableFile, this.bufferkb, preloadTime, 256, 512, '_', true, false), 100); } public void close() { @@ -124,14 +124,12 @@ public class plasmaCrawlRobotsTxt { public Entry getEntry(String hostName) { try { - Map record = robotsTable.get(hostName); + Map record = robotsTable.getMap(hostName); if (record == null) return null; return new Entry(hostName, record); - } catch (IOException e) { - return null; } catch (kelondroException e) { - resetDatabase(); - return null; + resetDatabase(); + return null; } } diff --git a/source/de/anomic/plasma/plasmaHTCache.java b/source/de/anomic/plasma/plasmaHTCache.java index e940525ca..0094ce989 100644 --- a/source/de/anomic/plasma/plasmaHTCache.java +++ b/source/de/anomic/plasma/plasmaHTCache.java @@ -78,7 +78,7 @@ import de.anomic.plasma.plasmaURL; import de.anomic.kelondro.kelondroBase64Order; import de.anomic.kelondro.kelondroDyn; import de.anomic.kelondro.kelondroMScoreCluster; -import de.anomic.kelondro.kelondroMap; +import de.anomic.kelondro.kelondroMapObjects; import de.anomic.net.URL; import de.anomic.plasma.cache.IResourceInfo; import de.anomic.plasma.cache.ResourceInfoFactory; @@ -97,7 +97,7 @@ public final class plasmaHTCache { private static final int stackLimit = 150; // if we exceed that limit, we do not check idle public static final long oneday = 1000 * 60 * 60 * 24; // milliseconds of a day - kelondroMap responseHeaderDB = null; + kelondroMapObjects responseHeaderDB = null; private final LinkedList cacheStack; private final Map cacheAge; // a - relation public long curCacheSize; @@ -174,7 +174,7 @@ public final class plasmaHTCache { // open the response header database File dbfile = new File(this.cachePath, "responseHeader.db"); try { - this.responseHeaderDB = new kelondroMap(new kelondroDyn(dbfile, bufferkb * 0x400, preloadTime, yacySeedDB.commonHashLength, 150, '#', true, false)); + this.responseHeaderDB = new kelondroMapObjects(new kelondroDyn(dbfile, bufferkb * 0x400, preloadTime, yacySeedDB.commonHashLength, 150, '#', true, false), 500); } catch (IOException e) { this.log.logSevere("the request header database could not be opened: " + e.getMessage()); System.exit(0); @@ -512,7 +512,7 @@ public final class plasmaHTCache { String urlHash = plasmaURL.urlHash(url.toNormalform()); // loading data from database - Map hdb = this.responseHeaderDB.get(urlHash); + Map hdb = this.responseHeaderDB.getMap(urlHash); if (hdb == null) return null; // generate the cached object @@ -752,11 +752,7 @@ public final class plasmaHTCache { if (url != null) return url; // try responseHeaderDB Map hdb; - try { - hdb = this.responseHeaderDB.get(urlHash); - } catch (IOException e) { - hdb = null; - } + hdb = this.responseHeaderDB.getMap(urlHash); if (hdb != null) { Object origRequestLine = hdb.get(httpHeader.X_YACY_ORIGINAL_REQUEST_LINE); if ((origRequestLine != null)&&(origRequestLine instanceof String)) { diff --git a/source/de/anomic/yacy/yacyDHTAction.java b/source/de/anomic/yacy/yacyDHTAction.java index 3dcdfa74c..c57801e57 100644 --- a/source/de/anomic/yacy/yacyDHTAction.java +++ b/source/de/anomic/yacy/yacyDHTAction.java @@ -145,6 +145,7 @@ public class yacyDHTAction implements yacyPeerAction { if (s.getFlagAcceptRemoteIndex()) return s; } } catch (kelondroException e) { + System.out.println("DEBUG acceptRemoteIndexSeedEnum:" + e.getMessage()); yacyCore.log.logSevere("database inconsistency (" + e.getMessage() + "), re-set of db."); seedDB.resetActiveTable(); return null; diff --git a/source/de/anomic/yacy/yacySeedDB.java b/source/de/anomic/yacy/yacySeedDB.java index 6a9960462..6aa5b3bf1 100644 --- a/source/de/anomic/yacy/yacySeedDB.java +++ b/source/de/anomic/yacy/yacySeedDB.java @@ -64,7 +64,7 @@ import de.anomic.kelondro.kelondroCache; import de.anomic.kelondro.kelondroDyn; import de.anomic.kelondro.kelondroException; import de.anomic.kelondro.kelondroMScoreCluster; -import de.anomic.kelondro.kelondroMap; +import de.anomic.kelondro.kelondroMapObjects; import de.anomic.kelondro.kelondroRecords; import de.anomic.net.URL; import de.anomic.plasma.plasmaSwitchboard; @@ -93,7 +93,7 @@ public final class yacySeedDB { // class objects protected File seedActiveDBFile, seedPassiveDBFile, seedPotentialDBFile; - protected kelondroMap seedActiveDB, seedPassiveDB, seedPotentialDB; + protected kelondroMapObjects seedActiveDB, seedPassiveDB, seedPotentialDB; private int seedDBBufferKB; private long preloadTime; @@ -203,18 +203,18 @@ public final class yacySeedDB { seedPotentialDB.cacheObjectStatus() }, 3); } - private synchronized kelondroMap openSeedTable(File seedDBFile) { + private synchronized kelondroMapObjects openSeedTable(File seedDBFile) { new File(seedDBFile.getParent()).mkdirs(); try { - return new kelondroMap(kelondroDyn.open(seedDBFile, (seedDBBufferKB * 0x400) / 3, preloadTime / 3, commonHashLength, 480, '#', false, false), sortFields, accFields); + return new kelondroMapObjects(kelondroDyn.open(seedDBFile, (seedDBBufferKB * 0x400) / 3, preloadTime / 3, commonHashLength, 480, '#', false, false), 500, sortFields, accFields); } catch (Exception e) { seedDBFile.delete(); // try again - return new kelondroMap(kelondroDyn.open(seedDBFile, (seedDBBufferKB * 0x400) / 3, preloadTime / 3, commonHashLength, 480, '#', false, false), sortFields, accFields); + return new kelondroMapObjects(kelondroDyn.open(seedDBFile, (seedDBBufferKB * 0x400) / 3, preloadTime / 3, commonHashLength, 480, '#', false, false), 500, sortFields, accFields); } } - protected synchronized kelondroMap resetSeedTable(kelondroMap seedDB, File seedDBFile) { + protected synchronized kelondroMapObjects resetSeedTable(kelondroMapObjects seedDB, File seedDBFile) { // this is an emergency function that should only be used if any problem with the // seed.db is detected yacyCore.log.logFine("seed-db " + seedDBFile.toString() + " reset (on-the-fly)"); @@ -357,14 +357,14 @@ public final class yacySeedDB { seedPassiveDB.remove(seed.hash); seedPotentialDB.remove(seed.hash); } catch (IOException e){ - yacyCore.log.logFine("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e); - seedActiveDB = resetSeedTable(seedActiveDB, seedActiveDBFile); + yacyCore.log.logSevere("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e); + resetActiveTable(); } catch (kelondroException e){ - yacyCore.log.logFine("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e); - seedActiveDB = resetSeedTable(seedActiveDB, seedActiveDBFile); + yacyCore.log.logSevere("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e); + resetActiveTable(); } catch (IllegalArgumentException e) { - yacyCore.log.logFine("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e); - seedActiveDB = resetSeedTable(seedActiveDB, seedActiveDBFile); + yacyCore.log.logSevere("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e); + resetActiveTable(); } } @@ -382,14 +382,14 @@ public final class yacySeedDB { seedPassiveDB.set(seed.hash, seedPropMap); } } catch (IOException e) { - yacyCore.log.logFine("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e); - seedPassiveDB = resetSeedTable(seedPassiveDB, seedPassiveDBFile); + yacyCore.log.logSevere("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e); + resetPassiveTable(); } catch (kelondroException e) { - yacyCore.log.logFine("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e); - seedPassiveDB = resetSeedTable(seedPassiveDB, seedPassiveDBFile); + yacyCore.log.logSevere("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e); + resetPassiveTable(); } catch (IllegalArgumentException e) { - yacyCore.log.logFine("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e); - seedPassiveDB = resetSeedTable(seedPassiveDB, seedPassiveDBFile); + yacyCore.log.logSevere("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e); + resetPassiveTable(); } } @@ -408,14 +408,14 @@ public final class yacySeedDB { seedPotentialDB.set(seed.hash, seedPropMap); } } catch (IOException e) { - yacyCore.log.logFine("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e); - seedPassiveDB = resetSeedTable(seedPassiveDB, seedPassiveDBFile); + yacyCore.log.logSevere("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e); + resetPotentialTable(); } catch (kelondroException e) { - yacyCore.log.logFine("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e); - seedPassiveDB = resetSeedTable(seedPassiveDB, seedPassiveDBFile); + yacyCore.log.logSevere("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e); + resetPotentialTable(); } catch (IllegalArgumentException e) { - yacyCore.log.logFine("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e); - seedPassiveDB = resetSeedTable(seedPassiveDB, seedPassiveDBFile); + yacyCore.log.logSevere("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e); + resetPotentialTable(); } } @@ -443,16 +443,12 @@ public final class yacySeedDB { } } - private yacySeed get(String hash, kelondroMap database) { + private yacySeed get(String hash, kelondroMapObjects database) { if (hash == null) return null; if ((mySeed != null) && (hash.equals(mySeed.hash))) return mySeed; - try { - Map entry = database.get(hash); + Map entry = database.getMap(hash); if (entry == null) return null; return new yacySeed(hash, entry); - } catch (IOException e) { - return null; - } } public yacySeed getConnected(String hash) { @@ -844,12 +840,12 @@ public final class yacySeedDB { class seedEnum implements Enumeration { - kelondroMap.mapIterator it; + kelondroMapObjects.mapIterator it; yacySeed nextSeed; - kelondroMap database; + kelondroMapObjects database; float minVersion; - public seedEnum(boolean up, boolean rot, byte[] firstKey, kelondroMap database, float minVersion) { + public seedEnum(boolean up, boolean rot, byte[] firstKey, kelondroMapObjects database, float minVersion) { this.database = database; this.minVersion = minVersion; try { @@ -874,7 +870,7 @@ public final class yacySeedDB { } } - public seedEnum(boolean up, String field, kelondroMap database) { + public seedEnum(boolean up, String field, kelondroMapObjects database) { this.database = database; try { it = database.maps(up, field); diff --git a/source/yacy.java b/source/yacy.java index dbc048e05..0405eb0b0 100644 --- a/source/yacy.java +++ b/source/yacy.java @@ -79,7 +79,7 @@ import de.anomic.kelondro.kelondroBitfield; import de.anomic.kelondro.kelondroDyn; import de.anomic.kelondro.kelondroException; import de.anomic.kelondro.kelondroMScoreCluster; -import de.anomic.kelondro.kelondroMap; +import de.anomic.kelondro.kelondroMapObjects; import de.anomic.kelondro.kelondroRow; import de.anomic.kelondro.kelondroTree; import de.anomic.net.URL; @@ -1324,9 +1324,9 @@ public final class yacy { String[] dbFileNames = {"seed.new.db","seed.old.db","seed.pot.db"}; for (int i=0; i < dbFileNames.length; i++) { File dbFile = new File(yacyDBPath,dbFileNames[i]); - kelondroMap db = new kelondroMap(new kelondroDyn(dbFile, (1024 * 0x400) / 3, 3000, yacySeedDB.commonHashLength, 480, '#', true, false), yacySeedDB.sortFields, yacySeedDB.accFields); + kelondroMapObjects db = new kelondroMapObjects(new kelondroDyn(dbFile, (1024 * 0x400) / 3, 3000, yacySeedDB.commonHashLength, 480, '#', true, false), 500, yacySeedDB.sortFields, yacySeedDB.accFields); - kelondroMap.mapIterator it; + kelondroMapObjects.mapIterator it; it = db.maps(true, false); while (it.hasNext()) { Map dna = (Map) it.next();