diff --git a/source/de/anomic/crawler/CrawlProfile.java b/source/de/anomic/crawler/CrawlProfile.java index f6f9463c2..e24ee1a4e 100644 --- a/source/de/anomic/crawler/CrawlProfile.java +++ b/source/de/anomic/crawler/CrawlProfile.java @@ -48,6 +48,7 @@ import java.util.HashSet; import java.util.Iterator; import java.util.Map; +import de.anomic.kelondro.kelondroBLOB; import de.anomic.kelondro.kelondroBLOBTree; import de.anomic.kelondro.kelondroBase64Order; import de.anomic.kelondro.kelondroCloneableIterator; @@ -68,7 +69,7 @@ public class CrawlProfile { public CrawlProfile(File file) { this.profileTableFile = file; profileTableFile.getParentFile().mkdirs(); - kelondroBLOBTree dyn = new kelondroBLOBTree(profileTableFile, true, true, yacySeedDB.commonHashLength, 2000, '#', kelondroNaturalOrder.naturalOrder, false, false, true); + kelondroBLOB dyn = new kelondroBLOBTree(profileTableFile, true, true, yacySeedDB.commonHashLength, 2000, '#', kelondroNaturalOrder.naturalOrder, false, false, true); profileTable = new kelondroMapObjects(dyn, 500); } @@ -77,7 +78,7 @@ public class CrawlProfile { if (profileTable != null) profileTable.close(); if (!(profileTableFile.delete())) throw new RuntimeException("cannot delete crawl profile database"); profileTableFile.getParentFile().mkdirs(); - kelondroBLOBTree dyn = new kelondroBLOBTree(profileTableFile, true, true, yacySeedDB.commonHashLength, 2000, '#', kelondroNaturalOrder.naturalOrder, false, false, true); + kelondroBLOB dyn = new kelondroBLOBTree(profileTableFile, true, true, yacySeedDB.commonHashLength, 2000, '#', kelondroNaturalOrder.naturalOrder, false, false, true); profileTable = new kelondroMapObjects(dyn, 500); } diff --git a/source/de/anomic/data/bookmarksDB.java b/source/de/anomic/data/bookmarksDB.java index 0801156dd..fc9106ea2 100644 --- a/source/de/anomic/data/bookmarksDB.java +++ b/source/de/anomic/data/bookmarksDB.java @@ -81,7 +81,6 @@ import de.anomic.kelondro.kelondroException; import de.anomic.kelondro.kelondroMapObjects; import de.anomic.kelondro.kelondroNaturalOrder; import de.anomic.kelondro.kelondroObjects; -import de.anomic.kelondro.kelondroObjectsMapEntry; import de.anomic.server.serverDate; import de.anomic.server.serverFileUtils; import de.anomic.server.logging.serverLog; @@ -227,7 +226,7 @@ public class bookmarksDB { // adding a bookmark to the bookmarksDB public void saveBookmark(Bookmark bookmark){ try { - bookmarksTable.set(bookmark.getUrlHash(), bookmark); + bookmarksTable.set(bookmark.getUrlHash(), bookmark.entry); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); @@ -241,9 +240,8 @@ public class bookmarksDB { public Bookmark getBookmark(String urlHash){ try { - kelondroObjectsMapEntry map = (kelondroObjectsMapEntry)bookmarksTable.get(urlHash); + HashMap map = bookmarksTable.get(urlHash); if (map == null) return null; - if (map instanceof Bookmark) return (Bookmark)map; return new Bookmark(map); } catch (IOException e) { return null; @@ -888,7 +886,7 @@ public class bookmarksDB { /** * Subclass of bookmarksDB, which provides the Bookmark object-type */ - public class Bookmark extends kelondroObjectsMapEntry { + public class Bookmark { public static final String BOOKMARK_URL="bookmarkUrl"; public static final String BOOKMARK_TITLE="bookmarkTitle"; public static final String BOOKMARK_DESCRIPTION="bookmarkDesc"; @@ -900,9 +898,10 @@ public class bookmarksDB { private String urlHash; private Set tags; private long timestamp; + HashMap entry; public Bookmark(String urlHash, HashMap map) { - super(map); + this.entry = map; this.urlHash=urlHash; if(map.containsKey(BOOKMARK_TAGS)) tags=listManager.string2set(map.get(BOOKMARK_TAGS)); @@ -912,7 +911,7 @@ public class bookmarksDB { } public Bookmark(String url){ - super(); + entry = new HashMap(); if(!url.toLowerCase().startsWith("http://") && !url.toLowerCase().startsWith("https://")){ url="http://"+url; } @@ -938,7 +937,7 @@ public class bookmarksDB { } public Bookmark(String urlHash, yacyURL url) { - super(); + entry = new HashMap(); this.urlHash=urlHash; entry.put(BOOKMARK_URL, url.toNormalform(false, true)); tags=new HashSet(); @@ -946,18 +945,18 @@ public class bookmarksDB { } public Bookmark(String urlHash, String url) { - super(); + entry = new HashMap(); this.urlHash=urlHash; entry.put(BOOKMARK_URL, url); tags=new HashSet(); timestamp=System.currentTimeMillis(); } - public Bookmark(kelondroObjectsMapEntry map) throws MalformedURLException { - this((new yacyURL(map.map().get(BOOKMARK_URL), null)).hash(), map.map()); + public Bookmark(HashMap map) throws MalformedURLException { + this((new yacyURL(map.get(BOOKMARK_URL), null)).hash(), map); } - Map toMap() { + private Map toMap() { entry.put(BOOKMARK_TAGS, listManager.collection2string(tags)); entry.put(BOOKMARK_TIMESTAMP, String.valueOf(this.timestamp)); return entry; diff --git a/source/de/anomic/kelondro/kelondroBLOB.java b/source/de/anomic/kelondro/kelondroBLOB.java index 5ac48e25a..ba7c831c1 100644 --- a/source/de/anomic/kelondro/kelondroBLOB.java +++ b/source/de/anomic/kelondro/kelondroBLOB.java @@ -28,17 +28,101 @@ package de.anomic.kelondro; import java.io.IOException; +import de.anomic.kelondro.kelondroBLOBTree.keyIterator; + public interface kelondroBLOB { + /** + * ask for the length of the primary key + * @return the length of the key + */ public int keylength(); + + /** + * clears the content of the database + * @throws IOException + */ public void clear() throws IOException; + + /** + * ask for the number of entries + * @return the number of entries in the table + */ public int size(); + + /** + * iterator over all keys + * @param up + * @param rotating + * @return + * @throws IOException + */ public kelondroCloneableIterator keys(boolean up, boolean rotating) throws IOException; + + /** + * iterate over all keys + * @param up + * @param firstKey + * @return + * @throws IOException + */ + public keyIterator keys(boolean up, byte[] firstKey) throws IOException; + + /** + * check if a specific key is in the database + * @param key the primary key + * @return + * @throws IOException + */ + public boolean has(String key) throws IOException; + + /** + * retrieve the whole BLOB from the table + * @param key the primary key + * @return + * @throws IOException + */ + public byte[] get(String key) throws IOException; + + /** + * retrieve a fragment of a BLOB from the table + * @param key the primary key + * @param pos the position within the BLOB fragment + * @param len the length of the fragment + * @return + * @throws IOException + */ public byte[] get(String key, int pos, int len) throws IOException; + + /** + * write a whole byte array as BLOB to the table + * @param key the primary key + * @param b + * @throws IOException + */ + public void put(String key, byte[] b) throws IOException; + + /** + * write a fragment of a BLOB to the table + * @param key the primary key + * @param pos the position of the BLOB fragment + * @param b a byte array + * @param off the offset within the array where the BLOB fragment starts + * @param len the length of the fragment + * @throws IOException + */ public void put(String key, int pos, byte[] b, int off, int len) throws IOException; + + /** + * remove a BLOB + * @param key the primary key + * @throws IOException + */ public void remove(String key) throws IOException; - public boolean exist(String key) throws IOException; - public kelondroRA getRA(String filekey); + + /** + * close the BLOB table + */ public void close(); } diff --git a/source/de/anomic/kelondro/kelondroBLOBTree.java b/source/de/anomic/kelondro/kelondroBLOBTree.java index 2eeed50fb..7f9bd9149 100644 --- a/source/de/anomic/kelondro/kelondroBLOBTree.java +++ b/source/de/anomic/kelondro/kelondroBLOBTree.java @@ -38,8 +38,6 @@ package de.anomic.kelondro; import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; import java.io.IOException; import java.util.Iterator; @@ -227,7 +225,7 @@ public class kelondroBLOBTree implements kelondroBLOB { } } - synchronized int get(String key, int pos) throws IOException { + private synchronized int get(String key, int pos) throws IOException { int reccnt = pos / reclen; // read within a single record byte[] buf = getValueCached(elementKey(key, reccnt)); @@ -237,6 +235,12 @@ public class kelondroBLOBTree implements kelondroBLOB { return buf[recpos] & 0xFF; } + public synchronized byte[] get(String key) throws IOException { + kelondroRA ra = getRA(key); + if (ra == null) return null; + return ra.readFully(); + } + public synchronized byte[] get(String key, int pos, int len) throws IOException { int recpos = pos % reclen; int reccnt = pos / reclen; @@ -281,6 +285,10 @@ public class kelondroBLOBTree implements kelondroBLOB { return result; } + public synchronized void put(String key, byte[] b) throws IOException { + put(key, 0, b, 0, b.length); + } + public synchronized void put(String key, int pos, byte[] b, int off, int len) throws IOException { int recpos = pos % reclen; int reccnt = pos / reclen; @@ -331,7 +339,7 @@ public class kelondroBLOBTree implements kelondroBLOB { //segmentCount--; writeSegmentCount(); } - public synchronized boolean exist(String key) throws IOException { + public synchronized boolean has(String key) throws IOException { return (key != null) && (getValueCached(elementKey(key, 0)) != null); } @@ -393,60 +401,6 @@ public class kelondroBLOBTree implements kelondroBLOB { } } - - public static void writeFile(kelondroBLOB blob, String key, File f) throws IOException { - // reads a file from the FS and writes it into the database - kelondroRA kra = null; - FileInputStream fis = null; - try { - kra = blob.getRA(key); - byte[] buffer = new byte[1024]; - byte[] result = new byte[(int) f.length()]; - fis = new FileInputStream(f); - int i; - int pos = 0; - while ((i = fis.read(buffer)) > 0) { - System.arraycopy(buffer, 0, result, pos, i); - pos += i; - } - fis.close(); - kra.writeArray(result); - } finally { - if (fis != null) - try { - fis.close(); - } catch (Exception e) { - } - if (kra != null) - try { - kra.close(); - } catch (Exception e) { - } - } - } - - public static void readFile(kelondroBLOB blob, String key, File f) throws IOException { - // reads a file from the DB and writes it to the FS - kelondroRA kra = null; - FileOutputStream fos = null; - try { - kra = blob.getRA(key); - byte[] result = kra.readArray(); - fos = new FileOutputStream(f); - fos.write(result); - } finally { - if (fos != null) - try { - fos.close(); - } catch (Exception e) { - } - if (kra != null) - try { - kra.close(); - } catch (Exception e) { - } - } - } public synchronized void close() { index.close(); @@ -471,22 +425,6 @@ public class kelondroBLOBTree implements kelondroBLOB { e.printStackTrace(); } } - if (args.length == 4) { - boolean writeFile = (args[0].equals("-db2f")); - File db = new File(args[1]); - String key = args[2]; - File f = new File(args[3]); - kelondroBLOBTree kd; - try { - kd = new kelondroBLOBTree(db, true, true, 80, 200, '_', kelondroNaturalOrder.naturalOrder, false, false, true); - if (writeFile) - readFile(kd, key, f); - else - writeFile(kd, key, f); - } catch (IOException e) { - System.out.println("ERROR: " + e.toString()); - } - } } public static int countElements(kelondroBLOBTree t) { diff --git a/source/de/anomic/kelondro/kelondroMapObjects.java b/source/de/anomic/kelondro/kelondroMapObjects.java index 321ecb65c..36d167936 100644 --- a/source/de/anomic/kelondro/kelondroMapObjects.java +++ b/source/de/anomic/kelondro/kelondroMapObjects.java @@ -41,12 +41,12 @@ public class kelondroMapObjects extends kelondroObjects { private HashMap accMap; // to store accumulations of specific fields private int elementCount; - public kelondroMapObjects(kelondroBLOBTree dyn, int cachesize) { + public kelondroMapObjects(kelondroBLOB dyn, int cachesize) { this(dyn, cachesize, null, null, null, null, null); } @SuppressWarnings({ "unchecked", "null" }) - public kelondroMapObjects(kelondroBLOBTree dyn, int cachesize, String[] sortfields, String[] longaccfields, String[] doubleaccfields, Method externalInitializer, Object externalHandler) { + public kelondroMapObjects(kelondroBLOB dyn, int cachesize, String[] sortfields, String[] longaccfields, String[] doubleaccfields, Method externalInitializer, Object externalHandler) { super(dyn, cachesize); // create fast ordering clusters and acc fields @@ -189,7 +189,7 @@ public class kelondroMapObjects extends kelondroObjects { } } - super.set(key, new kelondroObjectsMapEntry(newMap)); + super.set(key, newMap); // update sortCluster if (sortClusterMap != null) updateSortCluster(key, newMap); @@ -269,9 +269,9 @@ public class kelondroMapObjects extends kelondroObjects { public HashMap getMap(String key) { try { - kelondroObjectsMapEntry mapEntry = (kelondroObjectsMapEntry) super.get(key); + HashMap mapEntry = super.get(key); if (mapEntry == null) return null; - return mapEntry.map(); + return mapEntry; } catch (IOException e) { e.printStackTrace(); return null; @@ -280,9 +280,9 @@ public class kelondroMapObjects extends kelondroObjects { protected Map getMap(String key, boolean cache) { try { - kelondroObjectsMapEntry mapEntry = (kelondroObjectsMapEntry) super.get(key, cache); + HashMap mapEntry = super.get(key, cache); if (mapEntry == null) return null; - return mapEntry.map(); + return mapEntry; } catch (IOException e) { e.printStackTrace(); return null; diff --git a/source/de/anomic/kelondro/kelondroObjects.java b/source/de/anomic/kelondro/kelondroObjects.java index 4b6b79fea..eff683197 100644 --- a/source/de/anomic/kelondro/kelondroObjects.java +++ b/source/de/anomic/kelondro/kelondroObjects.java @@ -27,22 +27,26 @@ package de.anomic.kelondro; +import java.io.BufferedReader; +import java.io.ByteArrayInputStream; import java.io.IOException; +import java.io.InputStreamReader; import java.util.HashMap; import java.util.Iterator; +import java.util.Map; public class kelondroObjects { - private kelondroBLOBTree blob; + private kelondroBLOB blob; private kelondroMScoreCluster cacheScore; - private HashMap cache; + private HashMap> cache; private long startup; private int cachesize; - public kelondroObjects(kelondroBLOBTree blob, int cachesize) { + public kelondroObjects(kelondroBLOB blob, int cachesize) { this.blob = blob; - this.cache = new HashMap(); + this.cache = new HashMap>(); this.cacheScore = new kelondroMScoreCluster(); this.startup = System.currentTimeMillis(); this.cachesize = cachesize; @@ -50,7 +54,7 @@ public class kelondroObjects { public void clear() throws IOException { this.blob.clear(); - this.cache = new HashMap(); + this.cache = new HashMap>(); this.cacheScore = new kelondroMScoreCluster(); } @@ -58,16 +62,46 @@ public class kelondroObjects { return blob.keylength(); } - public synchronized void set(String key, kelondroObjectsEntry newMap) throws IOException { + + private static String map2string(final Map map, final String comment) throws IOException { + final Iterator> iter = map.entrySet().iterator(); + Map.Entry entry; + final StringBuffer bb = new StringBuffer(map.size() * 40); + bb.append("# ").append(comment).append("\r\n"); + while (iter.hasNext()) { + entry = iter.next(); + bb.append(entry.getKey()).append('='); + if (entry.getValue() != null) { bb.append(entry.getValue()); } + bb.append("\r\n"); + } + bb.append("# EOF\r\n"); + return bb.toString(); + } + + private static HashMap string2map(String s) throws IOException { + BufferedReader br = new BufferedReader(new InputStreamReader(new ByteArrayInputStream(s.getBytes()))); + final HashMap map = new HashMap(); + String line; + int pos; + while ((line = br.readLine()) != null) { // very slow readLine???? + line = line.trim(); + if (line.equals("# EOF")) return map; + if ((line.length() == 0) || (line.charAt(0) == '#')) continue; + pos = line.indexOf("="); + if (pos < 0) continue; + map.put(line.substring(0, pos), line.substring(pos + 1)); + } + return map; + } + + public synchronized void set(String key, HashMap newMap) throws IOException { assert (key != null); assert (key.length() > 0); assert (newMap != null); if (cacheScore == null) return; // may appear during shutdown // write entry - kelondroRA kra = blob.getRA(key); - newMap.write(kra); - kra.close(); + blob.put(key, map2string(newMap, "").getBytes()); // check for space in cache checkCacheSpace(); @@ -89,25 +123,25 @@ public class kelondroObjects { blob.remove(key); } - public synchronized kelondroObjectsEntry get(final String key) throws IOException { + public synchronized HashMap get(final String key) throws IOException { if (key == null) return null; return get(key, true); } - protected synchronized kelondroObjectsEntry get(final String key, final boolean storeCache) throws IOException { + protected synchronized HashMap get(final String key, final boolean storeCache) throws IOException { // load map from cache assert key != null; if (cache == null) return null; // case may appear during shutdown - kelondroObjectsEntry map = cache.get(key); + HashMap map = cache.get(key); if (map != null) return map; // load map from kra - if (!(blob.exist(key))) return null; + if (!(blob.has(key))) return null; // read object - kelondroRA kra = blob.getRA(key); - map = new kelondroObjectsMapEntry(kra); - kra.close(); + byte[] b = blob.get(key); + if (b == null) return null; + map = string2map(new String(b)); if (storeCache) { // cache it also @@ -167,7 +201,7 @@ public class kelondroObjects { blob.close(); } - public class objectIterator implements Iterator { + public class objectIterator implements Iterator> { // enumerates Map-Type elements // the key is also included in every map that is returned; it's key is 'key' @@ -183,14 +217,14 @@ public class kelondroObjects { return (!(finish)) && (keyIterator.hasNext()); } - public kelondroObjectsEntry next() { + public HashMap next() { final String nextKey = keyIterator.next(); if (nextKey == null) { finish = true; return null; } try { - final kelondroObjectsEntry obj = get(nextKey); + final HashMap obj = get(nextKey); if (obj == null) throw new kelondroException("no more elements available"); return obj; } catch (IOException e) { diff --git a/source/de/anomic/kelondro/kelondroObjectsEntry.java b/source/de/anomic/kelondro/kelondroObjectsEntry.java deleted file mode 100644 index 30f825f42..000000000 --- a/source/de/anomic/kelondro/kelondroObjectsEntry.java +++ /dev/null @@ -1,35 +0,0 @@ -// kelondroObjectsEntry.java -// ------------------------- -// (C) 29.01.2007 by Michael Peter Christen; mc@anomic.de, Frankfurt a. M., Germany -// first published 2004 as part of kelondroMap on http://www.anomic.de -// -// This is a part of YaCy, a peer-to-peer based web search engine -// -// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $ -// $LastChangedRevision: 1986 $ -// $LastChangedBy: orbiter $ -// -// LICENSE -// -// This program is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - -package de.anomic.kelondro; - -public interface kelondroObjectsEntry { - - public void write(kelondroRA ra); - public void read(kelondroRA ra); - -} diff --git a/source/de/anomic/kelondro/kelondroObjectsMapEntry.java b/source/de/anomic/kelondro/kelondroObjectsMapEntry.java deleted file mode 100644 index 0b6b33101..000000000 --- a/source/de/anomic/kelondro/kelondroObjectsMapEntry.java +++ /dev/null @@ -1,69 +0,0 @@ -// kelondroObjectsMapEntry.java -// ---------------------------- -// (C) 29.01.2007 by Michael Peter Christen; mc@anomic.de, Frankfurt a. M., Germany -// first published 2004 as part of kelondroMap on http://www.anomic.de -// -// This is a part of YaCy, a peer-to-peer based web search engine -// -// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $ -// $LastChangedRevision: 1986 $ -// $LastChangedBy: orbiter $ -// -// LICENSE -// -// This program is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - -package de.anomic.kelondro; - -import java.io.IOException; -import java.util.HashMap; - -public class kelondroObjectsMapEntry implements kelondroObjectsEntry { - - protected HashMap entry; - - public kelondroObjectsMapEntry() { - this.entry = new HashMap(); - } - - public kelondroObjectsMapEntry(HashMap map) { - this.entry = map; - } - - public kelondroObjectsMapEntry(kelondroRA ra) { - this.read(ra); - } - - public void read(kelondroRA ra) { - try { - this.entry = ra.readMap(); - } catch (IOException e) { - e.printStackTrace(); - } - } - - public void write(kelondroRA ra) { - try { - ra.writeMap(this.entry, ""); - } catch (IOException e) { - e.printStackTrace(); - } - } - - public HashMap map() { - return this.entry; - } - -}