boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

- removed unnecessary classes (these objects are much easier to handle using generics)

Browse Source 
- generalized BLOB referencing. This is the preparation to use another BLOB class, the kelondroHeap


git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4977 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 17 years ago
parent b38f467e3c
commit 81f75f5056
8 changed files with 172 additions and 220 deletions
Show Stats Download Patch File Download Diff File

5
source/de/anomic/crawler/CrawlProfile.java
Unescape View File

@ -48,6 +48,7 @@ import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
import de.anomic.kelondro.kelondroBLOB;
import de.anomic.kelondro.kelondroBLOBTree;
import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroCloneableIterator;
@ -68,7 +69,7 @@ public class CrawlProfile {
public CrawlProfile(File file) {
this.profileTableFile = file;
profileTableFile.getParentFile().mkdirs();
kelondroBLOBTree dyn = new kelondroBLOBTree(profileTableFile, true, true, yacySeedDB.commonHashLength, 2000, '#', kelondroNaturalOrder.naturalOrder, false, false, true);
kelondroBLOB dyn = new kelondroBLOBTree(profileTableFile, true, true, yacySeedDB.commonHashLength, 2000, '#', kelondroNaturalOrder.naturalOrder, false, false, true);
profileTable = new kelondroMapObjects(dyn, 500);
}
@ -77,7 +78,7 @@ public class CrawlProfile {
if (profileTable != null) profileTable.close();
if (!(profileTableFile.delete())) throw new RuntimeException("cannot delete crawl profile database");
profileTableFile.getParentFile().mkdirs();
kelondroBLOBTree dyn = new kelondroBLOBTree(profileTableFile, true, true, yacySeedDB.commonHashLength, 2000, '#', kelondroNaturalOrder.naturalOrder, false, false, true);
kelondroBLOB dyn = new kelondroBLOBTree(profileTableFile, true, true, yacySeedDB.commonHashLength, 2000, '#', kelondroNaturalOrder.naturalOrder, false, false, true);
profileTable = new kelondroMapObjects(dyn, 500);
}

23
source/de/anomic/data/bookmarksDB.java
Unescape View File

@ -81,7 +81,6 @@ import de.anomic.kelondro.kelondroException;
import de.anomic.kelondro.kelondroMapObjects;
import de.anomic.kelondro.kelondroNaturalOrder;
import de.anomic.kelondro.kelondroObjects;
import de.anomic.kelondro.kelondroObjectsMapEntry;
import de.anomic.server.serverDate;
import de.anomic.server.serverFileUtils;
import de.anomic.server.logging.serverLog;
@ -227,7 +226,7 @@ public class bookmarksDB {
// adding a bookmark to the bookmarksDB
public void saveBookmark(Bookmark bookmark){
try {
bookmarksTable.set(bookmark.getUrlHash(), bookmark);
bookmarksTable.set(bookmark.getUrlHash(), bookmark.entry);
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
@ -241,9 +240,8 @@ public class bookmarksDB {
public Bookmark getBookmark(String urlHash){
try {
kelondroObjectsMapEntry map = (kelondroObjectsMapEntry)bookmarksTable.get(urlHash);
HashMap<String, String> map = bookmarksTable.get(urlHash);
if (map == null) return null;
if (map instanceof Bookmark) return (Bookmark)map;
return new Bookmark(map);
} catch (IOException e) {
return null;
@ -888,7 +886,7 @@ public class bookmarksDB {
/**
* Subclass of bookmarksDB, which provides the Bookmark object-type
*/
public class Bookmark extends kelondroObjectsMapEntry {
public class Bookmark {
public static final String BOOKMARK_URL="bookmarkUrl";
public static final String BOOKMARK_TITLE="bookmarkTitle";
public static final String BOOKMARK_DESCRIPTION="bookmarkDesc";
@ -900,9 +898,10 @@ public class bookmarksDB {
private String urlHash;
private Set<String> tags;
private long timestamp;
HashMap<String, String> entry;
public Bookmark(String urlHash, HashMap<String, String> map) {
super(map);
this.entry = map;
this.urlHash=urlHash;
if(map.containsKey(BOOKMARK_TAGS))
tags=listManager.string2set(map.get(BOOKMARK_TAGS));
@ -912,7 +911,7 @@ public class bookmarksDB {
}
public Bookmark(String url){
super();
entry = new HashMap<String, String>();
if(!url.toLowerCase().startsWith("http://") && !url.toLowerCase().startsWith("https://")){
url="http://"+url;
}
@ -938,7 +937,7 @@ public class bookmarksDB {
}
public Bookmark(String urlHash, yacyURL url) {
super();
entry = new HashMap<String, String>();
this.urlHash=urlHash;
entry.put(BOOKMARK_URL, url.toNormalform(false, true));
tags=new HashSet<String>();
@ -946,18 +945,18 @@ public class bookmarksDB {
}
public Bookmark(String urlHash, String url) {
super();
entry = new HashMap<String, String>();
this.urlHash=urlHash;
entry.put(BOOKMARK_URL, url);
tags=new HashSet<String>();
timestamp=System.currentTimeMillis();
}
public Bookmark(kelondroObjectsMapEntry map) throws MalformedURLException {
this((new yacyURL(map.map().get(BOOKMARK_URL), null)).hash(), map.map());
public Bookmark(HashMap<String, String> map) throws MalformedURLException {
this((new yacyURL(map.get(BOOKMARK_URL), null)).hash(), map);
}
Map<String, String> toMap() {
private Map<String, String> toMap() {
entry.put(BOOKMARK_TAGS, listManager.collection2string(tags));
entry.put(BOOKMARK_TIMESTAMP, String.valueOf(this.timestamp));
return entry;

88
source/de/anomic/kelondro/kelondroBLOB.java
Unescape View File

@ -28,17 +28,101 @@ package de.anomic.kelondro;
import java.io.IOException;
import de.anomic.kelondro.kelondroBLOBTree.keyIterator;
public interface kelondroBLOB {
/**
* ask for the length of the primary key
* @return the length of the key
*/
public int keylength();
/**
* clears the content of the database
* @throws IOException
*/
public void clear() throws IOException;
/**
* ask for the number of entries
* @return the number of entries in the table
*/
public int size();
/**
* iterator over all keys
* @param up
* @param rotating
* @return
* @throws IOException
*/
public kelondroCloneableIterator<String> keys(boolean up, boolean rotating) throws IOException;
/**
* iterate over all keys
* @param up
* @param firstKey
* @return
* @throws IOException
*/
public keyIterator keys(boolean up, byte[] firstKey) throws IOException;
/**
* check if a specific key is in the database
* @param key the primary key
* @return
* @throws IOException
*/
public boolean has(String key) throws IOException;
/**
* retrieve the whole BLOB from the table
* @param key the primary key
* @return
* @throws IOException
*/
public byte[] get(String key) throws IOException;
/**
* retrieve a fragment of a BLOB from the table
* @param key the primary key
* @param pos the position within the BLOB fragment
* @param len the length of the fragment
* @return
* @throws IOException
*/
public byte[] get(String key, int pos, int len) throws IOException;
/**
* write a whole byte array as BLOB to the table
* @param key the primary key
* @param b
* @throws IOException
*/
public void put(String key, byte[] b) throws IOException;
/**
* write a fragment of a BLOB to the table
* @param key the primary key
* @param pos the position of the BLOB fragment
* @param b a byte array
* @param off the offset within the array where the BLOB fragment starts
* @param len the length of the fragment
* @throws IOException
*/
public void put(String key, int pos, byte[] b, int off, int len) throws IOException;
/**
* remove a BLOB
* @param key the primary key
* @throws IOException
*/
public void remove(String key) throws IOException;
public boolean exist(String key) throws IOException;
public kelondroRA getRA(String filekey);
/**
* close the BLOB table
*/
public void close();
}

86
source/de/anomic/kelondro/kelondroBLOBTree.java
Unescape View File

@ -38,8 +38,6 @@
package de.anomic.kelondro;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.Iterator;
@ -227,7 +225,7 @@ public class kelondroBLOBTree implements kelondroBLOB {
}
}
synchronized int get(String key, int pos) throws IOException {
private synchronized int get(String key, int pos) throws IOException {
int reccnt = pos / reclen;
// read within a single record
byte[] buf = getValueCached(elementKey(key, reccnt));
@ -237,6 +235,12 @@ public class kelondroBLOBTree implements kelondroBLOB {
return buf[recpos] & 0xFF;
}
public synchronized byte[] get(String key) throws IOException {
kelondroRA ra = getRA(key);
if (ra == null) return null;
return ra.readFully();
}
public synchronized byte[] get(String key, int pos, int len) throws IOException {
int recpos = pos % reclen;
int reccnt = pos / reclen;
@ -281,6 +285,10 @@ public class kelondroBLOBTree implements kelondroBLOB {
return result;
}
public synchronized void put(String key, byte[] b) throws IOException {
put(key, 0, b, 0, b.length);
}
public synchronized void put(String key, int pos, byte[] b, int off, int len) throws IOException {
int recpos = pos % reclen;
int reccnt = pos / reclen;
@ -331,7 +339,7 @@ public class kelondroBLOBTree implements kelondroBLOB {
//segmentCount--; writeSegmentCount();
}
public synchronized boolean exist(String key) throws IOException {
public synchronized boolean has(String key) throws IOException {
return (key != null) && (getValueCached(elementKey(key, 0)) != null);
}
@ -394,60 +402,6 @@ public class kelondroBLOBTree implements kelondroBLOB {
}
public static void writeFile(kelondroBLOB blob, String key, File f) throws IOException {
// reads a file from the FS and writes it into the database
kelondroRA kra = null;
FileInputStream fis = null;
try {
kra = blob.getRA(key);
byte[] buffer = new byte[1024];
byte[] result = new byte[(int) f.length()];
fis = new FileInputStream(f);
int i;
int pos = 0;
while ((i = fis.read(buffer)) > 0) {
System.arraycopy(buffer, 0, result, pos, i);
pos += i;
}
fis.close();
kra.writeArray(result);
} finally {
if (fis != null)
try {
fis.close();
} catch (Exception e) {
}
if (kra != null)
try {
kra.close();
} catch (Exception e) {
}
}
}
public static void readFile(kelondroBLOB blob, String key, File f) throws IOException {
// reads a file from the DB and writes it to the FS
kelondroRA kra = null;
FileOutputStream fos = null;
try {
kra = blob.getRA(key);
byte[] result = kra.readArray();
fos = new FileOutputStream(f);
fos.write(result);
} finally {
if (fos != null)
try {
fos.close();
} catch (Exception e) {
}
if (kra != null)
try {
kra.close();
} catch (Exception e) {
}
}
}
public synchronized void close() {
index.close();
}
@ -471,22 +425,6 @@ public class kelondroBLOBTree implements kelondroBLOB {
e.printStackTrace();
}
}
if (args.length == 4) {
boolean writeFile = (args[0].equals("-db2f"));
File db = new File(args[1]);
String key = args[2];
File f = new File(args[3]);
kelondroBLOBTree kd;
try {
kd = new kelondroBLOBTree(db, true, true, 80, 200, '_', kelondroNaturalOrder.naturalOrder, false, false, true);
if (writeFile)
readFile(kd, key, f);
else
writeFile(kd, key, f);
} catch (IOException e) {
System.out.println("ERROR: " + e.toString());
}
}
}
public static int countElements(kelondroBLOBTree t) {

14
source/de/anomic/kelondro/kelondroMapObjects.java
Unescape View File

@ -41,12 +41,12 @@ public class kelondroMapObjects extends kelondroObjects {
private HashMap<String, Object> accMap; // to store accumulations of specific fields
private int elementCount;
public kelondroMapObjects(kelondroBLOBTree dyn, int cachesize) {
public kelondroMapObjects(kelondroBLOB dyn, int cachesize) {
this(dyn, cachesize, null, null, null, null, null);
}
@SuppressWarnings({ "unchecked", "null" })
public kelondroMapObjects(kelondroBLOBTree dyn, int cachesize, String[] sortfields, String[] longaccfields, String[] doubleaccfields, Method externalInitializer, Object externalHandler) {
public kelondroMapObjects(kelondroBLOB dyn, int cachesize, String[] sortfields, String[] longaccfields, String[] doubleaccfields, Method externalInitializer, Object externalHandler) {
super(dyn, cachesize);
// create fast ordering clusters and acc fields
@ -189,7 +189,7 @@ public class kelondroMapObjects extends kelondroObjects {
}
}
super.set(key, new kelondroObjectsMapEntry(newMap));
super.set(key, newMap);
// update sortCluster
if (sortClusterMap != null) updateSortCluster(key, newMap);
@ -269,9 +269,9 @@ public class kelondroMapObjects extends kelondroObjects {
public HashMap<String, String> getMap(String key) {
try {
kelondroObjectsMapEntry mapEntry = (kelondroObjectsMapEntry) super.get(key);
HashMap<String, String> mapEntry = super.get(key);
if (mapEntry == null) return null;
return mapEntry.map();
return mapEntry;
} catch (IOException e) {
e.printStackTrace();
return null;
@ -280,9 +280,9 @@ public class kelondroMapObjects extends kelondroObjects {
protected Map<String, String> getMap(String key, boolean cache) {
try {
kelondroObjectsMapEntry mapEntry = (kelondroObjectsMapEntry) super.get(key, cache);
HashMap<String, String> mapEntry = super.get(key, cache);
if (mapEntry == null) return null;
return mapEntry.map();
return mapEntry;
} catch (IOException e) {
e.printStackTrace();
return null;

72
source/de/anomic/kelondro/kelondroObjects.java
Unescape View File

@ -27,22 +27,26 @@
package de.anomic.kelondro;
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
public class kelondroObjects {
private kelondroBLOBTree blob;
private kelondroBLOB blob;
private kelondroMScoreCluster<String> cacheScore;
private HashMap<String, kelondroObjectsEntry> cache;
private HashMap<String, HashMap<String, String>> cache;
private long startup;
private int cachesize;
public kelondroObjects(kelondroBLOBTree blob, int cachesize) {
public kelondroObjects(kelondroBLOB blob, int cachesize) {
this.blob = blob;
this.cache = new HashMap<String, kelondroObjectsEntry>();
this.cache = new HashMap<String, HashMap<String, String>>();
this.cacheScore = new kelondroMScoreCluster<String>();
this.startup = System.currentTimeMillis();
this.cachesize = cachesize;
@ -50,7 +54,7 @@ public class kelondroObjects {
public void clear() throws IOException {
this.blob.clear();
this.cache = new HashMap<String, kelondroObjectsEntry>();
this.cache = new HashMap<String, HashMap<String, String>>();
this.cacheScore = new kelondroMScoreCluster<String>();
}
@ -58,16 +62,46 @@ public class kelondroObjects {
return blob.keylength();
}
public synchronized void set(String key, kelondroObjectsEntry newMap) throws IOException {
private static String map2string(final Map<String, String> map, final String comment) throws IOException {
final Iterator<Map.Entry<String, String>> iter = map.entrySet().iterator();
Map.Entry<String, String> entry;
final StringBuffer bb = new StringBuffer(map.size() * 40);
bb.append("# ").append(comment).append("\r\n");
while (iter.hasNext()) {
entry = iter.next();
bb.append(entry.getKey()).append('=');
if (entry.getValue() != null) { bb.append(entry.getValue()); }
bb.append("\r\n");
}
bb.append("# EOF\r\n");
return bb.toString();
}
private static HashMap<String, String> string2map(String s) throws IOException {
BufferedReader br = new BufferedReader(new InputStreamReader(new ByteArrayInputStream(s.getBytes())));
final HashMap<String, String> map = new HashMap<String, String>();
String line;
int pos;
while ((line = br.readLine()) != null) { // very slow readLine????
line = line.trim();
if (line.equals("# EOF")) return map;
if ((line.length() == 0) || (line.charAt(0) == '#')) continue;
pos = line.indexOf("=");
if (pos < 0) continue;
map.put(line.substring(0, pos), line.substring(pos + 1));
}
return map;
}
public synchronized void set(String key, HashMap<String, String> newMap) throws IOException {
assert (key != null);
assert (key.length() > 0);
assert (newMap != null);
if (cacheScore == null) return; // may appear during shutdown
// write entry
kelondroRA kra = blob.getRA(key);
newMap.write(kra);
kra.close();
blob.put(key, map2string(newMap, "").getBytes());
// check for space in cache
checkCacheSpace();
@ -89,25 +123,25 @@ public class kelondroObjects {
blob.remove(key);
}
public synchronized kelondroObjectsEntry get(final String key) throws IOException {
public synchronized HashMap<String, String> get(final String key) throws IOException {
if (key == null) return null;
return get(key, true);
}
protected synchronized kelondroObjectsEntry get(final String key, final boolean storeCache) throws IOException {
protected synchronized HashMap<String, String> get(final String key, final boolean storeCache) throws IOException {
// load map from cache
assert key != null;
if (cache == null) return null; // case may appear during shutdown
kelondroObjectsEntry map = cache.get(key);
HashMap<String, String> map = cache.get(key);
if (map != null) return map;
// load map from kra
if (!(blob.exist(key))) return null;
if (!(blob.has(key))) return null;
// read object
kelondroRA kra = blob.getRA(key);
map = new kelondroObjectsMapEntry(kra);
kra.close();
byte[] b = blob.get(key);
if (b == null) return null;
map = string2map(new String(b));
if (storeCache) {
// cache it also
@ -167,7 +201,7 @@ public class kelondroObjects {
blob.close();
}
public class objectIterator implements Iterator<kelondroObjectsEntry> {
public class objectIterator implements Iterator<HashMap<String, String>> {
// enumerates Map-Type elements
// the key is also included in every map that is returned; it's key is 'key'
@ -183,14 +217,14 @@ public class kelondroObjects {
return (!(finish)) && (keyIterator.hasNext());
}
public kelondroObjectsEntry next() {
public HashMap<String, String> next() {
final String nextKey = keyIterator.next();
if (nextKey == null) {
finish = true;
return null;
}
try {
final kelondroObjectsEntry obj = get(nextKey);
final HashMap<String, String> obj = get(nextKey);
if (obj == null) throw new kelondroException("no more elements available");
return obj;
} catch (IOException e) {

35
source/de/anomic/kelondro/kelondroObjectsEntry.java
Unescape View File

@ -1,35 +0,0 @@
// kelondroObjectsEntry.java
// -------------------------
// (C) 29.01.2007 by Michael Peter Christen; mc@anomic.de, Frankfurt a. M., Germany
// first published 2004 as part of kelondroMap on http://www.anomic.de
//
// This is a part of YaCy, a peer-to-peer based web search engine
//
// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
// $LastChangedRevision: 1986 $
// $LastChangedBy: orbiter $
//
// LICENSE
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package de.anomic.kelondro;
public interface kelondroObjectsEntry {
public void write(kelondroRA ra);
public void read(kelondroRA ra);
}

69
source/de/anomic/kelondro/kelondroObjectsMapEntry.java
Unescape View File

@ -1,69 +0,0 @@
// kelondroObjectsMapEntry.java
// ----------------------------
// (C) 29.01.2007 by Michael Peter Christen; mc@anomic.de, Frankfurt a. M., Germany
// first published 2004 as part of kelondroMap on http://www.anomic.de
//
// This is a part of YaCy, a peer-to-peer based web search engine
//
// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
// $LastChangedRevision: 1986 $
// $LastChangedBy: orbiter $
//
// LICENSE
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package de.anomic.kelondro;
import java.io.IOException;
import java.util.HashMap;
public class kelondroObjectsMapEntry implements kelondroObjectsEntry {
protected HashMap<String, String> entry;
public kelondroObjectsMapEntry() {
this.entry = new HashMap<String, String>();
}
public kelondroObjectsMapEntry(HashMap<String, String> map) {
this.entry = map;
}
public kelondroObjectsMapEntry(kelondroRA ra) {
this.read(ra);
}
public void read(kelondroRA ra) {
try {
this.entry = ra.readMap();
} catch (IOException e) {
e.printStackTrace();
}
}
public void write(kelondroRA ra) {
try {
ra.writeMap(this.entry, "");
} catch (IOException e) {
e.printStackTrace();
}
}
public HashMap<String, String> map() {
return this.entry;
}
}
Write Preview
Loading…
Cancel
Save