boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

refactoring of kelondroObjects (mainly renaming to kelondroMap)

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4982 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 17 years ago
parent 441e9c861e
commit 4acf0a61cd
13 changed files with 157 additions and 109 deletions
Show Stats Download Patch File Download Diff File

22
source/de/anomic/crawler/CrawlProfile.java
Unescape View File

@ -53,7 +53,7 @@ import de.anomic.kelondro.kelondroBLOBTree;
import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroCloneableIterator;
import de.anomic.kelondro.kelondroException;
import de.anomic.kelondro.kelondroMapObjects;
import de.anomic.kelondro.kelondroMapDataMining;
import de.anomic.kelondro.kelondroNaturalOrder;
import de.anomic.server.serverCodings;
import de.anomic.yacy.yacySeedDB;
@ -63,14 +63,14 @@ public class CrawlProfile {
static HashMap<String, Map<String, DomProfile>> domsCache = new HashMap<String, Map<String, DomProfile>>();
kelondroMapObjects profileTable;
kelondroMapDataMining profileTable;
private File profileTableFile;
public CrawlProfile(File file) {
this.profileTableFile = file;
profileTableFile.getParentFile().mkdirs();
kelondroBLOB dyn = new kelondroBLOBTree(profileTableFile, true, true, yacySeedDB.commonHashLength, 2000, '#', kelondroNaturalOrder.naturalOrder, false, false, true);
profileTable = new kelondroMapObjects(dyn, 500);
profileTable = new kelondroMapDataMining(dyn, 500);
}
public void clear() {
@ -79,7 +79,7 @@ public class CrawlProfile {
if (!(profileTableFile.delete())) throw new RuntimeException("cannot delete crawl profile database");
profileTableFile.getParentFile().mkdirs();
kelondroBLOB dyn = new kelondroBLOBTree(profileTableFile, true, true, yacySeedDB.commonHashLength, 2000, '#', kelondroNaturalOrder.naturalOrder, false, false, true);
profileTable = new kelondroMapObjects(dyn, 500);
profileTable = new kelondroMapDataMining(dyn, 500);
}
public void close() {
@ -142,11 +142,11 @@ public class CrawlProfile {
public entry newEntry(HashMap<String, String> mem) {
entry ne = new entry(mem);
try {
profileTable.set(ne.handle(), ne.map());
profileTable.put(ne.handle(), ne.map());
} catch (kelondroException e) {
clear();
try {
profileTable.set(ne.handle(), ne.map());
profileTable.put(ne.handle(), ne.map());
} catch (IOException ee) {
e.printStackTrace();
System.exit(0);
@ -154,7 +154,7 @@ public class CrawlProfile {
} catch (IOException e) {
clear();
try {
profileTable.set(ne.handle(), ne.map());
profileTable.put(ne.handle(), ne.map());
} catch (IOException ee) {
e.printStackTrace();
System.exit(0);
@ -181,11 +181,11 @@ public class CrawlProfile {
remoteIndexing,
xsstopw, xdstopw, xpstopw);
try {
profileTable.set(ne.handle(), ne.map());
profileTable.put(ne.handle(), ne.map());
} catch (kelondroException e) {
clear();
try {
profileTable.set(ne.handle(), ne.map());
profileTable.put(ne.handle(), ne.map());
} catch (IOException ee) {
e.printStackTrace();
System.exit(0);
@ -193,7 +193,7 @@ public class CrawlProfile {
} catch (IOException e) {
clear();
try {
profileTable.set(ne.handle(), ne.map());
profileTable.put(ne.handle(), ne.map());
} catch (IOException ee) {
e.printStackTrace();
System.exit(0);
@ -210,7 +210,7 @@ public class CrawlProfile {
public void changeEntry(entry e, String propName, String newValue) throws IOException {
e.mem.put(propName, newValue);
profileTable.set(e.handle(), e.mem);
profileTable.put(e.handle(), e.mem);
}
public static class DomProfile {

10
source/de/anomic/crawler/RobotsTxt.java
Unescape View File

@ -65,7 +65,7 @@ import de.anomic.kelondro.kelondroBLOB;
import de.anomic.kelondro.kelondroBLOBHeap;
import de.anomic.kelondro.kelondroBLOBTree;
import de.anomic.kelondro.kelondroException;
import de.anomic.kelondro.kelondroMapObjects;
import de.anomic.kelondro.kelondroMapDataMining;
import de.anomic.kelondro.kelondroNaturalOrder;
import de.anomic.server.serverByteBuffer;
import de.anomic.server.serverFileUtils;
@ -76,7 +76,7 @@ public class RobotsTxt {
public static final String ROBOTS_DB_PATH_SEPARATOR = ";";
kelondroMapObjects robotsTable;
kelondroMapDataMining robotsTable;
private final File robotsTableFile;
public RobotsTxt(File robotsTableFile) {
@ -92,7 +92,7 @@ public class RobotsTxt {
} else {
blob = new kelondroBLOBTree(robotsTableFile, true, true, 256, 512, '_', kelondroNaturalOrder.naturalOrder, false, false, true);
}
robotsTable = new kelondroMapObjects(blob, 100);
robotsTable = new kelondroMapDataMining(blob, 100);
}
private void resetDatabase() {
@ -100,7 +100,7 @@ public class RobotsTxt {
if (robotsTable != null) robotsTable.close();
if (!(robotsTableFile.delete())) throw new RuntimeException("cannot delete robots.txt database");
robotsTableFile.getParentFile().mkdirs();
robotsTable = new kelondroMapObjects(new kelondroBLOBTree(robotsTableFile, true, true, 256, 512, '_', kelondroNaturalOrder.naturalOrder, false, false, true), 100);
robotsTable = new kelondroMapDataMining(new kelondroBLOBTree(robotsTableFile, true, true, 256, 512, '_', kelondroNaturalOrder.naturalOrder, false, false, true), 100);
}
public void clear() throws IOException {
@ -151,7 +151,7 @@ public class RobotsTxt {
private String addEntry(Entry entry) {
// writes a new page and returns key
try {
this.robotsTable.set(entry.hostName, entry.mem);
this.robotsTable.put(entry.hostName, entry.mem);
return entry.hostName;
} catch (IOException e) {
return null;

10
source/de/anomic/data/blogBoard.java
Unescape View File

@ -68,7 +68,7 @@ import org.xml.sax.SAXException;
import de.anomic.kelondro.kelondroBLOBTree;
import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroException;
import de.anomic.kelondro.kelondroMapObjects;
import de.anomic.kelondro.kelondroMapDataMining;
import de.anomic.kelondro.kelondroNaturalOrder;
import de.anomic.server.serverDate;
import de.anomic.server.logging.serverLog;
@ -78,12 +78,12 @@ public class blogBoard {
public static final int keyLength = 64;
private static final int recordSize = 512;
kelondroMapObjects database = null;
kelondroMapDataMining database = null;
public blogBoard(File actpath) {
new File(actpath.getParent()).mkdir();
if (database == null) {
database = new kelondroMapObjects(new kelondroBLOBTree(actpath, true, true, keyLength, recordSize, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 500);
database = new kelondroMapDataMining(new kelondroBLOBTree(actpath, true, true, keyLength, recordSize, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 500);
}
}
public int size() {
@ -128,7 +128,7 @@ public class blogBoard {
*/
public String writeBlogEntry(BlogEntry page) {
try {
database.set(page.key, page.record);
database.put(page.key, page.record);
return page.key;
} catch (IOException e) {
return null;
@ -137,7 +137,7 @@ public class blogBoard {
public BlogEntry readBlogEntry(String key) {
return readBlogEntry(key, database);
}
private BlogEntry readBlogEntry(String key, kelondroMapObjects base) {
private BlogEntry readBlogEntry(String key, kelondroMapDataMining base) {
key = normalize(key);
if (key.length() > keyLength)
key = key.substring(0, keyLength);

10
source/de/anomic/data/blogBoardComments.java
Unescape View File

@ -66,7 +66,7 @@ import org.xml.sax.SAXException;
import de.anomic.kelondro.kelondroBLOBTree;
import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroMapObjects;
import de.anomic.kelondro.kelondroMapDataMining;
import de.anomic.kelondro.kelondroNaturalOrder;
import de.anomic.server.logging.serverLog;
@ -81,11 +81,11 @@ public class blogBoardComments {
static {
SimpleFormatter.setTimeZone(TimeZone.getTimeZone("GMT"));
}
private kelondroMapObjects database = null;
private kelondroMapDataMining database = null;
public blogBoardComments(File actpath) {
new File(actpath.getParent()).mkdir();
if (database == null) {
database = new kelondroMapObjects(new kelondroBLOBTree(actpath, true, true, keyLength, recordSize, '_', kelondroNaturalOrder.naturalOrder, false, false, false), 500);
database = new kelondroMapDataMining(new kelondroBLOBTree(actpath, true, true, keyLength, recordSize, '_', kelondroNaturalOrder.naturalOrder, false, false, false), 500);
}
}
public int size() {
@ -120,7 +120,7 @@ public class blogBoardComments {
public String write(CommentEntry page) {
// writes a new page and returns key
try {
database.set(page.key, page.record);
database.put(page.key, page.record);
return page.key;
} catch (IOException e) {
return null;
@ -130,7 +130,7 @@ public class blogBoardComments {
//System.out.println("DEBUG: read from blogBoardComments");
return read(key, database);
}
private CommentEntry read(String key, kelondroMapObjects base) {
private CommentEntry read(String key, kelondroMapDataMining base) {
key = normalize(key);
if (key.length() > keyLength) key = key.substring(0, keyLength);
HashMap<String, String> record = base.getMap(key);

22
source/de/anomic/data/bookmarksDB.java
Unescape View File

@ -78,9 +78,9 @@ import de.anomic.index.indexWord;
import de.anomic.kelondro.kelondroBLOBTree;
import de.anomic.kelondro.kelondroCloneableIterator;
import de.anomic.kelondro.kelondroException;
import de.anomic.kelondro.kelondroMapObjects;
import de.anomic.kelondro.kelondroMapDataMining;
import de.anomic.kelondro.kelondroNaturalOrder;
import de.anomic.kelondro.kelondroObjects;
import de.anomic.kelondro.kelondroMap;
import de.anomic.server.serverDate;
import de.anomic.server.serverFileUtils;
import de.anomic.server.logging.serverLog;
@ -96,14 +96,14 @@ public class bookmarksDB {
final static int SHOW_ALL = -1;
// bookmarks
kelondroObjects bookmarksTable; // kelondroMap bookmarksTable;
kelondroMap bookmarksTable; // kelondroMap bookmarksTable;
// tags
kelondroMapObjects tagsTable;
kelondroMapDataMining tagsTable;
HashMap<String, Tag> tagCache;
// dates
kelondroMapObjects datesTable;
kelondroMapDataMining datesTable;
// ------------------------------------
@ -115,17 +115,17 @@ public class bookmarksDB {
tagCache=new HashMap<String, Tag>();
bookmarksFile.getParentFile().mkdirs();
//this.bookmarksTable = new kelondroMap(kelondroDyn.open(bookmarksFile, bufferkb * 1024, preloadTime, 12, 256, '_', true, false));
this.bookmarksTable = new kelondroObjects(new kelondroBLOBTree(bookmarksFile, true, true, 12, 256, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 1000);
this.bookmarksTable = new kelondroMap(new kelondroBLOBTree(bookmarksFile, true, true, 12, 256, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 1000);
// tags
tagsFile.getParentFile().mkdirs();
boolean tagsFileExisted = tagsFile.exists();
this.tagsTable = new kelondroMapObjects(new kelondroBLOBTree(tagsFile, true, true, 12, 256, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 500);
this.tagsTable = new kelondroMapDataMining(new kelondroBLOBTree(tagsFile, true, true, 12, 256, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 500);
if (!tagsFileExisted) rebuildTags();
// dates
boolean datesExisted = datesFile.exists();
this.datesTable = new kelondroMapObjects(new kelondroBLOBTree(datesFile, true, true, 20, 256, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 500);
this.datesTable = new kelondroMapDataMining(new kelondroBLOBTree(datesFile, true, true, 20, 256, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 500);
if (!datesExisted) rebuildDates();
}
@ -226,7 +226,7 @@ public class bookmarksDB {
// adding a bookmark to the bookmarksDB
public void saveBookmark(Bookmark bookmark){
try {
bookmarksTable.set(bookmark.getUrlHash(), bookmark.entry);
bookmarksTable.put(bookmark.getUrlHash(), bookmark.entry);
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
@ -362,7 +362,7 @@ public class bookmarksDB {
public void storeTag(Tag tag){
try {
if(tag.size() >0){
bookmarksDB.this.tagsTable.set(tag.getTagHash(), tag.getMap());
bookmarksDB.this.tagsTable.put(tag.getTagHash(), tag.getMap());
}else{
bookmarksDB.this.tagsTable.remove(tag.getTagHash());
}
@ -867,7 +867,7 @@ public class bookmarksDB {
public void setDatesTable(){
try {
if(this.size() >0){
bookmarksDB.this.datesTable.set(getDateString(), mem);
bookmarksDB.this.datesTable.put(getDateString(), mem);
}else{
bookmarksDB.this.datesTable.remove(getDateString());
}

8
source/de/anomic/data/messageBoard.java
Unescape View File

@ -51,7 +51,7 @@ import java.util.TimeZone;
import de.anomic.kelondro.kelondroBLOBTree;
import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroMapObjects;
import de.anomic.kelondro.kelondroMapDataMining;
import de.anomic.kelondro.kelondroNaturalOrder;
public class messageBoard {
@ -66,13 +66,13 @@ public class messageBoard {
SimpleFormatter.setTimeZone(TimeZone.getTimeZone("GMT"));
}
kelondroMapObjects database = null;
kelondroMapDataMining database = null;
private int sn = 0;
public messageBoard(File path) {
new File(path.getParent()).mkdir();
if (database == null) {
database = new kelondroMapObjects(new kelondroBLOBTree(path, true, true, categoryLength + dateFormat.length() + 2, recordSize, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 500);
database = new kelondroMapDataMining(new kelondroBLOBTree(path, true, true, categoryLength + dateFormat.length() + 2, recordSize, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 500);
}
sn = 0;
}
@ -208,7 +208,7 @@ public class messageBoard {
public String write(entry message) {
// writes a message and returns key
try {
database.set(message.key, message.record);
database.put(message.key, message.record);
return message.key;
} catch (IOException e) {
return null;

14
source/de/anomic/data/userDB.java
Unescape View File

@ -59,7 +59,7 @@ import de.anomic.kelondro.kelondroBLOBTree;
import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroCloneableIterator;
import de.anomic.kelondro.kelondroException;
import de.anomic.kelondro.kelondroMapObjects;
import de.anomic.kelondro.kelondroMapDataMining;
import de.anomic.kelondro.kelondroNaturalOrder;
import de.anomic.server.serverCodings;
@ -68,7 +68,7 @@ public final class userDB {
public static final int USERNAME_MAX_LENGTH = 128;
public static final int USERNAME_MIN_LENGTH = 4;
kelondroMapObjects userTable;
kelondroMapDataMining userTable;
private final File userTableFile;
HashMap<String, String> ipUsers = new HashMap<String, String>();
HashMap<String, Object> cookieUsers = new HashMap<String, Object>();
@ -76,7 +76,7 @@ public final class userDB {
public userDB(File userTableFile) {
this.userTableFile = userTableFile;
userTableFile.getParentFile().mkdirs();
this.userTable = new kelondroMapObjects(new kelondroBLOBTree(userTableFile, true, true, 128, 256, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 10);
this.userTable = new kelondroMapDataMining(new kelondroBLOBTree(userTableFile, true, true, 128, 256, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 10);
}
void resetDatabase() {
@ -84,7 +84,7 @@ public final class userDB {
if (userTable != null) userTable.close();
if (!(userTableFile.delete())) throw new RuntimeException("cannot delete user database");
userTableFile.getParentFile().mkdirs();
userTable = new kelondroMapObjects(new kelondroBLOBTree(userTableFile, true, true, 256, 512, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 10);
userTable = new kelondroMapDataMining(new kelondroBLOBTree(userTableFile, true, true, 256, 512, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 10);
}
public void close() {
@ -117,7 +117,7 @@ public final class userDB {
public String addEntry(Entry entry) {
try {
userTable.set(entry.userName,entry.mem);
userTable.put(entry.userName,entry.mem);
return entry.userName;
} catch (IOException e) {
return null;
@ -474,7 +474,7 @@ public final class userDB {
}
try {
userDB.this.userTable.set(getUserName(), this.mem);
userDB.this.userTable.put(getUserName(), this.mem);
} catch(Exception e){
e.printStackTrace();
}
@ -491,7 +491,7 @@ public final class userDB {
public void setProperty(String propName, String newValue) throws IOException {
this.mem.put(propName, newValue);
userDB.this.userTable.set(getUserName(), this.mem);
userDB.this.userTable.put(getUserName(), this.mem);
}
public String getProperty(String propName, String defaultValue) {

16
source/de/anomic/data/wikiBoard.java
Unescape View File

@ -51,7 +51,7 @@ import java.util.TimeZone;
import de.anomic.kelondro.kelondroBLOBTree;
import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroMapObjects;
import de.anomic.kelondro.kelondroMapDataMining;
import de.anomic.kelondro.kelondroNaturalOrder;
public class wikiBoard {
@ -66,18 +66,18 @@ public class wikiBoard {
SimpleFormatter.setTimeZone(TimeZone.getTimeZone("GMT"));
}
kelondroMapObjects datbase = null;
kelondroMapObjects bkpbase = null;
kelondroMapDataMining datbase = null;
kelondroMapDataMining bkpbase = null;
static HashMap<String, String> authors = new HashMap<String, String>();
public wikiBoard(File actpath, File bkppath) {
new File(actpath.getParent()).mkdirs();
if (datbase == null) {
datbase = new kelondroMapObjects(new kelondroBLOBTree(actpath, true, true, keyLength, recordSize, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 500);
datbase = new kelondroMapDataMining(new kelondroBLOBTree(actpath, true, true, keyLength, recordSize, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 500);
}
new File(bkppath.getParent()).mkdirs();
if (bkpbase == null) {
bkpbase = new kelondroMapObjects(new kelondroBLOBTree(bkppath, true, true, keyLength + dateFormat.length(), recordSize, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 500);
bkpbase = new kelondroMapDataMining(new kelondroBLOBTree(bkppath, true, true, keyLength + dateFormat.length(), recordSize, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 500);
}
}
@ -277,9 +277,9 @@ public class wikiBoard {
//System.out.println("key = " + page.key);
//System.out.println("oldDate = " + oldDate);
//System.out.println("record = " + oldEntry.record.toString());
bkpbase.set(page.key + dateString(oldDate), oldEntry.record);
bkpbase.put(page.key + dateString(oldDate), oldEntry.record);
// write the new page
datbase.set(page.key, page.record);
datbase.put(page.key, page.record);
return page.key;
} catch (IOException e) {
return null;
@ -290,7 +290,7 @@ public class wikiBoard {
return read(key, datbase);
}
entry read(String key, kelondroMapObjects base) {
entry read(String key, kelondroMapDataMining base) {
try {
key = normalize(key);
if (key.length() > keyLength) key = key.substring(0, keyLength);

77
source/de/anomic/kelondro/kelondroObjects.java → source/de/anomic/kelondro/kelondroMap.java
Unescape View File

@ -37,7 +37,7 @@ import java.util.Map;
import de.anomic.server.serverDate;
public class kelondroObjects {
public class kelondroMap {
private kelondroBLOB blob;
private kelondroMScoreCluster<String> cacheScore;
@ -45,15 +45,26 @@ public class kelondroObjects {
private long startup;
private int cachesize;
public kelondroObjects(kelondroBLOB blob, int cachesize) {
public kelondroMap(kelondroBLOB blob, int cachesize) {
this.blob = blob;
this.cache = new HashMap<String, HashMap<String, String>>();
this.cacheScore = new kelondroMScoreCluster<String>();
this.startup = System.currentTimeMillis();
this.cachesize = cachesize;
}
/**
* ask for the length of the primary key
* @return the length of the key
*/
public int keylength() {
return this.blob.keylength();
}
/**
* clears the content of the database
* @throws IOException
*/
public void clear() throws IOException {
this.blob.clear();
this.cache = new HashMap<String, HashMap<String, String>>();
@ -91,7 +102,13 @@ public class kelondroObjects {
return map;
}
public synchronized void set(String key, HashMap<String, String> newMap) throws IOException {
/**
* write a whole byte array as Map to the table
* @param key the primary key
* @param newMap
* @throws IOException
*/
public synchronized void put(String key, HashMap<String, String> newMap) throws IOException {
assert (key != null);
assert (key.length() > 0);
assert (newMap != null);
@ -109,6 +126,11 @@ public class kelondroObjects {
cache.put(key, newMap);
}
/**
* remove a Map
* @param key the primary key
* @throws IOException
*/
public synchronized void remove(String key) throws IOException {
// update elementCount
if (key == null) return;
@ -121,7 +143,26 @@ public class kelondroObjects {
// remove from file
blob.remove(key.getBytes());
}
/**
* check if a specific key is in the database
* @param key the primary key
* @return
* @throws IOException
*/
public boolean has(String key) throws IOException {
assert key != null;
if (cache == null) return false; // case may appear during shutdown
while (key.length() < blob.keylength()) key += "_";
return this.blob.has(key.getBytes());
}
/**
* retrieve the whole Map from the table
* @param key the primary key
* @return
* @throws IOException
*/
public synchronized HashMap<String, String> get(final String key) throws IOException {
if (key == null) return null;
return get(key, true);
@ -167,11 +208,29 @@ public class kelondroObjects {
}
}
/**
* iterator over all keys
* @param up
* @param rotating
* @return
* @throws IOException
*/
public synchronized kelondroCloneableIterator<byte[]> keys(final boolean up, final boolean rotating) throws IOException {
// simple enumeration of key names without special ordering
return blob.keys(up, rotating);
}
/**
* iterate over all keys
* @param up
* @param firstKey
* @return
* @throws IOException
*/
public kelondroCloneableIterator<byte[]> keys(boolean up, byte[] firstKey) throws IOException {
return keys(up, false, firstKey, null);
}
public synchronized kelondroCloneableIterator<byte[]> keys(final boolean up, final boolean rotating, final byte[] firstKey, final byte[] secondKey) throws IOException {
// simple enumeration of key names without special ordering
kelondroCloneableIterator<byte[]> i = blob.keys(up, firstKey);
@ -187,10 +246,17 @@ public class kelondroObjects {
return new objectIterator(keys(up, rotating, firstKey, secondKey));
}
/**
* ask for the number of entries
* @return the number of entries in the table
*/
public synchronized int size() {
return blob.size();
}
/**
* close the Map table
*/
public void close() {
// finish queue
//writeWorker.terminate(true);
@ -238,4 +304,5 @@ public class kelondroObjects {
throw new UnsupportedOperationException();
}
} // class mapIterator
}

10
source/de/anomic/kelondro/kelondroMapObjects.java → source/de/anomic/kelondro/kelondroMapDataMining.java
Unescape View File

@ -34,19 +34,19 @@ import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
public class kelondroMapObjects extends kelondroObjects {
public class kelondroMapDataMining extends kelondroMap {
private String[] sortfields, longaccfields, doubleaccfields;
private HashMap<String, kelondroMScoreCluster<String>> sortClusterMap; // a String-kelondroMScoreCluster - relation
private HashMap<String, Object> accMap; // to store accumulations of specific fields
private int elementCount;
public kelondroMapObjects(kelondroBLOB dyn, int cachesize) {
public kelondroMapDataMining(kelondroBLOB dyn, int cachesize) {
this(dyn, cachesize, null, null, null, null, null);
}
@SuppressWarnings({ "unchecked", "null" })
public kelondroMapObjects(kelondroBLOB dyn, int cachesize, String[] sortfields, String[] longaccfields, String[] doubleaccfields, Method externalInitializer, Object externalHandler) {
public kelondroMapDataMining(kelondroBLOB dyn, int cachesize, String[] sortfields, String[] longaccfields, String[] doubleaccfields, Method externalInitializer, Object externalHandler) {
super(dyn, cachesize);
// create fast ordering clusters and acc fields
@ -172,7 +172,7 @@ public class kelondroMapObjects extends kelondroObjects {
this.elementCount = 0;
}
public synchronized void set(String key, HashMap<String, String> newMap) throws IOException {
public synchronized void put(String key, HashMap<String, String> newMap) throws IOException {
assert (key != null);
assert (key.length() > 0);
assert (newMap != null);
@ -189,7 +189,7 @@ public class kelondroMapObjects extends kelondroObjects {
}
}
super.set(key, newMap);
super.put(key, newMap);
// update sortCluster
if (sortClusterMap != null) updateSortCluster(key, newMap);

27
source/de/anomic/plasma/plasmaHTCache.java
Unescape View File

@ -22,25 +22,6 @@
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
//
// Using this software in any meaning (reading, learning, copying, compiling,
// running) means that you agree that the Author(s) is (are) not responsible
// for cost, loss of data or any harm that may be caused directly or indirectly
// by usage of this softare or this documentation. The usage of this software
// is on your own risk. The installation and usage (starting/running) of this
// software may allow other people or application to access your computer and
// any attached devices and is highly dependent on the configuration of the
// software which must be done by the user of the software; the author(s) is
// (are) also not responsible for proper configuration and usage of the
// software, even if provoked by documentation provided together with
// the software.
//
// Any changes to this file according to the GPL as documented in the file
// gpl.txt aside this file in the shipment you received can be done to the
// lines that follows this copyright notice here, but changes must not be
// done inside the copyright notive above. A re-distribution must contain
// the intact and unchanged copyright notice.
// Contributions and changes to the program code must be marked as such.
/*
Class documentation:
@ -76,7 +57,7 @@ import de.anomic.kelondro.kelondroBLOBHeap;
import de.anomic.kelondro.kelondroBLOBTree;
import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroMScoreCluster;
import de.anomic.kelondro.kelondroMapObjects;
import de.anomic.kelondro.kelondroMapDataMining;
import de.anomic.plasma.cache.IResourceInfo;
import de.anomic.plasma.cache.ResourceInfoFactory;
import de.anomic.plasma.cache.UnsupportedProtocolException;
@ -99,7 +80,7 @@ public final class plasmaHTCache {
private static final int stackLimit = 150; // if we exceed that limit, we do not check idle
public static final long oneday = 1000 * 60 * 60 * 24; // milliseconds of a day
static kelondroMapObjects responseHeaderDB = null;
private static kelondroMapDataMining responseHeaderDB = null;
private static final ConcurrentLinkedQueue<Entry> cacheStack = new ConcurrentLinkedQueue<Entry>();
private static final ConcurrentHashMap<String, File> cacheAge = new ConcurrentHashMap<String, File>(); // a <date+hash, cache-path> - relation
public static long curCacheSize = 0;
@ -291,7 +272,7 @@ public final class plasmaHTCache {
} else {
blob = new kelondroBLOBTree(dbfile, true, true, yacySeedDB.commonHashLength, 150, '#', kelondroBase64Order.enhancedCoder, false, false, true);
}
responseHeaderDB = new kelondroMapObjects(blob, 500);
responseHeaderDB = new kelondroMapDataMining(blob, 500);
}
private static void deleteOldHTCache(File directory) {
@ -1061,7 +1042,7 @@ public final class plasmaHTCache {
hm.put("@@URL", this.url.toNormalform(false, false));
hm.put("@@DEPTH", Integer.toString(this.depth));
if (this.initiator != null) hm.put("@@INITIATOR", this.initiator);
responseHeaderDB.set(this.url.hash(), hm);
responseHeaderDB.put(this.url.hash(), hm);
} catch (Exception e) {
resetResponseHeaderDB();
return false;

34
source/de/anomic/yacy/yacySeedDB.java
Unescape View File

@ -70,7 +70,7 @@ import de.anomic.kelondro.kelondroBLOBTree;
import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroException;
import de.anomic.kelondro.kelondroMScoreCluster;
import de.anomic.kelondro.kelondroMapObjects;
import de.anomic.kelondro.kelondroMapDataMining;
import de.anomic.plasma.plasmaHTCache;
import de.anomic.server.serverCore;
import de.anomic.server.serverDate;
@ -107,7 +107,7 @@ public final class yacySeedDB implements httpdAlternativeDomainNames {
// class objects
protected File seedActiveDBFile, seedPassiveDBFile, seedPotentialDBFile;
protected File myOwnSeedFile;
protected kelondroMapObjects seedActiveDB, seedPassiveDB, seedPotentialDB;
protected kelondroMapDataMining seedActiveDB, seedPassiveDB, seedPotentialDB;
public int lastSeedUpload_seedDBSize = 0;
public long lastSeedUpload_timeStamp = System.currentTimeMillis();
@ -238,7 +238,7 @@ public final class yacySeedDB implements httpdAlternativeDomainNames {
}
@SuppressWarnings("unchecked")
private synchronized kelondroMapObjects openSeedTable(File seedDBFile) {
private synchronized kelondroMapDataMining openSeedTable(File seedDBFile) {
final boolean usetree = false;
new File(seedDBFile.getParent()).mkdirs();
Class[] args;
@ -259,15 +259,15 @@ public final class yacySeedDB implements httpdAlternativeDomainNames {
initializeHandlerMethod = null;
}
try {
return new kelondroMapObjects(new kelondroBLOBTree(seedDBFile, true, true, commonHashLength, 480, '#', kelondroBase64Order.enhancedCoder, usetree, false, true), 500, sortFields, longaccFields, doubleaccFields, initializeHandlerMethod, this);
return new kelondroMapDataMining(new kelondroBLOBTree(seedDBFile, true, true, commonHashLength, 480, '#', kelondroBase64Order.enhancedCoder, usetree, false, true), 500, sortFields, longaccFields, doubleaccFields, initializeHandlerMethod, this);
} catch (Exception e) {
// try again
kelondroBLOBTree.delete(seedDBFile);
return new kelondroMapObjects(new kelondroBLOBTree(seedDBFile, true, true, commonHashLength, 480, '#', kelondroBase64Order.enhancedCoder, usetree, false, true), 500, sortFields, longaccFields, doubleaccFields, initializeHandlerMethod, this);
return new kelondroMapDataMining(new kelondroBLOBTree(seedDBFile, true, true, commonHashLength, 480, '#', kelondroBase64Order.enhancedCoder, usetree, false, true), 500, sortFields, longaccFields, doubleaccFields, initializeHandlerMethod, this);
}
}
protected synchronized kelondroMapObjects resetSeedTable(kelondroMapObjects seedDB, File seedDBFile) {
protected synchronized kelondroMapDataMining resetSeedTable(kelondroMapDataMining seedDB, File seedDBFile) {
// this is an emergency function that should only be used if any problem with the
// seed.db is detected
yacyCore.log.logFine("seed-db " + seedDBFile.toString() + " reset (on-the-fly)");
@ -459,7 +459,7 @@ public final class yacySeedDB implements httpdAlternativeDomainNames {
nameLookupCache.put(seed.getName(), seed);
HashMap<String, String> seedPropMap = seed.getMap();
synchronized(seedPropMap) {
seedActiveDB.set(seed.hash, seedPropMap);
seedActiveDB.put(seed.hash, seedPropMap);
}
seedPassiveDB.remove(seed.hash);
seedPotentialDB.remove(seed.hash);
@ -486,7 +486,7 @@ public final class yacySeedDB implements httpdAlternativeDomainNames {
try {
HashMap<String, String> seedPropMap = seed.getMap();
synchronized(seedPropMap) {
seedPassiveDB.set(seed.hash, seedPropMap);
seedPassiveDB.put(seed.hash, seedPropMap);
}
} catch (IOException e) {
yacyCore.log.logSevere("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
@ -512,7 +512,7 @@ public final class yacySeedDB implements httpdAlternativeDomainNames {
try {
HashMap<String, String> seedPropMap = seed.getMap();
synchronized(seedPropMap) {
seedPotentialDB.set(seed.hash, seedPropMap);
seedPotentialDB.put(seed.hash, seedPropMap);
}
} catch (IOException e) {
yacyCore.log.logSevere("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
@ -564,7 +564,7 @@ public final class yacySeedDB implements httpdAlternativeDomainNames {
}
}
private yacySeed get(String hash, kelondroMapObjects database) {
private yacySeed get(String hash, kelondroMapDataMining database) {
if (hash == null) return null;
if ((this.mySeed != null) && (hash.equals(mySeed.hash))) return mySeed;
HashMap<String, String> entry = database.getMap(hash);
@ -599,13 +599,13 @@ public final class yacySeedDB implements httpdAlternativeDomainNames {
}
yacySeed s = get(hash, seedActiveDB);
if (s != null) try { seedActiveDB.set(hash, seed.getMap()); return;} catch (IOException e) {}
if (s != null) try { seedActiveDB.put(hash, seed.getMap()); return;} catch (IOException e) {}
s = get(hash, seedPassiveDB);
if (s != null) try { seedPassiveDB.set(hash, seed.getMap()); return;} catch (IOException e) {}
if (s != null) try { seedPassiveDB.put(hash, seed.getMap()); return;} catch (IOException e) {}
s = get(hash, seedPotentialDB);
if (s != null) try { seedPotentialDB.set(hash, seed.getMap()); return;} catch (IOException e) {}
if (s != null) try { seedPotentialDB.put(hash, seed.getMap()); return;} catch (IOException e) {}
}
public yacySeed lookupByName(String peerName) {
@ -959,12 +959,12 @@ public final class yacySeedDB implements httpdAlternativeDomainNames {
class seedEnum implements Iterator<yacySeed> {
kelondroMapObjects.mapIterator it;
kelondroMapDataMining.mapIterator it;
yacySeed nextSeed;
kelondroMapObjects database;
kelondroMapDataMining database;
float minVersion;
public seedEnum(boolean up, boolean rot, byte[] firstKey, byte[] secondKey, kelondroMapObjects database, float minVersion) {
public seedEnum(boolean up, boolean rot, byte[] firstKey, byte[] secondKey, kelondroMapDataMining database, float minVersion) {
this.database = database;
this.minVersion = minVersion;
try {
@ -989,7 +989,7 @@ public final class yacySeedDB implements httpdAlternativeDomainNames {
}
}
public seedEnum(boolean up, String field, kelondroMapObjects database) {
public seedEnum(boolean up, String field, kelondroMapDataMining database) {
this.database = database;
try {
it = database.maps(up, field);

6
source/yacy.java
Unescape View File

@ -81,7 +81,7 @@ import de.anomic.index.indexWord;
import de.anomic.kelondro.kelondroBLOBTree;
import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroMScoreCluster;
import de.anomic.kelondro.kelondroMapObjects;
import de.anomic.kelondro.kelondroMapDataMining;
import de.anomic.kelondro.kelondroRowCollection;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.plasma.plasmaWordIndex;
@ -900,9 +900,9 @@ public final class yacy {
String[] dbFileNames = {"seed.new.db","seed.old.db","seed.pot.db"};
for (int i=0; i < dbFileNames.length; i++) {
File dbFile = new File(yacyDBPath,dbFileNames[i]);
kelondroMapObjects db = new kelondroMapObjects(new kelondroBLOBTree(dbFile, true, true, yacySeedDB.commonHashLength, 480, '#', kelondroBase64Order.enhancedCoder, true, false, true), 500, yacySeedDB.sortFields, yacySeedDB.longaccFields, yacySeedDB.doubleaccFields, null, null);
kelondroMapDataMining db = new kelondroMapDataMining(new kelondroBLOBTree(dbFile, true, true, yacySeedDB.commonHashLength, 480, '#', kelondroBase64Order.enhancedCoder, true, false, true), 500, yacySeedDB.sortFields, yacySeedDB.longaccFields, yacySeedDB.doubleaccFields, null, null);
kelondroMapObjects.mapIterator it;
kelondroMapDataMining.mapIterator it;
it = db.maps(true, false);
while (it.hasNext()) {
Map<String, String> dna = it.next();

Write Preview
Loading…
Cancel
Save