- fix of bug in iterator in kelondroBLOBHeap which caused bug in crawl profile listing

- some refactoring of classes that use kelondroMap (Map instead of HashMap)

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5262 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 17 years ago
parent 2d65887723
commit 6fb865fbdc

Binary file not shown.

@ -121,7 +121,7 @@ public class CrawlProfile {
} catch (final IOException e) {}
}
public entry newEntry(final HashMap<String, String> mem) {
public entry newEntry(final Map<String, String> mem) {
final entry ne = new entry(mem);
try {
profileTable.put(ne.handle(), ne.map());
@ -193,7 +193,7 @@ public class CrawlProfile {
}
public entry getEntry(final String handle) {
HashMap<String, String> m;
Map<String, String> m;
try {
m = profileTable.get(handle);
} catch (final IOException e) {
@ -252,7 +252,7 @@ public class CrawlProfile {
public static final String XDSTOPW = "xdstopw";
public static final String XPSTOPW = "xpstopw";
HashMap<String, String> mem;
Map<String, String> mem;
private Map<String, DomProfile> doms;
public entry(final String name, final yacyURL startURL, final String generalFilter, final String specificFilter,
@ -299,13 +299,13 @@ public class CrawlProfile {
return str.toString();
}
public entry(final HashMap<String, String> mem) {
public entry(final Map<String, String> mem) {
this.mem = mem;
this.doms = domsCache.get(this.mem.get(HANDLE));
if (this.doms == null) this.doms = new HashMap<String, DomProfile>();
}
public HashMap<String, String> map() {
public Map<String, String> map() {
return mem;
}
public String handle() {

@ -177,7 +177,9 @@ public class NoticeURLImporter extends AbstractImporter implements Importer {
if (sourceEntry != null) {
this.profileCount++;
this.importProfileHandleCache.add(profileHandle);
this.activeCrawls.newEntry((HashMap<String, String>) sourceEntry.map().clone());
HashMap<String, String> mapclone = new HashMap<String, String>();
mapclone.putAll(sourceEntry.map());
this.activeCrawls.newEntry((HashMap<String, String>) mapclone);
} else {
this.log.logWarning("Profile '" + profileHandle + "' of url entry '" + nextHash + "' unknown.");
continue;

@ -37,6 +37,7 @@ import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import de.anomic.http.HttpClient;
@ -105,7 +106,7 @@ public class RobotsTxt {
// this method will always return a non-null value
Entry robotsTxt4Host = null;
try {
final HashMap<String, String> record = this.robotsTable.get(urlHostPort);
final Map<String, String> record = this.robotsTable.get(urlHostPort);
if (record != null) robotsTxt4Host = new Entry(urlHostPort, record);
} catch (final kelondroException e) {
resetDatabase();
@ -134,7 +135,7 @@ public class RobotsTxt {
// check the robots table again for all threads that come here because they waited for another one
// to complete a download
try {
final HashMap<String, String> record = this.robotsTable.get(urlHostPort);
final Map<String, String> record = this.robotsTable.get(urlHostPort);
if (record != null) robotsTxt4Host = new Entry(urlHostPort, record);
} catch (final kelondroException e) {
resetDatabase();
@ -261,11 +262,11 @@ public class RobotsTxt {
public static final String CRAWL_DELAY_MILLIS = "crawlDelayMillis";
// this is a simple record structure that holds all properties of a single crawl start
HashMap<String, String> mem;
Map<String, String> mem;
private LinkedList<String> allowPathList, denyPathList;
String hostName;
public Entry(final String hostName, final HashMap<String, String> mem) {
public Entry(final String hostName, final Map<String, String> mem) {
this.hostName = hostName.toLowerCase();
this.mem = mem;

@ -35,6 +35,7 @@ import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
import java.util.TreeSet;
import javax.xml.parsers.DocumentBuilder;
@ -121,7 +122,7 @@ public class blogBoard {
private BlogEntry readBlogEntry(String key, final kelondroMap base) {
key = normalize(key);
if (key.length() > keyLength) key = key.substring(0, keyLength);
HashMap<String, String> record;
Map<String, String> record;
try {
record = base.get(key);
} catch (final IOException e) {
@ -307,7 +308,7 @@ public class blogBoard {
public class BlogEntry {
String key;
HashMap<String, String> record;
Map<String, String> record;
public BlogEntry(final String nkey, final byte[] subject, final byte[] author, final String ip, final Date date, final byte[] page, final ArrayList<String> comments, final String commentMode) {
record = new HashMap<String, String>();
@ -325,7 +326,7 @@ public class blogBoard {
wikiBoard.setAuthor(ip, new String(author));
}
BlogEntry(final String key, final HashMap<String, String> record) {
BlogEntry(final String key, final Map<String, String> record) {
this.key = key;
this.record = record;
if (this.record.get("comments")==null) this.record.put("comments", listManager.collection2string(new ArrayList<String>()));

@ -34,6 +34,7 @@ import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.TimeZone;
import javax.xml.parsers.DocumentBuilder;
@ -114,7 +115,7 @@ public class blogBoardComments {
private CommentEntry read(String key, final kelondroMap base) {
key = normalize(key);
if (key.length() > keyLength) key = key.substring(0, keyLength);
HashMap<String, String> record;
Map<String, String> record;
try {
record = base.get(key);
} catch (final IOException e) {
@ -211,7 +212,7 @@ public class blogBoardComments {
public static class CommentEntry {
String key;
HashMap<String, String> record;
Map<String, String> record;
public CommentEntry(final String nkey, final byte[] subject, final byte[] author, final String ip, final Date date, final byte[] page) {
record = new HashMap<String, String>();
@ -226,7 +227,7 @@ public class blogBoardComments {
wikiBoard.setAuthor(ip, new String(author));
}
CommentEntry(final String key, final HashMap<String, String> record) {
CommentEntry(final String key, final Map<String, String> record) {
this.key = key;
this.record = record;
if (this.record.get("comments")==null) this.record.put("comments", listManager.collection2string(new ArrayList<String>()));

@ -410,7 +410,7 @@ public class bookmarksDB {
public Bookmark getBookmark(final String urlHash){
try {
final HashMap<String, String> map = bookmarksTable.get(urlHash);
final Map<String, String> map = bookmarksTable.get(urlHash);
if (map == null) return null;
return new Bookmark(map);
} catch (final IOException e) {
@ -505,7 +505,7 @@ public class bookmarksDB {
* @param hash an object of type String, containing a tagHash
*/
private Tag loadTag(final String hash){
HashMap<String, String> map;
Map<String, String> map;
Tag ret=null;
try {
map = tagsTable.get(hash);
@ -693,9 +693,9 @@ public class bookmarksDB {
// ---------------------------------------
public bookmarksDate getDate(final String date){
HashMap<String, String> map;
Map<String, String> map;
try {
map=datesTable.get(date);
map = datesTable.get(date);
} catch (final IOException e) {
map = null;
}
@ -913,10 +913,10 @@ public class bookmarksDB {
public static final String URL_HASHES="urlHashes";
public static final String TAG_NAME="tagName";
private final String tagHash;
private final HashMap<String, String> mem;
private final Map<String, String> mem;
private Set<String> urlHashes;
public Tag(final String hash, final HashMap<String, String> map){
public Tag(final String hash, final Map<String, String> map){
tagHash=hash;
mem=map;
if(mem.containsKey(URL_HASHES))
@ -938,7 +938,7 @@ public class bookmarksDB {
urlHashes=new HashSet<String>();
mem.put(TAG_NAME, name);
}
public HashMap<String, String> getMap(){
public Map<String, String> getMap(){
mem.put(URL_HASHES, listManager.collection2string(this.urlHashes));
return mem;
}
@ -993,7 +993,7 @@ public class bookmarksDB {
*/
public class bookmarksDate{
public static final String URL_HASHES="urlHashes";
private final HashMap<String, String> mem;
private final Map<String, String> mem;
String date;
public bookmarksDate(final String mydate){
@ -1003,7 +1003,7 @@ public class bookmarksDB {
mem.put(URL_HASHES, "");
}
public bookmarksDate(final String mydate, final HashMap<String, String> map){
public bookmarksDate(final String mydate, final Map<String, String> map){
//round to seconds, but store as milliseconds (java timestamp)
date=String.valueOf((Long.parseLong(mydate)/1000)*1000);
mem=map;
@ -1075,9 +1075,9 @@ public class bookmarksDB {
private String urlHash;
private Set<String> tags;
private long timestamp;
HashMap<String, String> entry;
Map<String, String> entry;
public Bookmark(final String urlHash, final HashMap<String, String> map) {
public Bookmark(final String urlHash, final Map<String, String> map) {
this.entry = map;
this.urlHash=urlHash;
if(map.containsKey(BOOKMARK_TAGS))
@ -1129,7 +1129,7 @@ public class bookmarksDB {
timestamp=System.currentTimeMillis();
}
public Bookmark(final HashMap<String, String> map) throws MalformedURLException {
public Bookmark(final Map<String, String> map) throws MalformedURLException {
this((new yacyURL(map.get(BOOKMARK_URL), null)).hash(), map);
}

@ -28,6 +28,7 @@ import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.TimeZone;
import de.anomic.kelondro.kelondroBLOBTree;
@ -90,7 +91,7 @@ public class messageBoard {
public class entry {
String key; // composed by category and date
HashMap<String, String> record; // contains author, target hash, subject and message
Map<String, String> record; // contains author, target hash, subject and message
public entry(final String category,
String authorName, String authorHash,
@ -118,7 +119,7 @@ public class messageBoard {
record.put("read", "false");
}
entry(final String key, final HashMap<String, String> record) {
entry(final String key, final Map<String, String> record) {
this.key = key;
this.record = record;
}
@ -197,7 +198,7 @@ public class messageBoard {
}
public entry read(final String key) {
HashMap<String, String> record;
Map<String, String> record;
try {
record = database.get(key);
} catch (final IOException e) {

@ -86,7 +86,7 @@ public final class userDB {
if(userName.length()>128){
userName=userName.substring(0, 127);
}
HashMap<String, String> record;
Map<String, String> record;
try {
record = userTable.get(userName);
} catch (final IOException e) {
@ -312,11 +312,11 @@ public final class userDB {
public static final int PROXY_TIMELIMIT_REACHED = 3;
// this is a simple record structure that hold all properties of a user
HashMap<String, String> mem;
Map<String, String> mem;
String userName;
private final Calendar oldDate, newDate;
public Entry(final String userName, final HashMap<String, String> mem) throws IllegalArgumentException {
public Entry(final String userName, final Map<String, String> mem) throws IllegalArgumentException {
if ((userName == null) || (userName.length() == 0))
throw new IllegalArgumentException("Username needed.");
if(userName.length()>128){

@ -28,6 +28,7 @@ import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.TimeZone;
import de.anomic.kelondro.kelondroBLOBTree;
@ -116,7 +117,7 @@ public class wikiBoard {
public class entry {
String key;
HashMap<String, String> record;
Map<String, String> record;
public entry(final String subject, String author, String ip, String reason, final byte[] page) throws IOException {
record = new HashMap<String, String>();
@ -137,7 +138,7 @@ public class wikiBoard {
//System.out.println("DEBUG: setting author " + author + " for ip = " + ip + ", authors = " + authors.toString());
}
entry(final String key, final HashMap<String, String> record) {
entry(final String key, final Map<String, String> record) {
this.key = key;
this.record = record;
}
@ -275,7 +276,7 @@ public class wikiBoard {
try {
key = normalize(key);
if (key.length() > keyLength) key = key.substring(0, keyLength);
final HashMap<String, String> record = base.get(key);
final Map<String, String> record = base.get(key);
if (record == null) return newEntry(key, "anonymous", "127.0.0.1", "New Page", "".getBytes());
return new entry(key, record);
} catch (final IOException e) {

@ -37,11 +37,11 @@ import de.anomic.server.logging.serverLog;
public final class kelondroBLOBHeap implements kelondroBLOB {
private kelondroBytesLongMap index; // key/seek relation for used records
private ArrayList<gap> free; // list of {size, seek} pairs denoting space and position of free records
private final File heapFile; // the file of the heap
private final kelondroByteOrder ordering; // the ordering on keys
private RandomAccessFile file; // a random access to the file
private kelondroBytesLongMap index; // key/seek relation for used records
private ArrayList<gap> free; // list of {size, seek} pairs denoting space and position of free records
private final File heapFile; // the file of the heap
private final kelondroByteOrder ordering; // the ordering on keys
private RandomAccessFile file; // a random access to the file
public static class gap {
public long seek;
@ -436,7 +436,7 @@ public final class kelondroBLOBHeap implements kelondroBLOB {
* @throws IOException
*/
public synchronized kelondroCloneableIterator<byte[]> keys(final boolean up, final boolean rotating) throws IOException {
return new kelondroRotateIterator<byte[]>(this.index.keys(up, null), null, 1);
return new kelondroRotateIterator<byte[]>(this.index.keys(up, null), null, this.index.size());
}
/**
@ -459,6 +459,15 @@ public final class kelondroBLOBHeap implements kelondroBLOB {
heap.put("aaaaaaaaaaab".getBytes(), "vier fuenf sechs".getBytes());
heap.put("aaaaaaaaaaac".getBytes(), "sieben acht neun".getBytes());
heap.put("aaaaaaaaaaad".getBytes(), "zehn elf zwoelf".getBytes());
// iterate over keys
Iterator<byte[]> i = heap.index.keys(true, null);
while (i.hasNext()) {
System.out.println("key_a: " + new String(i.next()));
}
i = heap.keys(true, false);
while (i.hasNext()) {
System.out.println("key_b: " + new String(i.next()));
}
heap.remove("aaaaaaaaaaab".getBytes());
heap.remove("aaaaaaaaaaac".getBytes());
heap.close();
@ -466,5 +475,5 @@ public final class kelondroBLOBHeap implements kelondroBLOB {
e.printStackTrace();
}
}
}

@ -29,6 +29,7 @@ package de.anomic.kelondro;
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.HashMap;
@ -41,13 +42,13 @@ public class kelondroMap {
private final kelondroBLOB blob;
private kelondroMScoreCluster<String> cacheScore;
private HashMap<String, HashMap<String, String>> cache;
private HashMap<String, Map<String, String>> cache;
private final long startup;
private final int cachesize;
public kelondroMap(final kelondroBLOB blob, final int cachesize) {
this.blob = blob;
this.cache = new HashMap<String, HashMap<String, String>>();
this.cache = new HashMap<String, Map<String, String>>();
this.cacheScore = new kelondroMScoreCluster<String>();
this.startup = System.currentTimeMillis();
this.cachesize = cachesize;
@ -67,7 +68,7 @@ public class kelondroMap {
*/
public void clear() throws IOException {
this.blob.clear();
this.cache = new HashMap<String, HashMap<String, String>>();
this.cache = new HashMap<String, Map<String, String>>();
this.cacheScore = new kelondroMScoreCluster<String>();
}
@ -108,7 +109,7 @@ public class kelondroMap {
* @param newMap
* @throws IOException
*/
public synchronized void put(String key, final HashMap<String, String> newMap) throws IOException {
public synchronized void put(String key, final Map<String, String> newMap) throws IOException {
assert (key != null);
assert (key.length() > 0);
assert (newMap != null);
@ -163,18 +164,18 @@ public class kelondroMap {
* @return
* @throws IOException
*/
public synchronized HashMap<String, String> get(final String key) throws IOException {
public synchronized Map<String, String> get(final String key) throws IOException {
if (key == null) return null;
return get(key, true);
}
protected synchronized HashMap<String, String> get(String key, final boolean storeCache) throws IOException {
protected synchronized Map<String, String> get(String key, final boolean storeCache) throws IOException {
// load map from cache
assert key != null;
if (cache == null) return null; // case may appear during shutdown
while (key.length() < blob.keylength()) key += "_";
HashMap<String, String> map = cache.get(key);
Map<String, String> map = cache.get(key);
if (map != null) return map;
// load map from kra
@ -269,7 +270,7 @@ public class kelondroMap {
blob.close();
}
public class objectIterator implements Iterator<HashMap<String, String>> {
public class objectIterator implements Iterator<Map<String, String>> {
// enumerates Map-Type elements
// the key is also included in every map that is returned; it's key is 'key'
@ -285,14 +286,14 @@ public class kelondroMap {
return (!(finish)) && (keyIterator.hasNext());
}
public HashMap<String, String> next() {
public Map<String, String> next() {
final byte[] nextKey = keyIterator.next();
if (nextKey == null) {
finish = true;
return null;
}
try {
final HashMap<String, String> obj = get(new String(nextKey));
final Map<String, String> obj = get(new String(nextKey));
if (obj == null) throw new kelondroException("no more elements available");
return obj;
} catch (final IOException e) {
@ -306,4 +307,30 @@ public class kelondroMap {
}
} // class mapIterator
public static void main(String[] args) {
// test the class
File f = new File("maptest");
if (f.exists()) f.delete();
try {
// make a blob
kelondroBLOB blob = new kelondroBLOBHeap(f, 12, kelondroNaturalOrder.naturalOrder);
// make map
kelondroMap map = new kelondroMap(blob, 1024);
// put some values into the map
HashMap<String, String> m = new HashMap<String, String>();
m.put("k", "000"); map.put("123", m);
m.put("k", "111"); map.put("456", m);
m.put("k", "222"); map.put("789", m);
// iterate over keys
Iterator<byte[]> i = map.keys(true, false);
while (i.hasNext()) {
System.out.println("key: " + new String(i.next()));
}
// clean up
map.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}

@ -345,13 +345,13 @@ public class kelondroMapDataMining extends kelondroMap {
super.close();
}
public class mapIterator implements Iterator<HashMap<String, String>> {
public class mapIterator implements Iterator<Map<String, String>> {
// enumerates Map-Type elements
// the key is also included in every map that is returned; it's key is 'key'
Iterator<byte[]> keyIterator;
boolean finish;
HashMap<String, String> n;
Map<String, String> n;
public mapIterator(final Iterator<byte[]> keyIterator) {
this.keyIterator = keyIterator;
@ -363,17 +363,17 @@ public class kelondroMapDataMining extends kelondroMap {
return this.n != null;
}
public HashMap<String, String> next() {
final HashMap<String, String> n1 = n;
public Map<String, String> next() {
final Map<String, String> n1 = n;
n = next0();
return n1;
}
private HashMap<String, String> next0() {
private Map<String, String> next0() {
if (finish) return null;
if (keyIterator == null) return null;
String nextKey;
HashMap<String, String> map;
Map<String, String> map;
while (keyIterator.hasNext()) {
nextKey = new String(keyIterator.next());
if (nextKey == null) {

@ -50,6 +50,7 @@ import java.net.URL;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.Random;
import java.util.Set;
import java.util.TreeMap;
@ -161,12 +162,12 @@ public class yacySeed implements Cloneable {
/** the peer-hash */
public String hash;
/** a set of identity founding values, eg. IP, name of the peer, YaCy-version, ...*/
private final HashMap<String, String> dna;
private final Map<String, String> dna;
public int available;
public int selectscore = -1; // only for debugging
public String alternativeIP = null;
public yacySeed(final String theHash, final HashMap<String, String> theDna) {
public yacySeed(final String theHash, final Map<String, String> theDna) {
// create a seed with a pre-defined hash map
this.hash = theHash;
this.dna = theDna;
@ -351,7 +352,7 @@ public class yacySeed implements Cloneable {
}
/** @return the DNA-map of this peer */
public final HashMap<String, String> getMap() {
public final Map<String, String> getMap() {
return this.dna;
}

@ -443,7 +443,7 @@ public final class yacySeedDB implements httpdAlternativeDomainNames {
//seed.put(yacySeed.LASTSEEN, yacyCore.shortFormatter.format(new Date(yacyCore.universalTime())));
try {
nameLookupCache.put(seed.getName(), seed);
final HashMap<String, String> seedPropMap = seed.getMap();
final Map<String, String> seedPropMap = seed.getMap();
synchronized (seedPropMap) {
seedActiveDB.put(seed.hash, seedPropMap);
}
@ -470,7 +470,7 @@ public final class yacySeedDB implements httpdAlternativeDomainNames {
} catch (final Exception e) { serverLog.logWarning("yacySeedDB", "could not remove hash ("+ e.getClass() +"): "+ e.getMessage()); }
//seed.put(yacySeed.LASTSEEN, yacyCore.shortFormatter.format(new Date(yacyCore.universalTime())));
try {
final HashMap<String, String> seedPropMap = seed.getMap();
final Map<String, String> seedPropMap = seed.getMap();
synchronized (seedPropMap) {
seedPassiveDB.put(seed.hash, seedPropMap);
}
@ -495,7 +495,7 @@ public final class yacySeedDB implements httpdAlternativeDomainNames {
} catch (final Exception e) { serverLog.logWarning("yacySeedDB", "could not remove hash ("+ e.getClass() +"): "+ e.getMessage()); }
//seed.put(yacySeed.LASTSEEN, yacyCore.shortFormatter.format(new Date(yacyCore.universalTime())));
try {
final HashMap<String, String> seedPropMap = seed.getMap();
final Map<String, String> seedPropMap = seed.getMap();
synchronized (seedPropMap) {
seedPotentialDB.put(seed.hash, seedPropMap);
}
@ -552,7 +552,7 @@ public final class yacySeedDB implements httpdAlternativeDomainNames {
private yacySeed get(final String hash, final kelondroMapDataMining database) {
if (hash == null) return null;
if ((this.mySeed != null) && (hash.equals(mySeed.hash))) return mySeed;
HashMap<String, String> entry;
Map<String, String> entry;
try {
entry = database.get(hash);
} catch (final IOException e) {
@ -1006,7 +1006,7 @@ public final class yacySeedDB implements httpdAlternativeDomainNames {
public yacySeed internalNext() {
if ((it == null) || (!(it.hasNext()))) return null;
try {
final HashMap<String, String> dna = it.next();
final Map<String, String> dna = it.next();
if (dna == null) return null;
final String hash = dna.remove("key");
//while (hash.length() < commonHashLength) { hash = hash + "_"; }

Loading…
Cancel
Save