boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

fix for bug http://bugs.yacy.net/view.php?id=10

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7642 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 14 years ago
parent 7c8e764201
commit 17530ca7b5
9 changed files with 84 additions and 35 deletions
Show Stats Download Patch File Download Diff File

10
source/de/anomic/crawler/CrawlSwitchboard.java
Unescape View File

@ -87,12 +87,12 @@ public final class CrawlSwitchboard {
final File profilesActiveFile = new File(queuesRoot, DBFILE_ACTIVE_CRAWL_PROFILES);
try {
this.profilesActiveCrawls = new MapHeap(profilesActiveFile, Word.commonHashLength, NaturalOrder.naturalOrder, 1024 * 64, 500, '_');
this.profilesActiveCrawls = new MapHeap(profilesActiveFile, Word.commonHashLength, NaturalOrder.naturalOrder, 1024 * 64, 500, ' ');
} catch (IOException e) {
Log.logException(e);Log.logException(e);
FileUtils.deletedelete(profilesActiveFile);
try {
this.profilesActiveCrawls = new MapHeap(profilesActiveFile, Word.commonHashLength, NaturalOrder.naturalOrder, 1024 * 64, 500, '_');
this.profilesActiveCrawls = new MapHeap(profilesActiveFile, Word.commonHashLength, NaturalOrder.naturalOrder, 1024 * 64, 500, ' ');
} catch (IOException e1) {
Log.logException(e1);
this.profilesActiveCrawls = null;
@ -107,12 +107,12 @@ public final class CrawlSwitchboard {
final File profilesPassiveFile = new File(queuesRoot, DBFILE_PASSIVE_CRAWL_PROFILES);
try {
this.profilesPassiveCrawls = new MapHeap(profilesPassiveFile, Word.commonHashLength, NaturalOrder.naturalOrder, 1024 * 64, 500, '_');
this.profilesPassiveCrawls = new MapHeap(profilesPassiveFile, Word.commonHashLength, NaturalOrder.naturalOrder, 1024 * 64, 500, ' ');
} catch (IOException e) {
Log.logException(e);Log.logException(e);
FileUtils.deletedelete(profilesActiveFile);
try {
this.profilesPassiveCrawls = new MapHeap(profilesPassiveFile, Word.commonHashLength, NaturalOrder.naturalOrder, 1024 * 64, 500, '_');
this.profilesPassiveCrawls = new MapHeap(profilesPassiveFile, Word.commonHashLength, NaturalOrder.naturalOrder, 1024 * 64, 500, ' ');
} catch (IOException e1) {
Log.logException(e1);
this.profilesPassiveCrawls = null;
@ -259,7 +259,7 @@ public final class CrawlSwitchboard {
final File pdb = new File(this.queuesRoot, DBFILE_ACTIVE_CRAWL_PROFILES);
if (pdb.exists()) FileUtils.deletedelete(pdb);
try {
this.profilesActiveCrawls = new MapHeap(pdb, Word.commonHashLength, NaturalOrder.naturalOrder, 1024 * 64, 500, '_');
this.profilesActiveCrawls = new MapHeap(pdb, Word.commonHashLength, NaturalOrder.naturalOrder, 1024 * 64, 500, ' ');
} catch (IOException e1) {
Log.logException(e1);
this.profilesActiveCrawls = null;

2
source/de/anomic/crawler/Latency.java
Unescape View File

@ -246,7 +246,7 @@ public class Latency {
robotsEntry = null;
}
robotsDelay = (robotsEntry == null) ? 0 : robotsEntry.getCrawlDelayMillis();
if (robotsEntry != null && robotsDelay == 0) return "no waiting for exclusive granted peer"; // no limits if granted exclusively for this peer
if (robotsEntry != null && robotsDelay == 0 && robotsEntry.getAgentName() != null) return "no waiting for exclusive granted peer"; // no limits if granted exclusively for this peer
}
s.append(", robots.delay = ").append(robotsDelay);

2
source/de/anomic/data/BookmarksDB.java
Unescape View File

@ -71,7 +71,7 @@ public class BookmarksDB {
bookmarksFile.getParentFile().mkdirs();
//this.bookmarksTable = new kelondroMap(kelondroDyn.open(bookmarksFile, bufferkb * 1024, preloadTime, 12, 256, '_', true, false));
//this.bookmarksTable = new MapView(BLOBTree.toHeap(bookmarksFile, true, true, 12, 256, '_', NaturalOrder.naturalOrder, bookmarksFileNew), 1000, '_');
this.bookmarks = new MapHeap(bookmarksFile, 12, NaturalOrder.naturalOrder, 1024 * 64, 1000, '_');
this.bookmarks = new MapHeap(bookmarksFile, 12, NaturalOrder.naturalOrder, 1024 * 64, 1000, ' ');
// tags
tags = new ConcurrentHashMap<String, Tag>();

21
source/de/anomic/data/UserDB.java
Unescape View File

@ -72,7 +72,7 @@ public final class UserDB {
FileUtils.deletedelete(userTableFile);
userTableFile.getParentFile().mkdirs();
try {
userTable = new MapHeap(userTableFile, 256, NaturalOrder.naturalOrder, 1024 * 64, 10, '_');
userTable = new MapHeap(userTableFile, 128, NaturalOrder.naturalOrder, 1024 * 64, 10, '_');
} catch (IOException e) {
Log.logException(e);
}
@ -390,9 +390,10 @@ public final class UserDB {
public long getTimeUsed() {
long ret = 0L;
if (this.mem.containsKey(TIME_USED)) {
String s = this.mem.get(TIME_USED);
if (s != null && s.length() > 0) {
try{
ret = Long.parseLong(this.mem.get(TIME_USED));
ret = Long.parseLong(s);
} catch (final NumberFormatException e){
Log.logException(e);
}
@ -408,15 +409,16 @@ public final class UserDB {
public long getTimeLimit() {
long ret = 0L;
if (this.mem.containsKey(TIME_LIMIT)) {
String s = this.mem.get(TIME_LIMIT);
if (s != null && s.length() > 0) {
try {
ret = Long.parseLong(this.mem.get(TIME_LIMIT));
ret = Long.parseLong(s);
} catch (final NumberFormatException e){
Log.logException(e);
}
} else {
try {
this.setProperty(TIME_LIMIT,"0");
this.setProperty(TIME_LIMIT, "0");
} catch (final Exception e) {
Log.logException(e);
}
@ -426,15 +428,16 @@ public final class UserDB {
public long getTrafficSize() {
long ret = 0L;
if (this.mem.containsKey(TRAFFIC_SIZE)) {
String s = this.mem.get(TRAFFIC_SIZE);
if (s != null && s.length() > 0) {
try {
ret = Long.parseLong(this.mem.get(TRAFFIC_SIZE));
ret = Long.parseLong(s);
} catch (final NumberFormatException e) {
Log.logException(e);
}
} else {
try {
this.setProperty(TRAFFIC_SIZE,"0");
this.setProperty(TRAFFIC_SIZE, "0");
} catch (final Exception e) {
Log.logException(e);
}

2
source/de/anomic/http/client/Cache.java
Unescape View File

@ -79,7 +79,7 @@ public final class Cache {
// open the response header database
final File dbfile = new File(cachePath, RESPONSE_HEADER_DB_NAME);
try {
responseHeaderDB = new MapHeap(dbfile, Word.commonHashLength, Base64Order.enhancedCoder, 1024 * 1024, 100, '_');
responseHeaderDB = new MapHeap(dbfile, Word.commonHashLength, Base64Order.enhancedCoder, 1024 * 1024, 100, ' ');
} catch (final IOException e) {
Log.logException(e);
}

1
source/de/anomic/yacy/yacySeedDB.java
Unescape View File

@ -119,6 +119,7 @@ public final class yacySeedDB implements AlternativeDomainNames {
this.myBotIDs = new HashSet<String>();
this.myBotIDs.add("yacy");
this.myBotIDs.add("yacybot");
this.myBotIDs.add("yacyproxy");
this.netRedundancy = redundancy;
this.scheme = new VerticalWordPartitionScheme(partitionExponent);

4
source/net/yacy/kelondro/blob/HeapReader.java
Unescape View File

@ -590,15 +590,13 @@ public class HeapReader {
} catch (IOException e) {
Log.logException(e);
}
} else {
// this is small.. just free resources, do not write index
}
if (free != null) free.clear();
free = null;
if (index != null) index.close();
index = null;
}
}
}
public synchronized void close() {
close(true);

2
source/net/yacy/kelondro/blob/MapDataMining.java
Unescape View File

@ -65,7 +65,7 @@ public class MapDataMining extends MapHeap {
final String[] longaccfields,
final String[] floataccfields,
final Object externalHandler) throws IOException {
super(heapFile, keylength, ordering, buffermax, cachesize, '_');
super(heapFile, keylength, ordering, buffermax, cachesize, ' ');
// create fast ordering clusters and acc fields
this.sortfields = sortfields;

65
source/net/yacy/kelondro/blob/MapHeap.java
Unescape View File

@ -268,6 +268,17 @@ public class MapHeap implements Map<byte[], Map<String, String>> {
return key;
}
private byte[] removeFillchar(byte[] key) {
if (key == null) return key;
int p = key.length - 1;
while (p >= 0 && key[p] == fillchar) p--;
if (p == key.length - 1) return key;
// copy part of key into new byte[]
byte[] k = new byte[p + 1];
System.arraycopy(key, 0, k, 0, k.length);
return k;
}
protected Map<String, String> get(byte[] key, final boolean storeCache) throws IOException, RowSpaceExceededException {
// load map from cache
assert key != null;
@ -324,27 +335,62 @@ public class MapHeap implements Map<byte[], Map<String, String>> {
*/
public synchronized CloneableIterator<byte[]> keys(final boolean up, final boolean rotating) throws IOException {
// simple enumeration of key names without special ordering
return blob.keys(up, rotating);
return new KeyIterator(up, rotating, null, null);
}
/**
* iterate over all keys
* return an iteration of the keys in the map
* the keys in the map are de-normalized which means that the fill-character is removed
* @param up
* @param rotating
* @param firstKey
* @param secondKey
* @return
* @throws IOException
*/
public CloneableIterator<byte[]> keys(final boolean up, final byte[] firstKey) throws IOException {
return keys(up, false, firstKey, null);
public synchronized CloneableIterator<byte[]> keys(final boolean up, final boolean rotating, final byte[] firstKey, final byte[] secondKey) throws IOException {
return new KeyIterator(up, rotating, firstKey, secondKey);
}
public synchronized CloneableIterator<byte[]> keys(final boolean up, final boolean rotating, final byte[] firstKey, final byte[] secondKey) throws IOException {
// simple enumeration of key names without special ordering
public class KeyIterator implements CloneableIterator<byte[]>, Iterator<byte[]> {
final boolean up, rotating;
final byte[] firstKey, secondKey;
Iterator<byte[]> iterator;
public KeyIterator(final boolean up, final boolean rotating, final byte[] firstKey, final byte[] secondKey) throws IOException {
this.up = up;
this.rotating = rotating;
this.firstKey = firstKey;
this.secondKey = secondKey;
final CloneableIterator<byte[]> i = blob.keys(up, firstKey);
if (rotating) return new RotateIterator<byte[]>(i, secondKey, blob.size());
return i;
iterator = (rotating) ? new RotateIterator<byte[]>(i, secondKey, blob.size()) : i;
}
public byte[] next() {
return removeFillchar(iterator.next());
}
@Override
public boolean hasNext() {
return iterator.hasNext();
}
@Override
public void remove() {
iterator.remove();
}
@Override
public CloneableIterator<byte[]> clone(Object modifier) {
try {
return new KeyIterator(this.up, this.rotating, this.firstKey, this.secondKey);
} catch (IOException e) {
return null;
}
}
}
public synchronized MapIterator entries(final boolean up, final boolean rotating) throws IOException {
return new MapIterator(keys(up, rotating));
@ -399,11 +445,12 @@ public class MapHeap implements Map<byte[], Map<String, String>> {
}
public Map<String, String> next() {
final byte[] nextKey = keyIterator.next();
byte[] nextKey = keyIterator.next();
if (nextKey == null) {
finish = true;
return null;
}
nextKey = normalizeKey(nextKey); // the key must be normalized because the keyIterator may iterate over not-normalized keys
try {
final Map<String, String> obj = get(nextKey, false);
if (obj == null) throw new kelondroException("no more elements available");

Write Preview
Loading…
Cancel
Save