boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

synchronization enhancements

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7954 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 14 years ago
parent dd4635e323
commit 23e81b28b2
4 changed files with 433 additions and 430 deletions
Show Stats Download Patch File Download Diff File

118
source/de/anomic/yacy/yacyNewsPool.java
Unescape View File

@ -57,7 +57,6 @@ import net.yacy.kelondro.data.meta.DigestURI;
import net.yacy.kelondro.index.RowSpaceExceededException;
import net.yacy.kelondro.logging.Log;
import net.yacy.repository.Blacklist;
import de.anomic.search.Switchboard;
public class yacyNewsPool {
@ -268,20 +267,20 @@ public class yacyNewsPool {
final File yacyDBPath,
final boolean useTailCache,
final boolean exceed134217727) {
newsDB = new yacyNewsDB(new File(yacyDBPath, "news1024.db"), 1024, useTailCache, exceed134217727);
outgoingNews = new yacyNewsQueue(new File(yacyDBPath, "newsOut.table"), newsDB);
publishedNews = new yacyNewsQueue(new File(yacyDBPath, "newsPublished.table"), newsDB);
incomingNews = new yacyNewsQueue(new File(yacyDBPath, "newsIn.table"), newsDB);
processedNews = new yacyNewsQueue(new File(yacyDBPath, "newsProcessed.table"), newsDB);
maxDistribution = 30;
this.newsDB = new yacyNewsDB(new File(yacyDBPath, "news1024.db"), 1024, useTailCache, exceed134217727);
this.outgoingNews = new yacyNewsQueue(new File(yacyDBPath, "newsOut.table"), this.newsDB);
this.publishedNews = new yacyNewsQueue(new File(yacyDBPath, "newsPublished.table"), this.newsDB);
this.incomingNews = new yacyNewsQueue(new File(yacyDBPath, "newsIn.table"), this.newsDB);
this.processedNews = new yacyNewsQueue(new File(yacyDBPath, "newsProcessed.table"), this.newsDB);
this.maxDistribution = 30;
}
public synchronized void close() {
newsDB.close();
outgoingNews.close();
publishedNews.close();
incomingNews.close();
processedNews.close();
this.newsDB.close();
this.outgoingNews.close();
this.publishedNews.close();
this.incomingNews.close();
this.processedNews.close();
}
public Iterator<yacyNewsDB.Record> recordIterator(final int dbKey, final boolean up) {
@ -291,24 +290,24 @@ public class yacyNewsPool {
}
public yacyNewsDB.Record parseExternal(final String external) {
return newsDB.newRecord(external);
return this.newsDB.newRecord(external);
}
public void publishMyNews(final yacySeed mySeed, final String category, final Map<String, String> attributes) {
publishMyNews(newsDB.newRecord(mySeed, category, attributes));
publishMyNews(this.newsDB.newRecord(mySeed, category, attributes));
}
public void publishMyNews(final yacySeed mySeed, final String category, final Properties attributes) {
publishMyNews(newsDB.newRecord(mySeed, category, attributes));
publishMyNews(this.newsDB.newRecord(mySeed, category, attributes));
}
private void publishMyNews(final yacyNewsDB.Record record) {
// this shall be called if our peer generated a new news record and wants to publish it
if (record == null) return;
try {
if (newsDB.get(record.id()) == null) {
incomingNews.push(record); // we want to see our own news..
outgoingNews.push(record); // .. and put it on the publishing list
if (this.newsDB.get(record.id()) == null) {
this.incomingNews.push(record); // we want to see our own news..
this.outgoingNews.push(record); // .. and put it on the publishing list
}
} catch (final Exception e) {
Log.logException(e);
@ -317,17 +316,17 @@ public class yacyNewsPool {
public yacyNewsDB.Record myPublication() throws IOException, RowSpaceExceededException {
// generate a record for next peer-ping
if (outgoingNews.isEmpty()) return null;
final yacyNewsDB.Record record = outgoingNews.pop();
if (this.outgoingNews.isEmpty()) return null;
final yacyNewsDB.Record record = this.outgoingNews.pop();
if (record == null) return null;
record.incDistribution();
if (record.distributed() >= maxDistribution) {
if (record.distributed() >= this.maxDistribution) {
// move record to its final position. This is only for history
publishedNews.push(record);
this.publishedNews.push(record);
} else {
outgoingNews.push(record);
this.outgoingNews.push(record);
}
return record;
}
@ -356,8 +355,8 @@ public class yacyNewsPool {
}
// double-check with old news
if (newsDB.get(record.id()) != null) return;
incomingNews.push(record);
if (this.newsDB.get(record.id()) != null) return;
this.incomingNews.push(record);
// add message to feed channel
//RSSFeed.channels(yacyCore.channelName).addMessage(new RSSMessage("Incoming News: " + record.category() + " from " + record.originator(), record.attributes().toString()));
@ -373,8 +372,10 @@ public class yacyNewsPool {
yacyNewsDB.Record record;
int pc = 0;
synchronized (this.incomingNews) {
final Iterator<yacyNewsDB.Record> i = incomingNews.records(true);
Set<String> removeIDs = new HashSet<String>();
final Iterator<yacyNewsDB.Record> i = this.incomingNews.records(true);
final Set<String> removeIDs = new HashSet<String>();
// this loop should not run too long! This may happen if the incoming news are long and not deleted after processing
final long start = System.currentTimeMillis();
while (i.hasNext()) {
// check for interruption
if (Thread.currentThread().isInterrupted()) throw new InterruptedException("Shutdown in progress");
@ -386,8 +387,12 @@ public class yacyNewsPool {
removeIDs.add(record.id());
pc++;
}
if (System.currentTimeMillis() - start > 100) break; // time-out for this to avoid deadlocks during concurrent peer-pings
}
for (final String id: removeIDs) {
final yacyNewsDB.Record deletedRecord = this.incomingNews.remove(id);
assert deletedRecord != null;
}
for (final String id: removeIDs) incomingNews.remove(id);
}
return pc;
}
@ -395,26 +400,27 @@ public class yacyNewsPool {
private boolean automaticProcessP(final yacySeedDB seedDB, final yacyNewsDB.Record record) {
if (record == null) return false;
if (record.category() == null) return true;
if ((System.currentTimeMillis() - record.created().getTime()) > (14 * MILLISECONDS_PER_DAY)) {
final long created = record.created().getTime();
if ((System.currentTimeMillis() - created) > (7L * MILLISECONDS_PER_DAY)) {
// remove everything after 1 week
return true;
}
if ((record.category().equals(CATEGORY_WIKI_UPDATE)) &&
((System.currentTimeMillis() - record.created().getTime()) > (3 * MILLISECONDS_PER_DAY))) {
((System.currentTimeMillis() - created) > (3L * MILLISECONDS_PER_DAY))) {
return true;
}
if ((record.category().equals(CATEGORY_BLOG_ADD)) &&
((System.currentTimeMillis() - record.created().getTime()) > (3 * MILLISECONDS_PER_DAY))) {
((System.currentTimeMillis() - created) > (3L * MILLISECONDS_PER_DAY))) {
return true;
}
if ((record.category().equals(CATEGORY_PROFILE_UPDATE)) &&
((System.currentTimeMillis() - record.created().getTime()) > (7 * MILLISECONDS_PER_DAY))) {
((System.currentTimeMillis() - created) > (3L * MILLISECONDS_PER_DAY))) {
return true;
}
if ((record.category().equals(CATEGORY_CRAWL_START)) &&
((System.currentTimeMillis() - record.created().getTime()) > (2 * MILLISECONDS_PER_DAY))) {
((System.currentTimeMillis() - created) > (3L * MILLISECONDS_PER_DAY))) {
final yacySeed seed = seedDB.get(record.originator());
if (seed == null) return false;
if (seed == null) return true;
try {
return (Integer.parseInt(seed.get(yacySeed.ISPEED, "-")) < 10);
} catch (final NumberFormatException ee) {
@ -456,20 +462,20 @@ public class yacyNewsPool {
public synchronized yacyNewsDB.Record getByID(final int dbKey, final String id) {
switch (dbKey) {
case INCOMING_DB: return incomingNews.get(id);
case PROCESSED_DB: return processedNews.get(id);
case OUTGOING_DB: return outgoingNews.get(id);
case PUBLISHED_DB: return publishedNews.get(id);
case INCOMING_DB: return this.incomingNews.get(id);
case PROCESSED_DB: return this.processedNews.get(id);
case OUTGOING_DB: return this.outgoingNews.get(id);
case PUBLISHED_DB: return this.publishedNews.get(id);
}
return null;
}
private yacyNewsQueue switchQueue(final int dbKey) {
switch (dbKey) {
case INCOMING_DB: return incomingNews;
case PROCESSED_DB: return processedNews;
case OUTGOING_DB: return outgoingNews;
case PUBLISHED_DB: return publishedNews;
case INCOMING_DB: return this.incomingNews;
case PROCESSED_DB: return this.processedNews;
case OUTGOING_DB: return this.outgoingNews;
case PUBLISHED_DB: return this.publishedNews;
}
return null;
}
@ -477,10 +483,10 @@ public class yacyNewsPool {
public void clear(final int dbKey) {
// clear a table
switch (dbKey) {
case INCOMING_DB: incomingNews.clear(); break;
case PROCESSED_DB: processedNews.clear(); break;
case OUTGOING_DB: outgoingNews.clear(); break;
case PUBLISHED_DB: publishedNews.clear(); break;
case INCOMING_DB: this.incomingNews.clear(); break;
case PROCESSED_DB: this.processedNews.clear(); break;
case OUTGOING_DB: this.outgoingNews.clear(); break;
case PUBLISHED_DB: this.publishedNews.clear(); break;
}
}
@ -488,10 +494,10 @@ public class yacyNewsPool {
// this is called if a queue element shall be moved to another queue or off the queue
// it depends on the dbKey how the record is handled
switch (dbKey) {
case INCOMING_DB: moveOff(incomingNews, processedNews, id); break;
case PROCESSED_DB: moveOff(processedNews, null,id); break;
case OUTGOING_DB: moveOff(outgoingNews, publishedNews, id); break;
case PUBLISHED_DB: moveOff(publishedNews, null, id); break;
case INCOMING_DB: moveOff(this.incomingNews, this.processedNews, id); break;
case PROCESSED_DB: moveOff(this.processedNews, null,id); break;
case OUTGOING_DB: moveOff(this.outgoingNews, this.publishedNews, id); break;
case PUBLISHED_DB: moveOff(this.publishedNews, null, id); break;
}
}
@ -503,8 +509,8 @@ public class yacyNewsPool {
}
if (toqueue != null) {
toqueue.push(record);
} else if ((incomingNews.get(id) == null) && (processedNews.get(id) == null) && (outgoingNews.get(id) == null) && (publishedNews.get(id) == null)) {
newsDB.remove(id);
} else if ((this.incomingNews.get(id) == null) && (this.processedNews.get(id) == null) && (this.outgoingNews.get(id) == null) && (this.publishedNews.get(id) == null)) {
this.newsDB.remove(id);
}
return true;
}
@ -513,10 +519,10 @@ public class yacyNewsPool {
// this is called if a queue element shall be moved to another queue or off the queue
// it depends on the dbKey how the record is handled
switch (dbKey) {
case INCOMING_DB: moveOffAll(incomingNews, processedNews); break;
case PROCESSED_DB: processedNews.clear(); break;
case OUTGOING_DB: moveOffAll(outgoingNews, publishedNews); break;
case PUBLISHED_DB: publishedNews.clear(); break;
case INCOMING_DB: moveOffAll(this.incomingNews, this.processedNews); break;
case PROCESSED_DB: this.processedNews.clear(); break;
case OUTGOING_DB: moveOffAll(this.outgoingNews, this.publishedNews); break;
case PUBLISHED_DB: this.publishedNews.clear(); break;
}
}

3
source/de/anomic/yacy/yacySeed.java
Unescape View File

@ -51,7 +51,6 @@ import java.text.ParseException;
import java.util.Comparator;
import java.util.Date;
import java.util.Iterator;
import java.util.Map;
import java.util.Random;
import java.util.Set;
import java.util.TreeMap;
@ -374,7 +373,7 @@ public class yacySeed implements Cloneable, Comparable<yacySeed>, Comparator<yac
}
/** @return the DNA-map of this peer */
public final Map<String, String> getMap() {
public final ConcurrentMap<String, String> getMap() {
return this.dna;
}

358
source/de/anomic/yacy/yacySeedDB.java
Unescape View File

@ -39,6 +39,7 @@ import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import net.yacy.cora.document.ASCII;
import net.yacy.cora.document.UTF8;
@ -56,11 +57,8 @@ import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.order.Base64Order;
import net.yacy.kelondro.util.FileUtils;
import net.yacy.kelondro.util.kelondroException;
//import de.anomic.http.client.Client;
import de.anomic.http.server.AlternativeDomainNames;
import de.anomic.http.server.HTTPDemon;
//import de.anomic.http.server.ResponseContainer;
import de.anomic.search.Switchboard;
import de.anomic.server.serverCore;
import de.anomic.server.serverSwitch;
@ -99,7 +97,7 @@ public final class yacySeedDB implements AlternativeDomainNames {
public PartitionScheme scheme;
private yacySeed mySeed; // my own seed
private Set<String> myBotIDs; // list of id's that this bot accepts as robots.txt identification
private final Set<String> myBotIDs; // list of id's that this bot accepts as robots.txt identification
private final Hashtable<String, String> nameLookupCache; // a name-to-hash relation
private final Hashtable<InetAddress, SoftReference<yacySeed>> ipLookupCache;
@ -126,9 +124,9 @@ public final class yacySeedDB implements AlternativeDomainNames {
this.scheme = new VerticalWordPartitionScheme(partitionExponent);
// set up seed database
this.seedActiveDB = openSeedTable(seedActiveDBFile);
this.seedPassiveDB = openSeedTable(seedPassiveDBFile);
this.seedPotentialDB = openSeedTable(seedPotentialDBFile);
this.seedActiveDB = openSeedTable(this.seedActiveDBFile);
this.seedPassiveDB = openSeedTable(this.seedPassiveDBFile);
this.seedPotentialDB = openSeedTable(this.seedPotentialDBFile);
// start our virtual DNS service for yacy peers with empty cache
this.nameLookupCache = new Hashtable<String, String>();
@ -148,11 +146,11 @@ public final class yacySeedDB implements AlternativeDomainNames {
this.newsPool = new yacyNewsPool(networkRoot, useTailCache, exceed134217727);
// deploy peer actions
this.peerActions = new yacyPeerActions(this, newsPool);
this.peerActions = new yacyPeerActions(this, this.newsPool);
}
public void relocate(
File newNetworkRoot,
final File newNetworkRoot,
final int redundancy,
final int partitionExponent,
final boolean useTailCache,
@ -166,13 +164,13 @@ public final class yacySeedDB implements AlternativeDomainNames {
this.peerActions.close();
// open new according to the newNetworkRoot
this.seedActiveDBFile = new File(newNetworkRoot, seedActiveDBFile.getName());
this.seedPassiveDBFile = new File(newNetworkRoot, seedPassiveDBFile.getName());
this.seedPotentialDBFile = new File(newNetworkRoot, seedPotentialDBFile.getName());
this.seedActiveDBFile = new File(newNetworkRoot, this.seedActiveDBFile.getName());
this.seedPassiveDBFile = new File(newNetworkRoot, this.seedPassiveDBFile.getName());
this.seedPotentialDBFile = new File(newNetworkRoot, this.seedPotentialDBFile.getName());
// replace my (old) seed with new seed definition from other network
// but keep the seed name
String peername = this.myName();
final String peername = myName();
this.mySeed = null; // my own seed
this.myOwnSeedFile = new File(newNetworkRoot, yacySeedDB.DBFILE_OWN_SEED);
initMySeed();
@ -182,9 +180,9 @@ public final class yacySeedDB implements AlternativeDomainNames {
this.scheme = new VerticalWordPartitionScheme(partitionExponent);
// set up seed database
this.seedActiveDB = openSeedTable(seedActiveDBFile);
this.seedPassiveDB = openSeedTable(seedPassiveDBFile);
this.seedPotentialDB = openSeedTable(seedPotentialDBFile);
this.seedActiveDB = openSeedTable(this.seedActiveDBFile);
this.seedPassiveDB = openSeedTable(this.seedPassiveDBFile);
this.seedPotentialDB = openSeedTable(this.seedPotentialDBFile);
// start our virtual DNS service for yacy peers with empty cache
this.nameLookupCache.clear();
@ -204,44 +202,44 @@ public final class yacySeedDB implements AlternativeDomainNames {
this.newsPool = new yacyNewsPool(newNetworkRoot, useTailCache, exceed134217727);
// deploy peer actions
this.peerActions = new yacyPeerActions(this, newsPool);
this.peerActions = new yacyPeerActions(this, this.newsPool);
}
private synchronized void initMySeed() {
if (this.mySeed != null) return;
// create or init own seed
if (myOwnSeedFile.length() > 0) try {
if (this.myOwnSeedFile.length() > 0) try {
// load existing identity
mySeed = yacySeed.load(myOwnSeedFile);
if (mySeed == null) throw new IOException("current seed is null");
this.mySeed = yacySeed.load(this.myOwnSeedFile);
if (this.mySeed == null) throw new IOException("current seed is null");
} catch (final IOException e) {
// create new identity
Log.logSevere("SEEDDB", "could not load stored mySeed.txt from " + myOwnSeedFile.toString() + ": " + e.getMessage() + ". creating new seed.", e);
mySeed = yacySeed.genLocalSeed(this);
Log.logSevere("SEEDDB", "could not load stored mySeed.txt from " + this.myOwnSeedFile.toString() + ": " + e.getMessage() + ". creating new seed.", e);
this.mySeed = yacySeed.genLocalSeed(this);
try {
mySeed.save(myOwnSeedFile);
this.mySeed.save(this.myOwnSeedFile);
} catch (final IOException ee) {
Log.logSevere("SEEDDB", "error saving mySeed.txt (1) to " + myOwnSeedFile.toString() + ": " + ee.getMessage(), ee);
Log.logSevere("SEEDDB", "error saving mySeed.txt (1) to " + this.myOwnSeedFile.toString() + ": " + ee.getMessage(), ee);
Log.logException(ee);
System.exit(-1);
}
} else {
// create new identity
Log.logInfo("SEEDDB", "could not find stored mySeed.txt at " + myOwnSeedFile.toString() + ": " + ". creating new seed.");
mySeed = yacySeed.genLocalSeed(this);
Log.logInfo("SEEDDB", "could not find stored mySeed.txt at " + this.myOwnSeedFile.toString() + ": " + ". creating new seed.");
this.mySeed = yacySeed.genLocalSeed(this);
try {
mySeed.save(myOwnSeedFile);
this.mySeed.save(this.myOwnSeedFile);
} catch (final IOException ee) {
Log.logSevere("SEEDDB", "error saving mySeed.txt (2) to " + myOwnSeedFile.toString() + ": " + ee.getMessage(), ee);
Log.logSevere("SEEDDB", "error saving mySeed.txt (2) to " + this.myOwnSeedFile.toString() + ": " + ee.getMessage(), ee);
Log.logException(ee);
System.exit(-1);
}
}
this.myBotIDs.add(this.mySeed.getName() + ".yacy");
this.myBotIDs.add(this.mySeed.hash + ".yacyh");
mySeed.setIP(""); // we delete the old information to see what we have now
mySeed.put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_VIRGIN); // markup startup condition
this.mySeed.setIP(""); // we delete the old information to see what we have now
this.mySeed.put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_VIRGIN); // markup startup condition
}
public Set<String> myBotIDs() {
@ -259,17 +257,17 @@ public final class yacySeedDB implements AlternativeDomainNames {
public yacySeed mySeed() {
if (this.mySeed == null) {
if (this.sizeConnected() == 0) try {Thread.sleep(5000);} catch (final InterruptedException e) {} // wait for init
if (sizeConnected() == 0) try {Thread.sleep(5000);} catch (final InterruptedException e) {} // wait for init
initMySeed();
// check if my seed has an IP assigned
if (this.myIP() == null || this.myIP().length() == 0) {
if (myIP() == null || myIP().length() == 0) {
this.mySeed.setIP(Domains.myPublicLocalIP().getHostAddress());
}
}
return this.mySeed;
}
public void setMyName(String name) {
public void setMyName(final String name) {
this.myBotIDs.remove(this.mySeed.getName() + ".yacy");
this.mySeed.setName(name);
this.myBotIDs.add(name + ".yacy");
@ -288,33 +286,33 @@ public final class yacySeedDB implements AlternativeDomainNames {
}
public String myName() {
return mySeed.getName();
return this.mySeed.getName();
}
public String myID() {
return mySeed.hash;
return this.mySeed.hash;
}
public synchronized void removeMySeed() {
if (seedActiveDB.isEmpty() && seedPassiveDB.isEmpty() && seedPotentialDB.isEmpty()) return; // avoid that the own seed is initialized too early
if (this.seedActiveDB.isEmpty() && this.seedPassiveDB.isEmpty() && this.seedPotentialDB.isEmpty()) return; // avoid that the own seed is initialized too early
if (this.mySeed == null) initMySeed();
try {
byte[] mySeedHash = ASCII.getBytes(mySeed.hash);
seedActiveDB.delete(mySeedHash);
seedPassiveDB.delete(mySeedHash);
seedPotentialDB.delete(mySeedHash);
final byte[] mySeedHash = ASCII.getBytes(this.mySeed.hash);
this.seedActiveDB.delete(mySeedHash);
this.seedPassiveDB.delete(mySeedHash);
this.seedPotentialDB.delete(mySeedHash);
} catch (final IOException e) { Log.logWarning("yacySeedDB", "could not remove hash ("+ e.getClass() +"): "+ e.getMessage()); }
}
public void saveMySeed() {
try {
this.mySeed().save(myOwnSeedFile);
} catch (final IOException e) { Log.logWarning("yacySeedDB", "could not save mySeed '"+ myOwnSeedFile +"': "+ e.getMessage()); }
mySeed().save(this.myOwnSeedFile);
} catch (final IOException e) { Log.logWarning("yacySeedDB", "could not save mySeed '"+ this.myOwnSeedFile +"': "+ e.getMessage()); }
}
public boolean noDHTActivity() {
// for small networks, we don't perform DHT transmissions, because it is possible to search over all peers
return this.sizeConnected() <= dhtActivityMagic;
return sizeConnected() <= dhtActivityMagic;
}
private synchronized MapDataMining openSeedTable(final File seedDBFile) {
@ -330,7 +328,7 @@ public final class yacySeedDB implements AlternativeDomainNames {
FileUtils.deletedelete(seedDBFile);
try {
return new MapDataMining(seedDBFile, Word.commonHashLength, Base64Order.enhancedCoder, 1024 * 512, 500, sortFields, longaccFields, doubleaccFields, this);
} catch (IOException e1) {
} catch (final IOException e1) {
Log.logException(e1);
System.exit(-1);
return null;
@ -351,31 +349,31 @@ public final class yacySeedDB implements AlternativeDomainNames {
return seedDB;
}
public synchronized void resetActiveTable() { seedActiveDB = resetSeedTable(seedActiveDB, seedActiveDBFile); }
private synchronized void resetPassiveTable() { seedPassiveDB = resetSeedTable(seedPassiveDB, seedPassiveDBFile); }
private synchronized void resetPotentialTable() { seedPotentialDB = resetSeedTable(seedPotentialDB, seedPotentialDBFile); }
public synchronized void resetActiveTable() { this.seedActiveDB = resetSeedTable(this.seedActiveDB, this.seedActiveDBFile); }
private synchronized void resetPassiveTable() { this.seedPassiveDB = resetSeedTable(this.seedPassiveDB, this.seedPassiveDBFile); }
private synchronized void resetPotentialTable() { this.seedPotentialDB = resetSeedTable(this.seedPotentialDB, this.seedPotentialDBFile); }
public void close() {
if (seedActiveDB != null) seedActiveDB.close();
if (seedPassiveDB != null) seedPassiveDB.close();
if (seedPotentialDB != null) seedPotentialDB.close();
newsPool.close();
peerActions.close();
if (this.seedActiveDB != null) this.seedActiveDB.close();
if (this.seedPassiveDB != null) this.seedPassiveDB.close();
if (this.seedPotentialDB != null) this.seedPotentialDB.close();
this.newsPool.close();
this.peerActions.close();
}
public Iterator<yacySeed> seedsSortedConnected(final boolean up, final String field) {
// enumerates seed-type objects: all seeds sequentially ordered by field
return new seedEnum(up, field, seedActiveDB);
return new seedEnum(up, field, this.seedActiveDB);
}
public Iterator<yacySeed> seedsSortedDisconnected(final boolean up, final String field) {
// enumerates seed-type objects: all seeds sequentially ordered by field
return new seedEnum(up, field, seedPassiveDB);
return new seedEnum(up, field, this.seedPassiveDB);
}
public Iterator<yacySeed> seedsSortedPotential(final boolean up, final String field) {
// enumerates seed-type objects: all seeds sequentially ordered by field
return new seedEnum(up, field, seedPotentialDB);
return new seedEnum(up, field, this.seedPotentialDB);
}
public TreeMap<byte[], String> /* peer-b64-hashes/ipport */ clusterHashes(final String clusterdefinition) {
@ -392,13 +390,13 @@ public final class yacySeedDB implements AlternativeDomainNames {
yacySeed seed;
String hash, yacydom, ipport;
int p;
for (int i = 0; i < addresses.length; i++) {
p = addresses[i].indexOf('=');
for (final String addresse : addresses) {
p = addresse.indexOf('=');
if (p >= 0) {
yacydom = addresses[i].substring(0, p);
ipport = addresses[i].substring(p + 1);
yacydom = addresse.substring(0, p);
ipport = addresse.substring(p + 1);
} else {
yacydom = addresses[i];
yacydom = addresse;
ipport = null;
}
if (yacydom.endsWith(".yacyh")) {
@ -419,7 +417,7 @@ public final class yacySeedDB implements AlternativeDomainNames {
clustermap.put(ASCII.getBytes(seed.hash), ipport);
}
} else {
yacyCore.log.logWarning("cluster peer '" + addresses[i] + "' has wrong syntax. the name must end with .yacy or .yacyh");
yacyCore.log.logWarning("cluster peer '" + addresse + "' has wrong syntax. the name must end with .yacy or .yacyh");
}
}
return clustermap;
@ -427,17 +425,17 @@ public final class yacySeedDB implements AlternativeDomainNames {
public Iterator<yacySeed> seedsConnected(final boolean up, final boolean rot, final byte[] firstHash, final float minVersion) {
// enumerates seed-type objects: all seeds sequentially without order
return new seedEnum(up, rot, (firstHash == null) ? null : firstHash, null, seedActiveDB, minVersion);
return new seedEnum(up, rot, (firstHash == null) ? null : firstHash, null, this.seedActiveDB, minVersion);
}
private Iterator<yacySeed> seedsDisconnected(final boolean up, final boolean rot, final byte[] firstHash, final float minVersion) {
// enumerates seed-type objects: all seeds sequentially without order
return new seedEnum(up, rot, (firstHash == null) ? null : firstHash, null, seedPassiveDB, minVersion);
return new seedEnum(up, rot, (firstHash == null) ? null : firstHash, null, this.seedPassiveDB, minVersion);
}
private Iterator<yacySeed> seedsPotential(final boolean up, final boolean rot, final byte[] firstHash, final float minVersion) {
// enumerates seed-type objects: all seeds sequentially without order
return new seedEnum(up, rot, (firstHash == null) ? null : firstHash, null, seedPotentialDB, minVersion);
return new seedEnum(up, rot, (firstHash == null) ? null : firstHash, null, this.seedPotentialDB, minVersion);
}
public yacySeed anySeedVersion(final float minVersion) {
@ -451,8 +449,8 @@ public final class yacySeedDB implements AlternativeDomainNames {
* @param limit the time limit in minutes. 1440 minutes is a day
* @return the number of peers seen in the given time
*/
public int sizeActiveSince(long limit) {
int c = seedActiveDB.size();
public int sizeActiveSince(final long limit) {
int c = this.seedActiveDB.size();
yacySeed seed;
Iterator<yacySeed> i = seedsSortedDisconnected(false, yacySeed.LASTSEEN);
while (i.hasNext()) {
@ -474,119 +472,119 @@ public final class yacySeedDB implements AlternativeDomainNames {
}
public int sizeConnected() {
return seedActiveDB.size();
return this.seedActiveDB.size();
}
public int sizeDisconnected() {
return seedPassiveDB.size();
return this.seedPassiveDB.size();
}
public int sizePotential() {
return seedPotentialDB.size();
return this.seedPotentialDB.size();
}
public long countActiveURL() { return seedActiveDB.getLongAcc(yacySeed.LCOUNT); }
public long countActiveRWI() { return seedActiveDB.getLongAcc(yacySeed.ICOUNT); }
public long countActivePPM() { return seedActiveDB.getLongAcc(yacySeed.ISPEED); }
public float countActiveQPM() { return seedActiveDB.getFloatAcc(yacySeed.RSPEED); }
public long countPassiveURL() { return seedPassiveDB.getLongAcc(yacySeed.LCOUNT); }
public long countPassiveRWI() { return seedPassiveDB.getLongAcc(yacySeed.ICOUNT); }
public long countPotentialURL() { return seedPotentialDB.getLongAcc(yacySeed.LCOUNT); }
public long countPotentialRWI() { return seedPotentialDB.getLongAcc(yacySeed.ICOUNT); }
public long countActiveURL() { return this.seedActiveDB.getLongAcc(yacySeed.LCOUNT); }
public long countActiveRWI() { return this.seedActiveDB.getLongAcc(yacySeed.ICOUNT); }
public long countActivePPM() { return this.seedActiveDB.getLongAcc(yacySeed.ISPEED); }
public float countActiveQPM() { return this.seedActiveDB.getFloatAcc(yacySeed.RSPEED); }
public long countPassiveURL() { return this.seedPassiveDB.getLongAcc(yacySeed.LCOUNT); }
public long countPassiveRWI() { return this.seedPassiveDB.getLongAcc(yacySeed.ICOUNT); }
public long countPotentialURL() { return this.seedPotentialDB.getLongAcc(yacySeed.LCOUNT); }
public long countPotentialRWI() { return this.seedPotentialDB.getLongAcc(yacySeed.ICOUNT); }
public synchronized void addConnected(final yacySeed seed) {
public void addConnected(final yacySeed seed) {
if (seed.isProper(false) != null) return;
//seed.put(yacySeed.LASTSEEN, yacyCore.shortFormatter.format(new Date(yacyCore.universalTime())));
synchronized (this) {
try {
nameLookupCache.put(seed.getName(), seed.hash);
final Map<String, String> seedPropMap = seed.getMap();
synchronized (seedPropMap) {
seedActiveDB.insert(ASCII.getBytes(seed.hash), seedPropMap);
}
seedPassiveDB.delete(ASCII.getBytes(seed.hash));
seedPotentialDB.delete(ASCII.getBytes(seed.hash));
this.nameLookupCache.put(seed.getName(), seed.hash);
final ConcurrentMap<String, String> seedPropMap = seed.getMap();
this.seedActiveDB.insert(ASCII.getBytes(seed.hash), seedPropMap);
this.seedPassiveDB.delete(ASCII.getBytes(seed.hash));
this.seedPotentialDB.delete(ASCII.getBytes(seed.hash));
} catch (final Exception e) {
yacyCore.log.logSevere("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
resetActiveTable();
}
}
}
protected synchronized void addDisconnected(final yacySeed seed) {
protected void addDisconnected(final yacySeed seed) {
if (seed.isProper(false) != null) return;
synchronized (this) {
try {
nameLookupCache.remove(seed.getName());
seedActiveDB.delete(ASCII.getBytes(seed.hash));
seedPotentialDB.delete(ASCII.getBytes(seed.hash));
this.nameLookupCache.remove(seed.getName());
this.seedActiveDB.delete(ASCII.getBytes(seed.hash));
this.seedPotentialDB.delete(ASCII.getBytes(seed.hash));
} catch (final Exception e) { Log.logWarning("yacySeedDB", "could not remove hash ("+ e.getClass() +"): "+ e.getMessage()); }
//seed.put(yacySeed.LASTSEEN, yacyCore.shortFormatter.format(new Date(yacyCore.universalTime())));
try {
final Map<String, String> seedPropMap = seed.getMap();
synchronized (seedPropMap) {
seedPassiveDB.insert(ASCII.getBytes(seed.hash), seedPropMap);
}
final ConcurrentMap<String, String> seedPropMap = seed.getMap();
this.seedPassiveDB.insert(ASCII.getBytes(seed.hash), seedPropMap);
} catch (final Exception e) {
yacyCore.log.logSevere("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
resetPassiveTable();
}
}
}
protected synchronized void addPotential(final yacySeed seed) {
protected void addPotential(final yacySeed seed) {
if (seed.isProper(false) != null) return;
synchronized (this) {
try {
nameLookupCache.remove(seed.getName());
seedActiveDB.delete(ASCII.getBytes(seed.hash));
seedPassiveDB.delete(ASCII.getBytes(seed.hash));
this.nameLookupCache.remove(seed.getName());
this.seedActiveDB.delete(ASCII.getBytes(seed.hash));
this.seedPassiveDB.delete(ASCII.getBytes(seed.hash));
} catch (final Exception e) { Log.logWarning("yacySeedDB", "could not remove hash ("+ e.getClass() +"): "+ e.getMessage()); }
//seed.put(yacySeed.LASTSEEN, yacyCore.shortFormatter.format(new Date(yacyCore.universalTime())));
try {
final Map<String, String> seedPropMap = seed.getMap();
synchronized (seedPropMap) {
seedPotentialDB.insert(ASCII.getBytes(seed.hash), seedPropMap);
}
final ConcurrentMap<String, String> seedPropMap = seed.getMap();
this.seedPotentialDB.insert(ASCII.getBytes(seed.hash), seedPropMap);
} catch (final Exception e) {
yacyCore.log.logSevere("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
resetPotentialTable();
}
}
}
public synchronized void removeDisconnected(final String peerHash) {
if (peerHash == null) return;
try {
seedPassiveDB.delete(ASCII.getBytes(peerHash));
this.seedPassiveDB.delete(ASCII.getBytes(peerHash));
} catch (final IOException e) { Log.logWarning("yacySeedDB", "could not remove hash ("+ e.getClass() +"): "+ e.getMessage()); }
}
public synchronized void removePotential(final String peerHash) {
if (peerHash == null) return;
try {
seedPotentialDB.delete(ASCII.getBytes(peerHash));
this.seedPotentialDB.delete(ASCII.getBytes(peerHash));
} catch (final IOException e) { Log.logWarning("yacySeedDB", "could not remove hash ("+ e.getClass() +"): "+ e.getMessage()); }
}
public boolean hasConnected(final byte[] hash) {
return seedActiveDB.containsKey(hash);
return this.seedActiveDB.containsKey(hash);
}
public boolean hasDisconnected(final byte[] hash) {
return seedPassiveDB.containsKey(hash);
return this.seedPassiveDB.containsKey(hash);
}
public boolean hasPotential(final byte[] hash) {
return seedPotentialDB.containsKey(hash);
return this.seedPotentialDB.containsKey(hash);
}
private yacySeed get(final String hash, final MapDataMining database) {
if (hash == null || hash.length() == 0) return null;
if ((this.mySeed != null) && (hash.equals(mySeed.hash))) return mySeed;
ConcurrentHashMap<String, String> entry = new ConcurrentHashMap<String, String>();
if ((this.mySeed != null) && (hash.equals(this.mySeed.hash))) return this.mySeed;
final ConcurrentHashMap<String, String> entry = new ConcurrentHashMap<String, String>();
try {
Map<String, String> map = database.get(ASCII.getBytes(hash));
final Map<String, String> map = database.get(ASCII.getBytes(hash));
if (map == null) return null;
entry.putAll(map);
} catch (final IOException e) {
Log.logException(e);
return null;
} catch (RowSpaceExceededException e) {
} catch (final RowSpaceExceededException e) {
Log.logException(e);
return null;
}
@ -594,15 +592,15 @@ public final class yacySeedDB implements AlternativeDomainNames {
}
public yacySeed getConnected(final String hash) {
return get(hash, seedActiveDB);
return get(hash, this.seedActiveDB);
}
public yacySeed getDisconnected(final String hash) {
return get(hash, seedPassiveDB);
return get(hash, this.seedPassiveDB);
}
public yacySeed getPotential(final String hash) {
return get(hash, seedPotentialDB);
return get(hash, this.seedPotentialDB);
}
public yacySeed get(final String hash) {
@ -614,19 +612,19 @@ public final class yacySeedDB implements AlternativeDomainNames {
public void update(final String hash, final yacySeed seed) {
if (this.mySeed == null) initMySeed();
if (hash.equals(mySeed.hash)) {
mySeed = seed;
if (hash.equals(this.mySeed.hash)) {
this.mySeed = seed;
return;
}
byte[] hashb = ASCII.getBytes(hash);
yacySeed s = get(hash, seedActiveDB);
if (s != null) try { seedActiveDB.insert(hashb, seed.getMap()); return;} catch (final Exception e) {Log.logException(e);}
final byte[] hashb = ASCII.getBytes(hash);
yacySeed s = get(hash, this.seedActiveDB);
if (s != null) try { this.seedActiveDB.insert(hashb, seed.getMap()); return;} catch (final Exception e) {Log.logException(e);}
s = get(hash, seedPassiveDB);
if (s != null) try { seedPassiveDB.insert(hashb, seed.getMap()); return;} catch (final Exception e) {Log.logException(e);}
s = get(hash, this.seedPassiveDB);
if (s != null) try { this.seedPassiveDB.insert(hashb, seed.getMap()); return;} catch (final Exception e) {Log.logException(e);}
s = get(hash, seedPotentialDB);
if (s != null) try { seedPotentialDB.insert(hashb, seed.getMap()); return;} catch (final Exception e) {Log.logException(e);}
s = get(hash, this.seedPotentialDB);
if (s != null) try { this.seedPotentialDB.insert(hashb, seed.getMap()); return;} catch (final Exception e) {Log.logException(e);}
}
public yacySeed lookupByName(String peerName) {
@ -636,11 +634,11 @@ public final class yacySeedDB implements AlternativeDomainNames {
// local peer?
if (peerName.equals("localpeer")) {
if (this.mySeed == null) initMySeed();
return mySeed;
return this.mySeed;
}
// then try to use the cache
String seedhash = nameLookupCache.get(peerName);
final String seedhash = this.nameLookupCache.get(peerName);
yacySeed seed;
if (seedhash != null) {
seed = this.get(seedhash);
@ -655,16 +653,16 @@ public final class yacySeedDB implements AlternativeDomainNames {
seed = e.next();
if (seed != null) {
name = seed.getName().toLowerCase();
if (seed.isProper(false) == null) nameLookupCache.put(name, seed.hash);
if (seed.isProper(false) == null) this.nameLookupCache.put(name, seed.hash);
if (name.equals(peerName)) return seed;
}
}
}
// check local seed
if (this.mySeed == null) initMySeed();
name = mySeed.getName().toLowerCase();
if (mySeed.isProper(false) == null) nameLookupCache.put(name, mySeed.hash);
if (name.equals(peerName)) return mySeed;
name = this.mySeed.getName().toLowerCase();
if (this.mySeed.isProper(false) == null) this.nameLookupCache.put(name, this.mySeed.hash);
if (name.equals(peerName)) return this.mySeed;
// nothing found
return null;
}
@ -682,11 +680,11 @@ public final class yacySeedDB implements AlternativeDomainNames {
// local peer?
if (Domains.isThisHostIP(peerIP)) {
if (this.mySeed == null) initMySeed();
return mySeed;
return this.mySeed;
}
// then try to use the cache
final SoftReference<yacySeed> ref = ipLookupCache.get(peerIP);
final SoftReference<yacySeed> ref = this.ipLookupCache.get(peerIP);
if (ref != null) {
seed = ref.get();
if (seed != null) return seed;
@ -708,7 +706,7 @@ public final class yacySeedDB implements AlternativeDomainNames {
Log.logWarning("YACY","lookupByIP/Connected: address of seed " + seed.getName() + "/" + seed.hash + " is null.");
try {
badPeerHashes.put(ASCII.getBytes(seed.hash));
} catch (RowSpaceExceededException e1) {
} catch (final RowSpaceExceededException e1) {
Log.logException(e1);
break;
}
@ -719,13 +717,13 @@ public final class yacySeedDB implements AlternativeDomainNames {
}
seedIPAddress = Domains.dnsResolve(addressStr);
if (seedIPAddress == null) continue;
if (seed.isProper(false) == null) ipLookupCache.put(seedIPAddress, new SoftReference<yacySeed>(seed));
if (seed.isProper(false) == null) this.ipLookupCache.put(seedIPAddress, new SoftReference<yacySeed>(seed));
if (seedIPAddress.equals(peerIP)) return seed;
}
}
// delete bad peers
final Iterator<byte[]> i = badPeerHashes.iterator();
while (i.hasNext()) try {seedActiveDB.delete(i.next());} catch (final IOException e1) {Log.logException(e1);}
while (i.hasNext()) try {this.seedActiveDB.delete(i.next());} catch (final IOException e1) {Log.logException(e1);}
badPeerHashes.clear();
}
@ -741,7 +739,7 @@ public final class yacySeedDB implements AlternativeDomainNames {
Log.logWarning("YACY","lookupByIPDisconnected: address of seed " + seed.getName() + "/" + seed.hash + " is null.");
try {
badPeerHashes.put(UTF8.getBytes(seed.hash));
} catch (RowSpaceExceededException e1) {
} catch (final RowSpaceExceededException e1) {
Log.logException(e1);
break;
}
@ -752,13 +750,13 @@ public final class yacySeedDB implements AlternativeDomainNames {
}
seedIPAddress = Domains.dnsResolve(addressStr);
if (seedIPAddress == null) continue;
if (seed.isProper(false) == null) ipLookupCache.put(seedIPAddress, new SoftReference<yacySeed>(seed));
if (seed.isProper(false) == null) this.ipLookupCache.put(seedIPAddress, new SoftReference<yacySeed>(seed));
if (seedIPAddress.equals(peerIP)) return seed;
}
}
// delete bad peers
final Iterator<byte[]> i = badPeerHashes.iterator();
while (i.hasNext()) try {seedActiveDB.delete(i.next());} catch (final IOException e1) {Log.logException(e1);}
while (i.hasNext()) try {this.seedActiveDB.delete(i.next());} catch (final IOException e1) {Log.logException(e1);}
badPeerHashes.clear();
}
@ -774,7 +772,7 @@ public final class yacySeedDB implements AlternativeDomainNames {
}
seedIPAddress = Domains.dnsResolve(addressStr);
if (seedIPAddress == null) continue;
if (seed.isProper(false) == null) ipLookupCache.put(seedIPAddress, new SoftReference<yacySeed>(seed));
if (seed.isProper(false) == null) this.ipLookupCache.put(seedIPAddress, new SoftReference<yacySeed>(seed));
if (seedIPAddress.equals(peerIP)) return seed;
}
}
@ -782,22 +780,22 @@ public final class yacySeedDB implements AlternativeDomainNames {
// check local seed
if (this.mySeed == null) return null;
addressStr = mySeed.getPublicAddress();
addressStr = this.mySeed.getPublicAddress();
if (addressStr == null) return null;
if ((pos = addressStr.indexOf(':'))!= -1) {
addressStr = addressStr.substring(0,pos);
}
seedIPAddress = Domains.dnsResolve(addressStr);
if (seedIPAddress == null) return null;
if (mySeed.isProper(false) == null) ipLookupCache.put(seedIPAddress, new SoftReference<yacySeed>(mySeed));
if (seedIPAddress.equals(peerIP)) return mySeed;
if (this.mySeed.isProper(false) == null) this.ipLookupCache.put(seedIPAddress, new SoftReference<yacySeed>(this.mySeed));
if (seedIPAddress.equals(peerIP)) return this.mySeed;
// nothing found
return null;
}
private ArrayList<String> storeSeedList(final File seedFile, final boolean addMySeed) throws IOException {
PrintWriter pw = null;
final ArrayList<String> v = new ArrayList<String>(seedActiveDB.size() + 1);
final ArrayList<String> v = new ArrayList<String>(this.seedActiveDB.size() + 1);
try {
pw = new PrintWriter(new BufferedWriter(new FileWriter(seedFile)));
@ -806,7 +804,7 @@ public final class yacySeedDB implements AlternativeDomainNames {
String line;
if (this.mySeed == null) initMySeed();
if (addMySeed) {
line = mySeed.genSeedStr(null);
line = this.mySeed.genSeedStr(null);
v.add(line);
pw.print(line + serverCore.CRLF_STRING);
}
@ -825,7 +823,7 @@ public final class yacySeedDB implements AlternativeDomainNames {
// store some of the not-so-old passive peer seeds (limit: 1 day)
se = seedsDisconnected(true, false, null, (float) 0.0);
long timeout = System.currentTimeMillis() - (1000L * 60L * 60L * 24L);
final long timeout = System.currentTimeMillis() - (1000L * 60L * 60L * 24L);
while (se.hasNext()) {
ys = se.next();
if (ys != null) {
@ -962,8 +960,8 @@ public final class yacySeedDB implements AlternativeDomainNames {
// check local seed
if (seed == null) {
if (this.mySeed == null) initMySeed();
if (hash.equals(mySeed.hash))
seed = mySeed;
if (hash.equals(this.mySeed.hash))
seed = this.mySeed;
else return null;
}
return seed.getPublicAddress() + ((subdom == null) ? "" : ("/" + subdom));
@ -979,7 +977,7 @@ public final class yacySeedDB implements AlternativeDomainNames {
seed = lookupByName(domain);
if (seed == null) return null;
if (this.mySeed == null) initMySeed();
if ((seed == mySeed) && (!(seed.isOnline()))) {
if ((seed == this.mySeed) && (!(seed.isOnline()))) {
// take local ip instead of external
return Switchboard.getSwitchboard().myPublicIP() + ":" + Switchboard.getSwitchboard().getConfig("port", "8090") + ((subdom == null) ? "" : ("/" + subdom));
}
@ -992,10 +990,10 @@ public final class yacySeedDB implements AlternativeDomainNames {
public String targetAddress(final String targetHash) {
// find target address
String address;
if (targetHash.equals(this.mySeed().hash)) {
address = this.mySeed().getClusterAddress();
if (targetHash.equals(mySeed().hash)) {
address = mySeed().getClusterAddress();
} else {
final yacySeed targetSeed = this.getConnected(targetHash);
final yacySeed targetSeed = getConnected(targetHash);
if (targetSeed == null) { return null; }
address = targetSeed.getClusterAddress();
}
@ -1007,64 +1005,64 @@ public final class yacySeedDB implements AlternativeDomainNames {
private MapDataMining.mapIterator it;
private yacySeed nextSeed;
private MapDataMining database;
private final MapDataMining database;
private float minVersion;
private seedEnum(final boolean up, final boolean rot, final byte[] firstKey, final byte[] secondKey, final MapDataMining database, final float minVersion) {
this.database = database;
this.minVersion = minVersion;
try {
it = (firstKey == null) ? database.maps(up, rot) : database.maps(up, rot, firstKey, secondKey);
this.it = (firstKey == null) ? database.maps(up, rot) : database.maps(up, rot, firstKey, secondKey);
float version;
while (true) {
nextSeed = internalNext();
if (nextSeed == null) break;
version = nextSeed.getVersion();
this.nextSeed = internalNext();
if (this.nextSeed == null) break;
version = this.nextSeed.getVersion();
if (version >= this.minVersion || version == 0.0) break; // include 0.0 to access always developer peers
}
} catch (final IOException e) {
Log.logException(e);
yacyCore.log.logSevere("ERROR seedLinEnum: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
if (database == seedActiveDB) seedActiveDB = resetSeedTable(seedActiveDB, seedActiveDBFile);
if (database == seedPassiveDB) seedPassiveDB = resetSeedTable(seedPassiveDB, seedPassiveDBFile);
it = null;
if (database == yacySeedDB.this.seedActiveDB) yacySeedDB.this.seedActiveDB = resetSeedTable(yacySeedDB.this.seedActiveDB, yacySeedDB.this.seedActiveDBFile);
if (database == yacySeedDB.this.seedPassiveDB) yacySeedDB.this.seedPassiveDB = resetSeedTable(yacySeedDB.this.seedPassiveDB, yacySeedDB.this.seedPassiveDBFile);
this.it = null;
} catch (final kelondroException e) {
Log.logException(e);
yacyCore.log.logSevere("ERROR seedLinEnum: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
if (database == seedActiveDB) seedActiveDB = resetSeedTable(seedActiveDB, seedActiveDBFile);
if (database == seedPassiveDB) seedPassiveDB = resetSeedTable(seedPassiveDB, seedPassiveDBFile);
it = null;
if (database == yacySeedDB.this.seedActiveDB) yacySeedDB.this.seedActiveDB = resetSeedTable(yacySeedDB.this.seedActiveDB, yacySeedDB.this.seedActiveDBFile);
if (database == yacySeedDB.this.seedPassiveDB) yacySeedDB.this.seedPassiveDB = resetSeedTable(yacySeedDB.this.seedPassiveDB, yacySeedDB.this.seedPassiveDBFile);
this.it = null;
}
}
private seedEnum(final boolean up, final String field, final MapDataMining database) {
this.database = database;
try {
it = database.maps(up, field);
nextSeed = internalNext();
this.it = database.maps(up, field);
this.nextSeed = internalNext();
} catch (final kelondroException e) {
Log.logException(e);
yacyCore.log.logSevere("ERROR seedLinEnum: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
if (database == seedActiveDB) seedActiveDB = resetSeedTable(seedActiveDB, seedActiveDBFile);
if (database == seedPassiveDB) seedPassiveDB = resetSeedTable(seedPassiveDB, seedPassiveDBFile);
if (database == seedPotentialDB) seedPotentialDB = resetSeedTable(seedPotentialDB, seedPotentialDBFile);
it = null;
if (database == yacySeedDB.this.seedActiveDB) yacySeedDB.this.seedActiveDB = resetSeedTable(yacySeedDB.this.seedActiveDB, yacySeedDB.this.seedActiveDBFile);
if (database == yacySeedDB.this.seedPassiveDB) yacySeedDB.this.seedPassiveDB = resetSeedTable(yacySeedDB.this.seedPassiveDB, yacySeedDB.this.seedPassiveDBFile);
if (database == yacySeedDB.this.seedPotentialDB) yacySeedDB.this.seedPotentialDB = resetSeedTable(yacySeedDB.this.seedPotentialDB, yacySeedDB.this.seedPotentialDBFile);
this.it = null;
}
}
public boolean hasNext() {
return (nextSeed != null);
return (this.nextSeed != null);
}
private yacySeed internalNext() {
if (it == null || !(it.hasNext())) return null;
if (this.it == null || !(this.it.hasNext())) return null;
try {
Map<String, String> dna0;
ConcurrentHashMap<String, String> dna;
while (it.hasNext()) {
while (this.it.hasNext()) {
try {
dna0 = it.next();
} catch (OutOfMemoryError e) {
dna0 = this.it.next();
} catch (final OutOfMemoryError e) {
Log.logException(e);
dna0 = null;
}
@ -1085,27 +1083,27 @@ public final class yacySeedDB implements AlternativeDomainNames {
} catch (final Exception e) {
Log.logException(e);
yacyCore.log.logSevere("ERROR internalNext: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
if (database == seedActiveDB) seedActiveDB = resetSeedTable(seedActiveDB, seedActiveDBFile);
if (database == seedPassiveDB) seedPassiveDB = resetSeedTable(seedPassiveDB, seedPassiveDBFile);
if (database == seedPotentialDB) seedPotentialDB = resetSeedTable(seedPotentialDB, seedPotentialDBFile);
if (this.database == yacySeedDB.this.seedActiveDB) yacySeedDB.this.seedActiveDB = resetSeedTable(yacySeedDB.this.seedActiveDB, yacySeedDB.this.seedActiveDBFile);
if (this.database == yacySeedDB.this.seedPassiveDB) yacySeedDB.this.seedPassiveDB = resetSeedTable(yacySeedDB.this.seedPassiveDB, yacySeedDB.this.seedPassiveDBFile);
if (this.database == yacySeedDB.this.seedPotentialDB) yacySeedDB.this.seedPotentialDB = resetSeedTable(yacySeedDB.this.seedPotentialDB, yacySeedDB.this.seedPotentialDBFile);
return null;
}
}
public yacySeed next() {
final yacySeed seed = nextSeed;
final yacySeed seed = this.nextSeed;
float version;
try {while (true) {
nextSeed = internalNext();
if (nextSeed == null) break;
version = nextSeed.getVersion();
this.nextSeed = internalNext();
if (this.nextSeed == null) break;
version = this.nextSeed.getVersion();
if (version >= this.minVersion || version == 0.0) break; // include 0.0 to access always developer peers
}} catch (final kelondroException e) {
Log.logException(e);
// emergency reset
yacyCore.log.logSevere("seed-db emergency reset", e);
database.clear();
nextSeed = null;
this.database.clear();
this.nextSeed = null;
return null;
}
return seed;

6
source/net/yacy/cora/date/GenericFormatter.java
Unescape View File

@ -95,15 +95,15 @@ public class GenericFormatter extends AbstractFormatter implements DateFormatter
public String format() {
if (Math.abs(System.currentTimeMillis() - this.last_time) < this.maxCacheDiff) return this.last_format;
synchronized (this.dateFormat) {
// threads that had been waiting here may use the cache now instead of calculating the date again
final long time = System.currentTimeMillis();
if (Math.abs(time - this.last_time) < this.maxCacheDiff) return this.last_format;
// if the cache is not fresh, calculate the date
synchronized (this.dateFormat) {
if (Math.abs(time - this.last_time) < this.maxCacheDiff) return this.last_format;
this.last_format = this.dateFormat.format(new Date(time));
this.last_time = time;
}
this.last_time = time;
return this.last_format;
}

Write Preview
Loading…
Cancel
Save