*) Automatic Reset of Stack Crawler DB on startup errors

See: http://www.yacy-forum.de/viewtopic.php?t=1432

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@1045 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
theli 20 years ago
parent fb766413d1
commit 6f9f8ed8f8

@ -60,6 +60,7 @@ import org.apache.commons.pool.impl.GenericObjectPool;
import de.anomic.data.robotsParser; import de.anomic.data.robotsParser;
import de.anomic.http.httpc; import de.anomic.http.httpc;
import de.anomic.kelondro.kelondroException;
import de.anomic.kelondro.kelondroTree; import de.anomic.kelondro.kelondroTree;
import de.anomic.kelondro.kelondroRecords.Node; import de.anomic.kelondro.kelondroRecords.Node;
import de.anomic.server.serverCodings; import de.anomic.server.serverCodings;
@ -467,7 +468,7 @@ public final class plasmaCrawlStacker {
private final serverSemaphore readSync; private final serverSemaphore readSync;
private final serverSemaphore writeSync; private final serverSemaphore writeSync;
private final LinkedList urlEntryHashCache; private final LinkedList urlEntryHashCache;
private final kelondroTree urlEntryCache; private kelondroTree urlEntryCache;
public stackCrawlQueue(File cacheStacksPath, int bufferkb) throws IOException { public stackCrawlQueue(File cacheStacksPath, int bufferkb) throws IOException {
// init the read semaphore // init the read semaphore
@ -486,7 +487,7 @@ public final class plasmaCrawlStacker {
if (cacheFile.exists()) { if (cacheFile.exists()) {
// open existing cache // open existing cache
this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400); this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400);
try {
// loop through the list and fill the messageList with url hashs // loop through the list and fill the messageList with url hashs
Iterator iter = this.urlEntryCache.nodeIterator(true,false); Iterator iter = this.urlEntryCache.nodeIterator(true,false);
Node n; Node n;
@ -500,6 +501,15 @@ public final class plasmaCrawlStacker {
this.urlEntryHashCache.add(urlHash); this.urlEntryHashCache.add(urlHash);
this.readSync.V(); this.readSync.V();
} }
} catch (kelondroException e) {
/* if we have an error, we start with a fresh database */
plasmaCrawlStacker.this.log.logSevere("Unable to initialize crawl stacker queue. Reseting DB.\n",e);
// deleting old db and creating a new db
try {this.urlEntryCache.close();}catch(Exception ex){}
cacheFile.delete();
this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, plasmaCrawlNURL.ce);
}
} else { } else {
// create new cache // create new cache
cacheFile.getParentFile().mkdirs(); cacheFile.getParentFile().mkdirs();

Loading…
Cancel
Save