small refactoring to prepare for new queues

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6173 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 16 years ago
parent 16efcd0366
commit 024744245c

@ -134,7 +134,7 @@ public final class CrawlSwitchboard {
", " + profilesPassiveFile.length()/1024);
// init queues
this.queuePreStack = new IndexingStack(peers, new File(queuesRoot, "urlNoticePreStack.stack"), this.profilesActiveCrawls);
this.queuePreStack = new IndexingStack(peers, queuesRoot, "urlNoticePreStack.stack", this.profilesActiveCrawls);
}
public void clear() {

@ -51,7 +51,11 @@ import de.anomic.yacy.yacyURL;
import de.anomic.yacy.logging.Log;
public class IndexingStack {
private static final byte[] zbytes = new byte[0];
//private static final String queuesPrefix = "indexing";
//private static final String sbQueueName = "sbQueueStack";
protected final CrawlProfile profiles;
protected final RecordStack sbQueueStack;
protected final yacySeedDB peers;
@ -59,12 +63,13 @@ public class IndexingStack {
public IndexingStack(
final yacySeedDB peers,
final File sbQueueStackPath,
final File queuesRoot,
final String sbQueueStackName,
final CrawlProfile profiles) {
this.profiles = profiles;
this.peers = peers;
this.queueInProcess = new ConcurrentHashMap<String, QueueEntry>();
this.sbQueueStack = RecordStack.open(sbQueueStackPath, rowdef);
this.sbQueueStack = RecordStack.open(new File(queuesRoot, sbQueueStackName), rowdef);
}
public static final Row rowdef = new Row(
@ -92,12 +97,12 @@ public class IndexingStack {
if (sbQueueStack == null) return; // may occur during shutdown
sbQueueStack.push(sbQueueStack.row().newEntry(new byte[][]{
entry.url.toString().getBytes(),
(entry.referrerHash == null) ? "".getBytes() : entry.referrerHash.getBytes(),
(entry.referrerHash == null) ? zbytes : entry.referrerHash.getBytes(),
Base64Order.enhancedCoder.encodeLong((entry.ifModifiedSince == null) ? 0 : entry.ifModifiedSince.getTime(), 11).getBytes(),
new byte[]{entry.flags},
(entry.initiator == null) ? "".getBytes() : entry.initiator.getBytes(),
(entry.initiator == null) ? zbytes : entry.initiator.getBytes(),
Base64Order.enhancedCoder.encodeLong(entry.depth, rowdef.width(5)).getBytes(),
(entry.profileHandle == null) ? "".getBytes() : entry.profileHandle.getBytes(),
(entry.profileHandle == null) ? zbytes : entry.profileHandle.getBytes(),
(entry.anchorName == null) ? "-".getBytes("UTF-8") : entry.anchorName.getBytes("UTF-8")
}));
}

@ -1262,7 +1262,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<IndexingStack.
if (crawler.queuePreStack.size() == sizeBefore) {
// this is a severe problem: because this time a null is returned, it means that this status will last forever
// to re-enable use of the sbQueue, it must be emptied completely
log.logSevere("deQueue: does not shrink after pop() == null. Emergency reset of " + crawler.queuePreStack.file());
log.logSevere("deQueue: does not shrink after pop() == null. Emergency reset of sbQueue");
crawler.queuePreStack.clear();
}
return null;

Loading…
Cancel
Save