small refactoring to prepare for new queues

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6173 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 16 years ago
parent 16efcd0366
commit 024744245c

@ -134,7 +134,7 @@ public final class CrawlSwitchboard {
", " + profilesPassiveFile.length()/1024); ", " + profilesPassiveFile.length()/1024);
// init queues // init queues
this.queuePreStack = new IndexingStack(peers, new File(queuesRoot, "urlNoticePreStack.stack"), this.profilesActiveCrawls); this.queuePreStack = new IndexingStack(peers, queuesRoot, "urlNoticePreStack.stack", this.profilesActiveCrawls);
} }
public void clear() { public void clear() {

@ -51,7 +51,11 @@ import de.anomic.yacy.yacyURL;
import de.anomic.yacy.logging.Log; import de.anomic.yacy.logging.Log;
public class IndexingStack { public class IndexingStack {
private static final byte[] zbytes = new byte[0];
//private static final String queuesPrefix = "indexing";
//private static final String sbQueueName = "sbQueueStack";
protected final CrawlProfile profiles; protected final CrawlProfile profiles;
protected final RecordStack sbQueueStack; protected final RecordStack sbQueueStack;
protected final yacySeedDB peers; protected final yacySeedDB peers;
@ -59,12 +63,13 @@ public class IndexingStack {
public IndexingStack( public IndexingStack(
final yacySeedDB peers, final yacySeedDB peers,
final File sbQueueStackPath, final File queuesRoot,
final String sbQueueStackName,
final CrawlProfile profiles) { final CrawlProfile profiles) {
this.profiles = profiles; this.profiles = profiles;
this.peers = peers; this.peers = peers;
this.queueInProcess = new ConcurrentHashMap<String, QueueEntry>(); this.queueInProcess = new ConcurrentHashMap<String, QueueEntry>();
this.sbQueueStack = RecordStack.open(sbQueueStackPath, rowdef); this.sbQueueStack = RecordStack.open(new File(queuesRoot, sbQueueStackName), rowdef);
} }
public static final Row rowdef = new Row( public static final Row rowdef = new Row(
@ -92,12 +97,12 @@ public class IndexingStack {
if (sbQueueStack == null) return; // may occur during shutdown if (sbQueueStack == null) return; // may occur during shutdown
sbQueueStack.push(sbQueueStack.row().newEntry(new byte[][]{ sbQueueStack.push(sbQueueStack.row().newEntry(new byte[][]{
entry.url.toString().getBytes(), entry.url.toString().getBytes(),
(entry.referrerHash == null) ? "".getBytes() : entry.referrerHash.getBytes(), (entry.referrerHash == null) ? zbytes : entry.referrerHash.getBytes(),
Base64Order.enhancedCoder.encodeLong((entry.ifModifiedSince == null) ? 0 : entry.ifModifiedSince.getTime(), 11).getBytes(), Base64Order.enhancedCoder.encodeLong((entry.ifModifiedSince == null) ? 0 : entry.ifModifiedSince.getTime(), 11).getBytes(),
new byte[]{entry.flags}, new byte[]{entry.flags},
(entry.initiator == null) ? "".getBytes() : entry.initiator.getBytes(), (entry.initiator == null) ? zbytes : entry.initiator.getBytes(),
Base64Order.enhancedCoder.encodeLong(entry.depth, rowdef.width(5)).getBytes(), Base64Order.enhancedCoder.encodeLong(entry.depth, rowdef.width(5)).getBytes(),
(entry.profileHandle == null) ? "".getBytes() : entry.profileHandle.getBytes(), (entry.profileHandle == null) ? zbytes : entry.profileHandle.getBytes(),
(entry.anchorName == null) ? "-".getBytes("UTF-8") : entry.anchorName.getBytes("UTF-8") (entry.anchorName == null) ? "-".getBytes("UTF-8") : entry.anchorName.getBytes("UTF-8")
})); }));
} }

@ -1262,7 +1262,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<IndexingStack.
if (crawler.queuePreStack.size() == sizeBefore) { if (crawler.queuePreStack.size() == sizeBefore) {
// this is a severe problem: because this time a null is returned, it means that this status will last forever // this is a severe problem: because this time a null is returned, it means that this status will last forever
// to re-enable use of the sbQueue, it must be emptied completely // to re-enable use of the sbQueue, it must be emptied completely
log.logSevere("deQueue: does not shrink after pop() == null. Emergency reset of " + crawler.queuePreStack.file()); log.logSevere("deQueue: does not shrink after pop() == null. Emergency reset of sbQueue");
crawler.queuePreStack.clear(); crawler.queuePreStack.clear();
} }
return null; return null;

Loading…
Cancel
Save