fixed bug introduced in SVN 5756 in EcoTable.put()

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5759 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 16 years ago
parent 587838bd09
commit 029495e64d

@ -356,12 +356,14 @@ public class Balancer {
public synchronized void push(final CrawlEntry entry) throws IOException { public synchronized void push(final CrawlEntry entry) throws IOException {
assert entry != null; assert entry != null;
if (urlFileIndex.has(entry.url().hash().getBytes())) { if (urlFileIndex.has(entry.url().hash().getBytes())) {
//serverLog.logWarning("BALANCER", "double-check has failed for urlhash " + entry.url().hash() + " in " + stackname + " - fixed"); Log.logWarning("BALANCER", "double-check has failed for urlhash " + entry.url().hash() + " in " + stackname + " - fixed");
return; return;
} }
// add to index // add to index
int s = urlFileIndex.size();
urlFileIndex.put(entry.toRow()); urlFileIndex.put(entry.toRow());
assert s < urlFileIndex.size();
// add the hash to a queue // add the hash to a queue
pushHashToDomainStacks(entry.url().hash(), true); pushHashToDomainStacks(entry.url().hash(), true);

@ -284,6 +284,7 @@ public class CrawlQueues {
*/ */
private boolean crawlIsPossible(int stackType, final String type) { private boolean crawlIsPossible(int stackType, final String type) {
int value; int value;
//System.out.println("stacksize = " + noticeURL.stackSize(stackType));
if (noticeURL.stackSize(stackType) == 0) { if (noticeURL.stackSize(stackType) == 0) {
//log.logDebug("GlobalCrawl: queue is empty"); //log.logDebug("GlobalCrawl: queue is empty");
return false; return false;

@ -277,23 +277,37 @@ public final class CrawlStacker {
) /* qualified */; ) /* qualified */;
if (!local && !global && !remote && !proxy) { if (!local && !global && !remote && !proxy) {
this.log.logSevere("URL '" + entry.url().toString() + "' cannot be crawled. initiator = " + entry.initiator() + ", profile.handle = " + profile.handle()); String error = "URL '" + entry.url().toString() + "' cannot be crawled. initiator = " + entry.initiator() + ", profile.handle = " + profile.handle();
} else { this.log.logSevere(error);
if (global) { return error;
// it may be possible that global == true and local == true, so do not check an error case against it }
if (proxy) this.log.logWarning("URL '" + entry.url().toString() + "' has conflicting initiator properties: global = true, proxy = true, initiator = " + entry.initiator() + ", profile.handle = " + profile.handle());
if (remote) this.log.logWarning("URL '" + entry.url().toString() + "' has conflicting initiator properties: global = true, remote = true, initiator = " + entry.initiator() + ", profile.handle = " + profile.handle()); if (global) {
nextQueue.noticeURL.push(NoticedURL.STACK_TYPE_LIMIT, entry); // it may be possible that global == true and local == true, so do not check an error case against it
} else if (local) { if (proxy) this.log.logWarning("URL '" + entry.url().toString() + "' has conflicting initiator properties: global = true, proxy = true, initiator = " + entry.initiator() + ", profile.handle = " + profile.handle());
if (proxy) this.log.logWarning("URL '" + entry.url().toString() + "' has conflicting initiator properties: local = true, proxy = true, initiator = " + entry.initiator() + ", profile.handle = " + profile.handle()); if (remote) this.log.logWarning("URL '" + entry.url().toString() + "' has conflicting initiator properties: global = true, remote = true, initiator = " + entry.initiator() + ", profile.handle = " + profile.handle());
if (remote) this.log.logWarning("URL '" + entry.url().toString() + "' has conflicting initiator properties: local = true, remote = true, initiator = " + entry.initiator() + ", profile.handle = " + profile.handle()); int b = nextQueue.noticeURL.stackSize(NoticedURL.STACK_TYPE_LIMIT);
nextQueue.noticeURL.push(NoticedURL.STACK_TYPE_CORE, entry); nextQueue.noticeURL.push(NoticedURL.STACK_TYPE_LIMIT, entry);
} else if (proxy) { assert b < nextQueue.noticeURL.stackSize(NoticedURL.STACK_TYPE_LIMIT);
if (remote) this.log.logWarning("URL '" + entry.url().toString() + "' has conflicting initiator properties: proxy = true, remote = true, initiator = " + entry.initiator() + ", profile.handle = " + profile.handle()); this.log.logInfo("stacked/global: " + entry.url().toString() + ", stacksize = " + nextQueue.noticeURL.stackSize(NoticedURL.STACK_TYPE_LIMIT));
nextQueue.noticeURL.push(NoticedURL.STACK_TYPE_CORE, entry); } else if (local) {
} else if (remote) { if (proxy) this.log.logWarning("URL '" + entry.url().toString() + "' has conflicting initiator properties: local = true, proxy = true, initiator = " + entry.initiator() + ", profile.handle = " + profile.handle());
nextQueue.noticeURL.push(NoticedURL.STACK_TYPE_REMOTE, entry); if (remote) this.log.logWarning("URL '" + entry.url().toString() + "' has conflicting initiator properties: local = true, remote = true, initiator = " + entry.initiator() + ", profile.handle = " + profile.handle());
} int b = nextQueue.noticeURL.stackSize(NoticedURL.STACK_TYPE_CORE);
nextQueue.noticeURL.push(NoticedURL.STACK_TYPE_CORE, entry);
assert b < nextQueue.noticeURL.stackSize(NoticedURL.STACK_TYPE_CORE);
this.log.logInfo("stacked/local: " + entry.url().toString() + ", stacksize = " + nextQueue.noticeURL.stackSize(NoticedURL.STACK_TYPE_CORE));
} else if (proxy) {
if (remote) this.log.logWarning("URL '" + entry.url().toString() + "' has conflicting initiator properties: proxy = true, remote = true, initiator = " + entry.initiator() + ", profile.handle = " + profile.handle());
int b = nextQueue.noticeURL.stackSize(NoticedURL.STACK_TYPE_CORE);
nextQueue.noticeURL.push(NoticedURL.STACK_TYPE_CORE, entry);
assert b < nextQueue.noticeURL.stackSize(NoticedURL.STACK_TYPE_CORE);
this.log.logInfo("stacked/proxy: " + entry.url().toString() + ", stacksize = " + nextQueue.noticeURL.stackSize(NoticedURL.STACK_TYPE_CORE));
} else if (remote) {
int b = nextQueue.noticeURL.stackSize(NoticedURL.STACK_TYPE_REMOTE);
nextQueue.noticeURL.push(NoticedURL.STACK_TYPE_REMOTE, entry);
assert b < nextQueue.noticeURL.stackSize(NoticedURL.STACK_TYPE_REMOTE);
this.log.logInfo("stacked/remote: " + entry.url().toString() + ", stacksize = " + nextQueue.noticeURL.stackSize(NoticedURL.STACK_TYPE_REMOTE));
} }
return null; return null;

@ -421,7 +421,7 @@ public class EcoTable implements ObjectIndex {
assert table == null || table.size() == index.size(); assert table == null || table.size() == index.size();
assert row != null; assert row != null;
assert row.bytes() != null; assert row.bytes() != null;
if (file == null || table == null || row == null || row.bytes() == null) return; if (file == null || row == null || row.bytes() == null) return;
final int i = index.get(row.getPrimaryKeyBytes()); final int i = index.get(row.getPrimaryKeyBytes());
if (i == -1) { if (i == -1) {
addUnique(row); addUnique(row);

@ -66,6 +66,7 @@ public final class ReferenceContainerArray {
payloadrow.primaryKeyLength, payloadrow.primaryKeyLength,
wordOrder, wordOrder,
0); 0);
assert merger != null;
this.merger = merger; this.merger = merger;
} }

@ -135,11 +135,11 @@ public final class plasmaWordIndex {
} }
} }
} }
this.merger = (useCell) ? new IODispatcher(1, 1) : null;
if (this.merger != null) this.merger.start();
// check if the peer has migrated the index // check if the peer has migrated the index
if (new File(indexPrimaryTextLocation, "RICOLLECTION").exists()) { if (new File(indexPrimaryTextLocation, "RICOLLECTION").exists()) {
this.merger = (useCell) ? new IODispatcher(1, 1) : null;
if (this.merger != null) this.merger.start();
this.index = (useCell) ? this.index = (useCell) ?
new IndexCollectionMigration( new IndexCollectionMigration(
indexPrimaryTextLocation, indexPrimaryTextLocation,
@ -159,6 +159,8 @@ public final class plasmaWordIndex {
redundancy, redundancy,
log); log);
} else { } else {
this.merger = new IODispatcher(1, 1);
this.merger.start();
this.index = new IndexCell( this.index = new IndexCell(
new File(indexPrimaryTextLocation, "RICELL"), new File(indexPrimaryTextLocation, "RICELL"),
wordOrder, wordOrder,

Loading…
Cancel
Save