boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

fixed bug introduced in SVN 5756 in EcoTable.put()

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5759 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 16 years ago
parent 587838bd09
commit 029495e64d
6 changed files with 42 additions and 22 deletions
Show Stats Download Patch File Download Diff File

6
source/de/anomic/crawler/Balancer.java
Unescape View File

@ -356,12 +356,14 @@ public class Balancer {
public synchronized void push(final CrawlEntry entry) throws IOException {
assert entry != null;
if (urlFileIndex.has(entry.url().hash().getBytes())) {
//serverLog.logWarning("BALANCER", "double-check has failed for urlhash " + entry.url().hash() + " in " + stackname + " - fixed");
Log.logWarning("BALANCER", "double-check has failed for urlhash " + entry.url().hash() + " in " + stackname + " - fixed");
return;
}
// add to index
int s = urlFileIndex.size();
urlFileIndex.put(entry.toRow());
assert s < urlFileIndex.size();
// add the hash to a queue
pushHashToDomainStacks(entry.url().hash(), true);
@ -617,7 +619,7 @@ public class Balancer {
// if we need to flush anything, then flush the domain stack first,
// to avoid that new urls get hidden by old entries from the file stack
if (urlRAMStack == null) return 0;
// ensure that the domain stacks are filled enough
shiftFileToDomStacks(count);

1
source/de/anomic/crawler/CrawlQueues.java
Unescape View File

@ -284,6 +284,7 @@ public class CrawlQueues {
*/
private boolean crawlIsPossible(int stackType, final String type) {
int value;
//System.out.println("stacksize = " + noticeURL.stackSize(stackType));
if (noticeURL.stackSize(stackType) == 0) {
//log.logDebug("GlobalCrawl: queue is empty");
return false;

48
source/de/anomic/crawler/CrawlStacker.java
Unescape View File

@ -277,23 +277,37 @@ public final class CrawlStacker {
) /* qualified */;
if (!local && !global && !remote && !proxy) {
this.log.logSevere("URL '" + entry.url().toString() + "' cannot be crawled. initiator = " + entry.initiator() + ", profile.handle = " + profile.handle());
} else {
if (global) {
// it may be possible that global == true and local == true, so do not check an error case against it
if (proxy) this.log.logWarning("URL '" + entry.url().toString() + "' has conflicting initiator properties: global = true, proxy = true, initiator = " + entry.initiator() + ", profile.handle = " + profile.handle());
if (remote) this.log.logWarning("URL '" + entry.url().toString() + "' has conflicting initiator properties: global = true, remote = true, initiator = " + entry.initiator() + ", profile.handle = " + profile.handle());
nextQueue.noticeURL.push(NoticedURL.STACK_TYPE_LIMIT, entry);
} else if (local) {
if (proxy) this.log.logWarning("URL '" + entry.url().toString() + "' has conflicting initiator properties: local = true, proxy = true, initiator = " + entry.initiator() + ", profile.handle = " + profile.handle());
if (remote) this.log.logWarning("URL '" + entry.url().toString() + "' has conflicting initiator properties: local = true, remote = true, initiator = " + entry.initiator() + ", profile.handle = " + profile.handle());
nextQueue.noticeURL.push(NoticedURL.STACK_TYPE_CORE, entry);
} else if (proxy) {
if (remote) this.log.logWarning("URL '" + entry.url().toString() + "' has conflicting initiator properties: proxy = true, remote = true, initiator = " + entry.initiator() + ", profile.handle = " + profile.handle());
nextQueue.noticeURL.push(NoticedURL.STACK_TYPE_CORE, entry);
} else if (remote) {
nextQueue.noticeURL.push(NoticedURL.STACK_TYPE_REMOTE, entry);
}
String error = "URL '" + entry.url().toString() + "' cannot be crawled. initiator = " + entry.initiator() + ", profile.handle = " + profile.handle();
this.log.logSevere(error);
return error;
}
if (global) {
// it may be possible that global == true and local == true, so do not check an error case against it
if (proxy) this.log.logWarning("URL '" + entry.url().toString() + "' has conflicting initiator properties: global = true, proxy = true, initiator = " + entry.initiator() + ", profile.handle = " + profile.handle());
if (remote) this.log.logWarning("URL '" + entry.url().toString() + "' has conflicting initiator properties: global = true, remote = true, initiator = " + entry.initiator() + ", profile.handle = " + profile.handle());
int b = nextQueue.noticeURL.stackSize(NoticedURL.STACK_TYPE_LIMIT);
nextQueue.noticeURL.push(NoticedURL.STACK_TYPE_LIMIT, entry);
assert b < nextQueue.noticeURL.stackSize(NoticedURL.STACK_TYPE_LIMIT);
this.log.logInfo("stacked/global: " + entry.url().toString() + ", stacksize = " + nextQueue.noticeURL.stackSize(NoticedURL.STACK_TYPE_LIMIT));
} else if (local) {
if (proxy) this.log.logWarning("URL '" + entry.url().toString() + "' has conflicting initiator properties: local = true, proxy = true, initiator = " + entry.initiator() + ", profile.handle = " + profile.handle());
if (remote) this.log.logWarning("URL '" + entry.url().toString() + "' has conflicting initiator properties: local = true, remote = true, initiator = " + entry.initiator() + ", profile.handle = " + profile.handle());
int b = nextQueue.noticeURL.stackSize(NoticedURL.STACK_TYPE_CORE);
nextQueue.noticeURL.push(NoticedURL.STACK_TYPE_CORE, entry);
assert b < nextQueue.noticeURL.stackSize(NoticedURL.STACK_TYPE_CORE);
this.log.logInfo("stacked/local: " + entry.url().toString() + ", stacksize = " + nextQueue.noticeURL.stackSize(NoticedURL.STACK_TYPE_CORE));
} else if (proxy) {
if (remote) this.log.logWarning("URL '" + entry.url().toString() + "' has conflicting initiator properties: proxy = true, remote = true, initiator = " + entry.initiator() + ", profile.handle = " + profile.handle());
int b = nextQueue.noticeURL.stackSize(NoticedURL.STACK_TYPE_CORE);
nextQueue.noticeURL.push(NoticedURL.STACK_TYPE_CORE, entry);
assert b < nextQueue.noticeURL.stackSize(NoticedURL.STACK_TYPE_CORE);
this.log.logInfo("stacked/proxy: " + entry.url().toString() + ", stacksize = " + nextQueue.noticeURL.stackSize(NoticedURL.STACK_TYPE_CORE));
} else if (remote) {
int b = nextQueue.noticeURL.stackSize(NoticedURL.STACK_TYPE_REMOTE);
nextQueue.noticeURL.push(NoticedURL.STACK_TYPE_REMOTE, entry);
assert b < nextQueue.noticeURL.stackSize(NoticedURL.STACK_TYPE_REMOTE);
this.log.logInfo("stacked/remote: " + entry.url().toString() + ", stacksize = " + nextQueue.noticeURL.stackSize(NoticedURL.STACK_TYPE_REMOTE));
}
return null;

2
source/de/anomic/kelondro/table/EcoTable.java
Unescape View File

@ -421,7 +421,7 @@ public class EcoTable implements ObjectIndex {
assert table == null || table.size() == index.size();
assert row != null;
assert row.bytes() != null;
if (file == null || table == null || row == null || row.bytes() == null) return;
if (file == null || row == null || row.bytes() == null) return;
final int i = index.get(row.getPrimaryKeyBytes());
if (i == -1) {
addUnique(row);

1
source/de/anomic/kelondro/text/ReferenceContainerArray.java
Unescape View File

@ -66,6 +66,7 @@ public final class ReferenceContainerArray {
payloadrow.primaryKeyLength,
wordOrder,
0);
assert merger != null;
this.merger = merger;
}

6
source/de/anomic/plasma/plasmaWordIndex.java
Unescape View File

@ -135,11 +135,11 @@ public final class plasmaWordIndex {
}
}
}
this.merger = (useCell) ? new IODispatcher(1, 1) : null;
if (this.merger != null) this.merger.start();
// check if the peer has migrated the index
if (new File(indexPrimaryTextLocation, "RICOLLECTION").exists()) {
this.merger = (useCell) ? new IODispatcher(1, 1) : null;
if (this.merger != null) this.merger.start();
this.index = (useCell) ?
new IndexCollectionMigration(
indexPrimaryTextLocation,
@ -159,6 +159,8 @@ public final class plasmaWordIndex {
redundancy,
log);
} else {
this.merger = new IODispatcher(1, 1);
this.merger.start();
this.index = new IndexCell(
new File(indexPrimaryTextLocation, "RICELL"),
wordOrder,

Write Preview
Loading…
Cancel
Save