boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

*) simplifying code by moving closeTransferIndexes into final block

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@1522 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
theli 19 years ago
parent 0a30868c75
commit 008bcb7fb8
1 changed files with 103 additions and 96 deletions
Show Stats Download Patch File Download Diff File

199
source/de/anomic/plasma/plasmaWordIndexDistribution.java
Unescape View File

@ -121,6 +121,10 @@ public final class plasmaWordIndexDistribution {
} }
} }
private boolean isClosed() {
return (this.closed || Thread.currentThread().isInterrupted());
}
public boolean job() { public boolean job() {
if (this.closed) { if (this.closed) {
@ -192,111 +196,114 @@ public final class plasmaWordIndexDistribution {
public int performTransferIndex(int indexCount, int peerCount, boolean delete) { public int performTransferIndex(int indexCount, int peerCount, boolean delete) {
if ((yacyCore.seedDB == null) || (yacyCore.seedDB.sizeConnected() == 0)) return -1; if ((yacyCore.seedDB == null) || (yacyCore.seedDB.sizeConnected() == 0)) return -1;
// collect index // collect index
String startPointHash = selectTransferStart(); plasmaWordIndexEntryContainer[] indexContainers = null;
log.logFine("Selected hash " + startPointHash + " as start point for index distribution, distance = " + yacyDHTAction.dhtDistance(yacyCore.seedDB.mySeed.hash, startPointHash)); try {
Object[] selectResult = selectTransferContainers(startPointHash, indexCount); String startPointHash = selectTransferStart();
plasmaWordIndexEntryContainer[] indexContainers = (plasmaWordIndexEntryContainer[]) selectResult[0]; this.log.logFine("Selected hash " + startPointHash + " as start point for index distribution, distance = " + yacyDHTAction.dhtDistance(yacyCore.seedDB.mySeed.hash, startPointHash));
HashMap urlCache = (HashMap) selectResult[1]; // String (url-hash) / plasmaCrawlLURL.Entry
if ((indexContainers == null) || (indexContainers.length == 0)) { Object[] selectResult = selectTransferContainers(startPointHash, indexCount);
log.logFine("No index available for index transfer, hash start-point " + startPointHash); indexContainers = (plasmaWordIndexEntryContainer[]) selectResult[0];
return -1; HashMap urlCache = (HashMap) selectResult[1]; // String (url-hash) / plasmaCrawlLURL.Entry
}
// count the indexes again, can be smaller as expected if ((indexContainers == null) || (indexContainers.length == 0)) {
indexCount = 0; this.log.logFine("No index available for index transfer, hash start-point " + startPointHash);
for (int i = 0; i < indexContainers.length; i++) { return -1;
indexCount += indexContainers[i].size(); }
} // count the indexes again, can be smaller as expected
if (indexCount < 50) { indexCount = 0;
log.logFine("Too few (" + indexCount + ") indexes selected for transfer."); for (int i = 0; i < indexContainers.length; i++) {
closeTransferIndexes(indexContainers); indexCount += indexContainers[i].size();
return -1; // failed }
} if (indexCount < 50) {
this.log.logFine("Too few (" + indexCount + ") indexes selected for transfer.");
// find start point for DHT-selection return -1; // failed
String keyhash = indexContainers[indexContainers.length - 1].wordHash(); // DHT targets must have greater hashes }
// find a list of DHT-peers // find start point for DHT-selection
yacySeed[] seeds = new yacySeed[peerCount + 10]; String keyhash = indexContainers[indexContainers.length - 1].wordHash(); // DHT targets must have greater hashes
int hc0 = 0;
double ownDistance = Math.min(yacyDHTAction.dhtDistance(yacyCore.seedDB.mySeed.hash, indexContainers[0].wordHash()), // find a list of DHT-peers
yacyDHTAction.dhtDistance(yacyCore.seedDB.mySeed.hash, indexContainers[indexContainers.length - 1].wordHash())); yacySeed[] seeds = new yacySeed[peerCount + 10];
double maxDistance = Math.min(ownDistance, 0.4); int hc0 = 0;
synchronized (yacyCore.dhtAgent) { double ownDistance = Math.min(yacyDHTAction.dhtDistance(yacyCore.seedDB.mySeed.hash, indexContainers[0].wordHash()),
double avdist; yacyDHTAction.dhtDistance(yacyCore.seedDB.mySeed.hash, indexContainers[indexContainers.length - 1].wordHash()));
Enumeration e = yacyCore.dhtAgent.getAcceptRemoteIndexSeeds(keyhash); double maxDistance = Math.min(ownDistance, 0.4);
while ((e.hasMoreElements()) && (hc0 < seeds.length)) { synchronized (yacyCore.dhtAgent) {
if (closed) { double avdist;
log.logSevere("Index distribution interrupted by close, nothing deleted locally."); Enumeration e = yacyCore.dhtAgent.getAcceptRemoteIndexSeeds(keyhash);
return -1; // interrupted while ((e.hasMoreElements()) && (hc0 < seeds.length)) {
} if (this.isClosed()) {
seeds[hc0] = (yacySeed) e.nextElement(); this.log.logSevere("Index distribution interrupted by close, nothing deleted locally.");
if (seeds[hc0] != null) { return -1; // interrupted
avdist = Math.max(yacyDHTAction.dhtDistance(seeds[hc0].hash, indexContainers[0].wordHash()), }
yacyDHTAction.dhtDistance(seeds[hc0].hash, indexContainers[indexContainers.length - 1].wordHash())); seeds[hc0] = (yacySeed) e.nextElement();
if (avdist < maxDistance) { if (seeds[hc0] != null) {
log.logInfo("Selected " + ((hc0 < peerCount) ? "primary" : "reserve") + " DHT target peer " + seeds[hc0].getName() + ":" + seeds[hc0].hash + ", distance = " + avdist); avdist = Math.max(yacyDHTAction.dhtDistance(seeds[hc0].hash, indexContainers[0].wordHash()),
hc0++; yacyDHTAction.dhtDistance(seeds[hc0].hash, indexContainers[indexContainers.length - 1].wordHash()));
if (avdist < maxDistance) {
this.log.logInfo("Selected " + ((hc0 < peerCount) ? "primary" : "reserve") + " DHT target peer " + seeds[hc0].getName() + ":" + seeds[hc0].hash + ", distance = " + avdist);
hc0++;
}
} }
} }
e = null; // finish enumeration
} }
e = null; // finish enumeration
} if (hc0 < peerCount) {
this.log.logWarning("found not enough (" + hc0 + ") peers for distribution");
if (hc0 < peerCount) { return -1; // failed
log.logWarning("found not enough (" + hc0 + ") peers for distribution");
closeTransferIndexes(indexContainers);
return -1; // failed
}
// send away the indexes to all these indexes
String error;
String peerNames = "";
long start;
int hc1 = 0;
for (int i = 0; i < hc0; i++) {
if (closed) {
log.logSevere("Index distribution interrupted by close, nothing deleted locally.");
return -1; // interrupted
} }
start = System.currentTimeMillis();
error = yacyClient.transferIndex( // send away the indexes to all these indexes
seeds[i], String error;
indexContainers, String peerNames = "";
urlCache, long start;
this.gzipBody4Distribution, int hc1 = 0;
this.timeout4Distribution); for (int i = 0; i < hc0; i++) {
if (error == null) { if (this.isClosed()) {
log.logInfo("Index transfer of " + indexCount + " words [" + indexContainers[0].wordHash() + " .. " + indexContainers[indexContainers.length - 1].wordHash() + "] to peer " + seeds[i].getName() + ":" + seeds[i].hash + " in " + ((System.currentTimeMillis() - start) / 1000) this.log.logSevere("Index distribution interrupted by close, nothing deleted locally.");
+ " seconds successfull (" + (1000 * indexCount / (System.currentTimeMillis() - start + 1)) + " words/s)"); return -1; // interrupted
peerNames += ", " + seeds[i].getName(); }
hc1++; start = System.currentTimeMillis();
} else { error = yacyClient.transferIndex(
log.logWarning("Index transfer to peer " + seeds[i].getName() + ":" + seeds[i].hash + " failed:'" + error + "', disconnecting peer"); seeds[i],
yacyCore.peerActions.peerDeparture(seeds[i]); indexContainers,
urlCache,
this.gzipBody4Distribution,
this.timeout4Distribution);
if (error == null) {
this.log.logInfo("Index transfer of " + indexCount + " words [" + indexContainers[0].wordHash() + " .. " + indexContainers[indexContainers.length - 1].wordHash() + "] to peer " + seeds[i].getName() + ":" + seeds[i].hash + " in " + ((System.currentTimeMillis() - start) / 1000)
+ " seconds successfull (" + (1000 * indexCount / (System.currentTimeMillis() - start + 1)) + " words/s)");
peerNames += ", " + seeds[i].getName();
hc1++;
} else {
this.log.logWarning("Index transfer to peer " + seeds[i].getName() + ":" + seeds[i].hash + " failed:'" + error + "', disconnecting peer");
yacyCore.peerActions.peerDeparture(seeds[i]);
}
if (hc1 >= peerCount) break;
} }
if (hc1 >= peerCount) break; if (peerNames.length() > 0) peerNames = peerNames.substring(2); // remove comma
}
if (peerNames.length() > 0) peerNames = peerNames.substring(2); // remove comma // clean up and finish with deletion of indexes
if (hc1 >= peerCount) {
// clean up and finish with deletion of indexes // success
if (hc1 >= peerCount) { if (delete) {
// success int deletedURLs = deleteTransferIndexes(indexContainers);
if (delete) { this.log.logFine("Deleted from " + indexContainers.length + " transferred RWIs locally, removed " + deletedURLs + " URL references");
int deletedURLs = deleteTransferIndexes(indexContainers); indexContainers = null;
log.logFine("Deleted from " + indexContainers.length + " transferred RWIs locally, removed " + deletedURLs + " URL references"); return indexCount;
}
return indexCount; return indexCount;
} else {
// simply close the indexEntities
closeTransferIndexes(indexContainers);
} }
return indexCount; this.log.logSevere("Index distribution failed. Too few peers (" + hc1 + ") received the index, not deleted locally.");
} else {
log.logSevere("Index distribution failed. Too few peers (" + hc1 + ") received the index, not deleted locally.");
// simply close the indexEntities
closeTransferIndexes(indexContainers);
return -1; return -1;
} finally {
if (indexContainers != null) {
// simply close the indexEntities
closeTransferIndexes(indexContainers);
}
} }
} }

Write Preview
Loading…
Cancel
Save