boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

more DHT bugfixes and better logging of DHT effects

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@542 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 20 years ago
parent dbba052331
commit 25f632dbd9
6 changed files with 35 additions and 16 deletions
Show Stats Download Patch File Download Diff File

2
build.properties
Unescape View File

@ -3,7 +3,7 @@ javacSource=1.4
javacTarget=1.4
# Release Configuration
releaseVersion=0.397
releaseVersion=0.398
releaseFile=yacy_dev_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz
#releaseFile=yacy_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz
releaseDir=yacy_dev_v${releaseVersion}_${DSTAMP}_${releaseNr}

7
htroot/yacy/transferRWI.java
Unescape View File

@ -55,6 +55,7 @@ import de.anomic.plasma.plasmaWordIndexEntryContainer;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacySeed;
import de.anomic.yacy.yacyDHTAction;
public class transferRWI {
@ -123,11 +124,13 @@ public class transferRWI {
Iterator it = unknownURL.iterator();
while (it.hasNext()) unknownURLs += "," + (String) it.next();
if (unknownURLs.length() > 0) unknownURLs = unknownURLs.substring(1);
yacySeed otherPeer = yacyCore.seedDB.get(iam);
String otherPeerName = iam + ":" + ((otherPeer == null) ? "NULL" : (otherPeer.getName() + "/" + otherPeer.getVersion()));
if (wordhashes.length == 0)
switchboard.getLog().logInfo("Received 0 Words from peer " + iam + ", requested " + unknownURL.size() + " URLs");
switchboard.getLog().logInfo("Received 0 Words from " + otherPeerName + ", requested " + unknownURL.size() + " URLs");
else {
double avdist = (yacyDHTAction.dhtDistance(yacyCore.seedDB.mySeed.hash, wordhashes[0]) + yacyDHTAction.dhtDistance(yacyCore.seedDB.mySeed.hash, wordhashes[wordhashes.length - 1])) / 2.0;
switchboard.getLog().logInfo("Received " + received + " Words [" + wordhashes[0] + " .. " + wordhashes[wordhashes.length - 1] + "]/" + avdist + " from peer " + iam + ", requested " + unknownURL.size() + " URLs");
switchboard.getLog().logInfo("Received " + received + " Words [" + wordhashes[0] + " .. " + wordhashes[wordhashes.length - 1] + "]/" + avdist + " from " + otherPeerName + ", requested " + unknownURL.size() + " URLs");
}
result = "ok";
} else {

12
source/de/anomic/plasma/plasmaSearch.java
Unescape View File

@ -302,14 +302,16 @@ public final class plasmaSearch {
plasmaSearch.result acc = new result(searchhashes, stopwords, priority);
if (searchResult == null) return acc; // strange case where searchResult is not proper: acc is then empty
if (searchResult.size() == 0) return acc; // case that we have nothing to do
Enumeration e = searchResult.elements(true);
plasmaWordIndexEntry entry;
long startCreateTime = System.currentTimeMillis();
try {
while ((e.hasMoreElements()) &&
((acc.sizeFetched() < minEntries) || (System.currentTimeMillis() - startCreateTime < maxTime))) {
entry = (plasmaWordIndexEntry) e.nextElement();
while (e.hasMoreElements()) {
if ((acc.sizeFetched() >= minEntries) &&
(System.currentTimeMillis() - startCreateTime >= maxTime)) break;
entry = (plasmaWordIndexEntry) e.nextElement();
acc.addResult(entry);
}
} catch (kelondroException ee) {
@ -317,7 +319,7 @@ public final class plasmaSearch {
}
long startSortTime = System.currentTimeMillis();
acc.sortResults();
System.out.println("plasmaSearch.order: minEntries = " + minEntries + ", effectiveEntries = " + acc.sizeOrdered() + ", demanded Time = " + maxTime + ", effectiveTime = " + (System.currentTimeMillis() - startCreateTime) + ", createTime = " + (startSortTime - startCreateTime) + ", sortTime = " + (System.currentTimeMillis() - startSortTime));
serverLog.logDebug("PLASMA", "plasmaSearch.order: minEntries = " + minEntries + ", effectiveEntries = " + acc.sizeOrdered() + ", demanded Time = " + maxTime + ", effectiveTime = " + (System.currentTimeMillis() - startCreateTime) + ", createTime = " + (startSortTime - startCreateTime) + ", sortTime = " + (System.currentTimeMillis() - startSortTime));
return acc;
}

1
source/de/anomic/plasma/plasmaSwitchboard.java
Unescape View File

@ -1454,6 +1454,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
long timestamp = System.currentTimeMillis();
plasmaWordIndexEntity idx = searchManager.searchHashes(hashes, duetime * 8 / 10); // a nameless temporary index, not sorted by special order but by hash
long remainingTime = duetime - (System.currentTimeMillis() - timestamp);
if (remainingTime < 500) remainingTime = 500;
plasmaSearch.result acc = searchManager.order(idx, hashes, stopwords, new char[]{plasmaSearch.O_QUALITY, plasmaSearch.O_AGE}, remainingTime, 10);
// result is a List of urlEntry elements

16
source/de/anomic/plasma/plasmaWordIndexDistribution.java
Unescape View File

@ -115,7 +115,7 @@ public class plasmaWordIndexDistribution {
indexCount--;
else
indexCount++;
if (indexCount < 30) indexCount = 30;
if (indexCount < 50) indexCount = 50;
// show success
return true;
@ -159,17 +159,23 @@ public class plasmaWordIndexDistribution {
String error;
String peerNames = "";
double avdist;
long start;
while ((e.hasMoreElements()) && (hc < peerCount)) {
if (closed) {
log.logError("Index distribution interrupted by close, nothing deleted locally.");
return -1; // interrupted
}
seed = (yacySeed) e.nextElement();
if (seed != null) {
if ((seed != null) &&
((avdist = (yacyDHTAction.dhtDistance(seed.hash, indexEntities[0].wordHash()) +
yacyDHTAction.dhtDistance(seed.hash, indexEntities[indexEntities.length-1].wordHash())) / 2.0) < 0.3)) {
start = System.currentTimeMillis();
error = yacyClient.transferIndex(seed, indexEntities, urlCache);
if (error == null) {
avdist = (yacyDHTAction.dhtDistance(seed.hash, indexEntities[0].wordHash()) + yacyDHTAction.dhtDistance(seed.hash, indexEntities[indexEntities.length-1].wordHash())) / 2.0;
log.logInfo("Index transfer of " + indexCount + " words [" + indexEntities[0].wordHash() + " .. " + indexEntities[indexEntities.length-1].wordHash() + "]/" + avdist + " to peer " + seed.getName() + ":" + seed.hash + " successfull");
log.logInfo("Index transfer of " + indexCount + " words [" + indexEntities[0].wordHash() + " .. " + indexEntities[indexEntities.length-1].wordHash() + "]/" +
avdist + " to peer " + seed.getName() + ":" + seed.hash + " in " +
((System.currentTimeMillis() - start) / 1000) + " seconds successfull (" +
(indexCount / ((System.currentTimeMillis() - start) / 1000)) + " words/s)");
peerNames += ", " + seed.getName();
hc++;
} else {
@ -212,7 +218,7 @@ public class plasmaWordIndexDistribution {
private String selectTransferStart() {
String startPointHash;
// first try to select with increasing probality a good start point
for (int i = 9; i > 0; i--) {
if (Math.round(Math.random() * 6) != 4) for (int i = 9; i > 0; i--) {
startPointHash = serverCodings.encodeMD5B64(Long.toString(i + System.currentTimeMillis()), true).substring(2, 2 + yacySeedDB.commonHashLength);
if (yacyDHTAction.dhtDistance(yacyCore.seedDB.mySeed.hash, startPointHash) > ((double) i / (double) 10)) return startPointHash;
}

13
source/de/anomic/yacy/yacySearch.java
Unescape View File

@ -50,6 +50,7 @@ import de.anomic.plasma.plasmaCrawlLURL;
import de.anomic.plasma.plasmaURLPattern;
import de.anomic.plasma.plasmaSearch;
import de.anomic.plasma.plasmaSnippetCache;
import de.anomic.server.logging.serverLog;
public class yacySearch extends Thread {
@ -114,13 +115,17 @@ public class yacySearch extends Thread {
Iterator i = wordhashes.iterator();
int c;
String wordhash;
double distance;
while (i.hasNext()) {
wordhash = (String) i.next();
dhtEnum = yacyCore.dhtAgent.getDHTSeeds(true, wordhash);
c = seedcount;
while ((dhtEnum.hasMoreElements()) && (c > 0)) {
seed = (yacySeed) dhtEnum.nextElement();
//System.out.println("Selected peer " + seed.hash + "/" + seed.getName() + " for wordhash " + wordhash + ", score " + c);
if (seed == null) continue;
distance = yacyDHTAction.dhtDistance(seed.hash, wordhash);
if (distance > 0.9) continue; // catch bug in peer selection
serverLog.logDebug("PLASMA", "selectPeers/DHTorder: " + seed.hash + ":" + seed.getName() + "/" + distance + " for wordhash " + wordhash + ", score " + c);
ranking.addScore(seed.hash, c--);
seeds.put(seed.hash, seed);
}
@ -133,8 +138,9 @@ public class yacySearch extends Thread {
if (c > yacyCore.seedDB.sizeConnected()) c = yacyCore.seedDB.sizeConnected();
while ((dhtEnum.hasMoreElements()) && (c > 0)) {
seed = (yacySeed) dhtEnum.nextElement();
score = (int) Math.round(Math.random() * c / 2);
//System.out.println("Selected peer " + seed.hash + "/" + seed.getName() + " for maxRWI=" + seed.getMap().get("ICount") + ", score " + score);
if (seed == null) continue;
score = (int) Math.round(Math.random() * ((c / 3) + 3));
serverLog.logDebug("PLASMA", "selectPeers/RWIcount: " + seed.hash + ":" + seed.getName() + ", RWIcount=" + seed.getMap().get("ICount") + ", score " + score);
ranking.addScore(seed.hash, score);
seeds.put(seed.hash, seed);
c--;
@ -148,6 +154,7 @@ public class yacySearch extends Thread {
while ((e.hasNext()) && (c < result.length)) {
seed = (yacySeed) seeds.get((String) e.next());
seed.selectscore = c;
serverLog.logDebug("PLASMA", "selectPeers/_lineup_: " + seed.hash + ":" + seed.getName() + " is choice " + c);
result[c++] = seed;
}

Write Preview
Loading…
Cancel
Save