diff --git a/build.properties b/build.properties index 9cd6a43fb..67ebae010 100644 --- a/build.properties +++ b/build.properties @@ -3,7 +3,7 @@ javacSource=1.4 javacTarget=1.4 # Release Configuration -releaseVersion=0.397 +releaseVersion=0.398 releaseFile=yacy_dev_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz #releaseFile=yacy_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz releaseDir=yacy_dev_v${releaseVersion}_${DSTAMP}_${releaseNr} diff --git a/htroot/yacy/transferRWI.java b/htroot/yacy/transferRWI.java index 9ddf62a62..d983bd922 100644 --- a/htroot/yacy/transferRWI.java +++ b/htroot/yacy/transferRWI.java @@ -55,6 +55,7 @@ import de.anomic.plasma.plasmaWordIndexEntryContainer; import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; import de.anomic.yacy.yacyCore; +import de.anomic.yacy.yacySeed; import de.anomic.yacy.yacyDHTAction; public class transferRWI { @@ -123,11 +124,13 @@ public class transferRWI { Iterator it = unknownURL.iterator(); while (it.hasNext()) unknownURLs += "," + (String) it.next(); if (unknownURLs.length() > 0) unknownURLs = unknownURLs.substring(1); + yacySeed otherPeer = yacyCore.seedDB.get(iam); + String otherPeerName = iam + ":" + ((otherPeer == null) ? "NULL" : (otherPeer.getName() + "/" + otherPeer.getVersion())); if (wordhashes.length == 0) - switchboard.getLog().logInfo("Received 0 Words from peer " + iam + ", requested " + unknownURL.size() + " URLs"); + switchboard.getLog().logInfo("Received 0 Words from " + otherPeerName + ", requested " + unknownURL.size() + " URLs"); else { double avdist = (yacyDHTAction.dhtDistance(yacyCore.seedDB.mySeed.hash, wordhashes[0]) + yacyDHTAction.dhtDistance(yacyCore.seedDB.mySeed.hash, wordhashes[wordhashes.length - 1])) / 2.0; - switchboard.getLog().logInfo("Received " + received + " Words [" + wordhashes[0] + " .. " + wordhashes[wordhashes.length - 1] + "]/" + avdist + " from peer " + iam + ", requested " + unknownURL.size() + " URLs"); + switchboard.getLog().logInfo("Received " + received + " Words [" + wordhashes[0] + " .. " + wordhashes[wordhashes.length - 1] + "]/" + avdist + " from " + otherPeerName + ", requested " + unknownURL.size() + " URLs"); } result = "ok"; } else { diff --git a/source/de/anomic/plasma/plasmaSearch.java b/source/de/anomic/plasma/plasmaSearch.java index 65b1cc2ce..90ab758d2 100644 --- a/source/de/anomic/plasma/plasmaSearch.java +++ b/source/de/anomic/plasma/plasmaSearch.java @@ -302,14 +302,16 @@ public final class plasmaSearch { plasmaSearch.result acc = new result(searchhashes, stopwords, priority); if (searchResult == null) return acc; // strange case where searchResult is not proper: acc is then empty - + if (searchResult.size() == 0) return acc; // case that we have nothing to do + Enumeration e = searchResult.elements(true); plasmaWordIndexEntry entry; long startCreateTime = System.currentTimeMillis(); try { - while ((e.hasMoreElements()) && - ((acc.sizeFetched() < minEntries) || (System.currentTimeMillis() - startCreateTime < maxTime))) { - entry = (plasmaWordIndexEntry) e.nextElement(); + while (e.hasMoreElements()) { + if ((acc.sizeFetched() >= minEntries) && + (System.currentTimeMillis() - startCreateTime >= maxTime)) break; + entry = (plasmaWordIndexEntry) e.nextElement(); acc.addResult(entry); } } catch (kelondroException ee) { @@ -317,7 +319,7 @@ public final class plasmaSearch { } long startSortTime = System.currentTimeMillis(); acc.sortResults(); - System.out.println("plasmaSearch.order: minEntries = " + minEntries + ", effectiveEntries = " + acc.sizeOrdered() + ", demanded Time = " + maxTime + ", effectiveTime = " + (System.currentTimeMillis() - startCreateTime) + ", createTime = " + (startSortTime - startCreateTime) + ", sortTime = " + (System.currentTimeMillis() - startSortTime)); + serverLog.logDebug("PLASMA", "plasmaSearch.order: minEntries = " + minEntries + ", effectiveEntries = " + acc.sizeOrdered() + ", demanded Time = " + maxTime + ", effectiveTime = " + (System.currentTimeMillis() - startCreateTime) + ", createTime = " + (startSortTime - startCreateTime) + ", sortTime = " + (System.currentTimeMillis() - startSortTime)); return acc; } diff --git a/source/de/anomic/plasma/plasmaSwitchboard.java b/source/de/anomic/plasma/plasmaSwitchboard.java index 869fa4c09..fc613e587 100644 --- a/source/de/anomic/plasma/plasmaSwitchboard.java +++ b/source/de/anomic/plasma/plasmaSwitchboard.java @@ -1454,6 +1454,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser long timestamp = System.currentTimeMillis(); plasmaWordIndexEntity idx = searchManager.searchHashes(hashes, duetime * 8 / 10); // a nameless temporary index, not sorted by special order but by hash long remainingTime = duetime - (System.currentTimeMillis() - timestamp); + if (remainingTime < 500) remainingTime = 500; plasmaSearch.result acc = searchManager.order(idx, hashes, stopwords, new char[]{plasmaSearch.O_QUALITY, plasmaSearch.O_AGE}, remainingTime, 10); // result is a List of urlEntry elements diff --git a/source/de/anomic/plasma/plasmaWordIndexDistribution.java b/source/de/anomic/plasma/plasmaWordIndexDistribution.java index f21e13b51..cc2ea7a21 100644 --- a/source/de/anomic/plasma/plasmaWordIndexDistribution.java +++ b/source/de/anomic/plasma/plasmaWordIndexDistribution.java @@ -115,7 +115,7 @@ public class plasmaWordIndexDistribution { indexCount--; else indexCount++; - if (indexCount < 30) indexCount = 30; + if (indexCount < 50) indexCount = 50; // show success return true; @@ -159,17 +159,23 @@ public class plasmaWordIndexDistribution { String error; String peerNames = ""; double avdist; + long start; while ((e.hasMoreElements()) && (hc < peerCount)) { if (closed) { log.logError("Index distribution interrupted by close, nothing deleted locally."); return -1; // interrupted } seed = (yacySeed) e.nextElement(); - if (seed != null) { + if ((seed != null) && + ((avdist = (yacyDHTAction.dhtDistance(seed.hash, indexEntities[0].wordHash()) + + yacyDHTAction.dhtDistance(seed.hash, indexEntities[indexEntities.length-1].wordHash())) / 2.0) < 0.3)) { + start = System.currentTimeMillis(); error = yacyClient.transferIndex(seed, indexEntities, urlCache); if (error == null) { - avdist = (yacyDHTAction.dhtDistance(seed.hash, indexEntities[0].wordHash()) + yacyDHTAction.dhtDistance(seed.hash, indexEntities[indexEntities.length-1].wordHash())) / 2.0; - log.logInfo("Index transfer of " + indexCount + " words [" + indexEntities[0].wordHash() + " .. " + indexEntities[indexEntities.length-1].wordHash() + "]/" + avdist + " to peer " + seed.getName() + ":" + seed.hash + " successfull"); + log.logInfo("Index transfer of " + indexCount + " words [" + indexEntities[0].wordHash() + " .. " + indexEntities[indexEntities.length-1].wordHash() + "]/" + + avdist + " to peer " + seed.getName() + ":" + seed.hash + " in " + + ((System.currentTimeMillis() - start) / 1000) + " seconds successfull (" + + (indexCount / ((System.currentTimeMillis() - start) / 1000)) + " words/s)"); peerNames += ", " + seed.getName(); hc++; } else { @@ -212,7 +218,7 @@ public class plasmaWordIndexDistribution { private String selectTransferStart() { String startPointHash; // first try to select with increasing probality a good start point - for (int i = 9; i > 0; i--) { + if (Math.round(Math.random() * 6) != 4) for (int i = 9; i > 0; i--) { startPointHash = serverCodings.encodeMD5B64(Long.toString(i + System.currentTimeMillis()), true).substring(2, 2 + yacySeedDB.commonHashLength); if (yacyDHTAction.dhtDistance(yacyCore.seedDB.mySeed.hash, startPointHash) > ((double) i / (double) 10)) return startPointHash; } diff --git a/source/de/anomic/yacy/yacySearch.java b/source/de/anomic/yacy/yacySearch.java index 7de28b808..08bda1d96 100644 --- a/source/de/anomic/yacy/yacySearch.java +++ b/source/de/anomic/yacy/yacySearch.java @@ -50,6 +50,7 @@ import de.anomic.plasma.plasmaCrawlLURL; import de.anomic.plasma.plasmaURLPattern; import de.anomic.plasma.plasmaSearch; import de.anomic.plasma.plasmaSnippetCache; +import de.anomic.server.logging.serverLog; public class yacySearch extends Thread { @@ -114,13 +115,17 @@ public class yacySearch extends Thread { Iterator i = wordhashes.iterator(); int c; String wordhash; + double distance; while (i.hasNext()) { wordhash = (String) i.next(); dhtEnum = yacyCore.dhtAgent.getDHTSeeds(true, wordhash); c = seedcount; while ((dhtEnum.hasMoreElements()) && (c > 0)) { seed = (yacySeed) dhtEnum.nextElement(); - //System.out.println("Selected peer " + seed.hash + "/" + seed.getName() + " for wordhash " + wordhash + ", score " + c); + if (seed == null) continue; + distance = yacyDHTAction.dhtDistance(seed.hash, wordhash); + if (distance > 0.9) continue; // catch bug in peer selection + serverLog.logDebug("PLASMA", "selectPeers/DHTorder: " + seed.hash + ":" + seed.getName() + "/" + distance + " for wordhash " + wordhash + ", score " + c); ranking.addScore(seed.hash, c--); seeds.put(seed.hash, seed); } @@ -133,8 +138,9 @@ public class yacySearch extends Thread { if (c > yacyCore.seedDB.sizeConnected()) c = yacyCore.seedDB.sizeConnected(); while ((dhtEnum.hasMoreElements()) && (c > 0)) { seed = (yacySeed) dhtEnum.nextElement(); - score = (int) Math.round(Math.random() * c / 2); - //System.out.println("Selected peer " + seed.hash + "/" + seed.getName() + " for maxRWI=" + seed.getMap().get("ICount") + ", score " + score); + if (seed == null) continue; + score = (int) Math.round(Math.random() * ((c / 3) + 3)); + serverLog.logDebug("PLASMA", "selectPeers/RWIcount: " + seed.hash + ":" + seed.getName() + ", RWIcount=" + seed.getMap().get("ICount") + ", score " + score); ranking.addScore(seed.hash, score); seeds.put(seed.hash, seed); c--; @@ -148,6 +154,7 @@ public class yacySearch extends Thread { while ((e.hasNext()) && (c < result.length)) { seed = (yacySeed) seeds.get((String) e.next()); seed.selectscore = c; + serverLog.logDebug("PLASMA", "selectPeers/_lineup_: " + seed.hash + ":" + seed.getName() + " is choice " + c); result[c++] = seed; }