From 248c24b60a6e4db1a22322358310194d4efbd409 Mon Sep 17 00:00:00 2001 From: orbiter Date: Tue, 9 Aug 2005 20:43:37 +0000 Subject: [PATCH] intermission-feature usage in case of local and remote search git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@510 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- build.properties | 2 +- .../de/anomic/plasma/plasmaSwitchboard.java | 13 +++- source/de/anomic/plasma/plasmaWordIndex.java | 4 ++ .../anomic/plasma/plasmaWordIndexCache.java | 19 +++++- .../plasma/plasmaWordIndexDistribution.java | 66 +------------------ .../anomic/server/serverAbstractThread.java | 2 +- source/de/anomic/yacy/yacySearch.java | 7 +- 7 files changed, 38 insertions(+), 75 deletions(-) diff --git a/build.properties b/build.properties index 86160f68b..62e0b5517 100644 --- a/build.properties +++ b/build.properties @@ -3,7 +3,7 @@ javacSource=1.4 javacTarget=1.4 # Release Configuration -releaseVersion=0.394 +releaseVersion=0.395 releaseFile=yacy_dev_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz #releaseFile=yacy_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz releaseDir=yacy_dev_v${releaseVersion}_${DSTAMP}_${releaseNr} diff --git a/source/de/anomic/plasma/plasmaSwitchboard.java b/source/de/anomic/plasma/plasmaSwitchboard.java index f57e6e9bf..194deb6f8 100644 --- a/source/de/anomic/plasma/plasmaSwitchboard.java +++ b/source/de/anomic/plasma/plasmaSwitchboard.java @@ -261,7 +261,6 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser // start indexing management log.logSystem("Starting Indexing Management"); urlPool = new plasmaURLPool(plasmaPath, ramLURL, ramNURL, ramEURL); - wordIndex = new plasmaWordIndex(plasmaPath, ramRWI, log); int wordCacheMax = Integer.parseInt((String) getConfig("wordCacheMax", "10000")); @@ -1265,6 +1264,10 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser public serverObjects searchFromLocal(Set querywords, String order1, String order2, int count, boolean global, long time /*milliseconds*/, String urlmask) { + // tell all threads to do nothing for a specific time + wordIndex.intermission(time); + intermissionAllThreads(time); + serverObjects prop = new serverObjects(); try { char[] order = new char[2]; @@ -1282,7 +1285,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser Set queryhashes = plasmaSearch.words2hashes(querywords); // log - log.logInfo("INIT WORD SEARCH: " + gs + " - " + count + " links, " + (time / 1000) + " seconds"); + log.logInfo("INIT WORD SEARCH: " + gs + ":" + queryhashes + " - " + count + " links, " + (time / 1000) + " seconds"); long timestamp = System.currentTimeMillis(); // start a presearch, which makes only sense if we idle afterwards. @@ -1431,8 +1434,12 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser } public serverObjects searchFromRemote(Set hashes, int count, boolean global, long duetime) { + + // tell all threads to do nothing for a specific time + wordIndex.intermission(duetime); + intermissionAllThreads(duetime); + if (hashes == null) hashes = new HashSet(); - serverObjects prop = new serverObjects(); try { log.logInfo("INIT HASH SEARCH: " + hashes + " - " + count + " links"); diff --git a/source/de/anomic/plasma/plasmaWordIndex.java b/source/de/anomic/plasma/plasmaWordIndex.java index 1da846107..bfcbb2df0 100644 --- a/source/de/anomic/plasma/plasmaWordIndex.java +++ b/source/de/anomic/plasma/plasmaWordIndex.java @@ -103,6 +103,10 @@ public final class plasmaWordIndex { return ramCache.removeEntries(wordHash, urlHashes, deleteComplete); } + public void intermission(long pause) { + this.ramCache.intermission(pause); + } + public void close(int waitingBoundSeconds) { ramCache.close(waitingBoundSeconds); } diff --git a/source/de/anomic/plasma/plasmaWordIndexCache.java b/source/de/anomic/plasma/plasmaWordIndexCache.java index b26b08849..1e6fc7c98 100644 --- a/source/de/anomic/plasma/plasmaWordIndexCache.java +++ b/source/de/anomic/plasma/plasmaWordIndexCache.java @@ -295,6 +295,10 @@ public final class plasmaWordIndexCache implements plasmaWordIndexInterface { return urlCount; } + public void intermission(long pause) { + flushThread.intermission(pause); + } + // cache settings public int maxURLinWordCache() { @@ -334,16 +338,27 @@ public final class plasmaWordIndexCache implements plasmaWordIndexInterface { private class flush extends Thread { boolean terminate, pause; + long intermission; public flush() { terminate = false; - pause = false; + intermission = 0; } - + + public void intermission(long pause) { + this.intermission = System.currentTimeMillis() + pause; + } + public void run() { String nextHash; Runtime rt = Runtime.getRuntime(); while (!terminate) { + if (intermission > 0) { + if (this.intermission > System.currentTimeMillis()) { + try {this.sleep(this.intermission - System.currentTimeMillis());} catch (InterruptedException e) {} + } + this.intermission = 0; + } if (pause) { try {this.sleep(300);} catch (InterruptedException e) {} } else { diff --git a/source/de/anomic/plasma/plasmaWordIndexDistribution.java b/source/de/anomic/plasma/plasmaWordIndexDistribution.java index 8fe5e1add..7ca979e77 100644 --- a/source/de/anomic/plasma/plasmaWordIndexDistribution.java +++ b/source/de/anomic/plasma/plasmaWordIndexDistribution.java @@ -127,68 +127,6 @@ public class plasmaWordIndexDistribution { this.seniorPeerCount = seniorPeerCount; } -// For testing purposes only ... -// public int performTransferWholeIndex() { -// -// boolean success = true; -// int indexCount = 1000; -// int totalCount = 0; -// String peerHash = "Z-X31fMiBs9h"; -// yacySeed seed = (yacySeed) yacyCore.seedDB.getConnected(peerHash); -// String startPointHash = serverCodings.encodeMD5B64("" + System.currentTimeMillis(), true).substring(0, yacySeedDB.commonHashLength); -// -// if ((yacyCore.seedDB == null) || (yacyCore.seedDB.sizeConnected() == 0)) return -1; -// -// while (success) { -// // collect index -// //String startPointHash = yacyCore.seedCache.mySeed.hash; -// -// plasmaWordIndexEntity[] indexEntities = selectTransferIndexes(startPointHash, indexCount); -// if ((indexEntities == null) || (indexEntities.length == 0)) { -// log.logDebug("No index available for index transfer, hash start-point " + startPointHash); -// return -1; -// } -// // count the indexes again, can be smaller as expected -// indexCount = 0; for (int i = 0; i < indexEntities.length; i++) indexCount += indexEntities[i].size(); -// -// // iterate over DHT-peers and send away the indexes -// String error; -// String peerNames = ""; -// -// -// if ((seed != null) && (indexCount > 0)) { -// error = yacyClient.transferIndex(seed,indexEntities, urlPool.loadedURL); -// if (error == null) { -// log.logInfo("Index transfer of " + indexCount + " words [" + indexEntities[0].wordHash() + " .. " + indexEntities[indexEntities.length-1].wordHash() + "] to peer " + seed.getName() + ":" + seed.hash + " successfull"); -// peerNames += ", " + seed.getName(); -// -// } else { -// log.logWarning("Index transfer to peer " + seed.getName() + ":" + seed.hash + " failed:'" + error + "', disconnecting peer"); -// yacyCore.peerActions.peerDeparture(seed); -// success = false; -// } -// } else { -// success = false; -// } -// -// try { -// if (deleteTransferIndexes(indexEntities)) { -// log.logDebug("Deleted all transferred whole-word indexes locally"); -// totalCount += indexCount;; -// startPointHash = indexEntities[indexEntities.length - 1].wordHash(); -// } else { -// log.logError("Deleted not all transferred whole-word indexes"); -// return -1; -// } -// } catch (IOException ee) { -// log.logError("Deletion of indexes not possible:" + ee.getMessage()); -// ee.printStackTrace(); -// return -1; -// } -// } -// return totalCount; -// } - public int performTransferIndex(int indexCount, int peerCount, boolean delete) { if ((yacyCore.seedDB == null) || (yacyCore.seedDB.sizeConnected() == 0)) return -1; @@ -204,7 +142,7 @@ public class plasmaWordIndexDistribution { indexCount = 0; for (int i = 0; i < indexEntities.length; i++) indexCount += indexEntities[i].size(); // find start point for DHT-selection - String keyhash = indexEntities[indexEntities.length - 1].wordHash(); + String keyhash = indexEntities[indexEntities.length - 1].wordHash(); // DHT targets must have greater hashes // iterate over DHT-peers and send away the indexes yacySeed seed; @@ -271,7 +209,7 @@ public class plasmaWordIndexDistribution { Enumeration urlEnum; plasmaWordIndexEntry indexEntry; while ((count > 0) && (wordHashIterator.hasNext()) && - ((nexthash = (String) wordHashIterator.next()) != null) && (nexthash.trim().length() > 0)) { + ((nexthash = (String) wordHashIterator.next()) != null) && (nexthash.trim().length() > 0)) { indexEntity = wordIndex.getEntity(nexthash, true); if (indexEntity.size() == 0) { indexEntity.deleteComplete(); diff --git a/source/de/anomic/server/serverAbstractThread.java b/source/de/anomic/server/serverAbstractThread.java index 6cd6ee8a4..b6ec4bc80 100644 --- a/source/de/anomic/server/serverAbstractThread.java +++ b/source/de/anomic/server/serverAbstractThread.java @@ -233,7 +233,7 @@ public abstract class serverAbstractThread extends Thread implements serverThrea while (running) { if (this.intermission > 0) { if (this.intermission > System.currentTimeMillis()) { - ratz(System.currentTimeMillis() - this.intermission); + ratz(this.intermission - System.currentTimeMillis()); } this.intermission = 0; } diff --git a/source/de/anomic/yacy/yacySearch.java b/source/de/anomic/yacy/yacySearch.java index 8f6046b89..4fc9e9770 100644 --- a/source/de/anomic/yacy/yacySearch.java +++ b/source/de/anomic/yacy/yacySearch.java @@ -81,7 +81,7 @@ public class yacySearch extends Thread { public void run() { this.links = yacyClient.search(set2string(wordhashes), count, global, targetPeer, urlManager, searchManager, blacklist, snippetCache, duetime); if (links != 0) { - //yacyCore.log.logInfo("REMOTE SEARCH - remote peer '" + targetPeer.get("Name", "anonymous") + "' contributed " + links + " links for word hash " + wordhashes); + //yacyCore.log.logInfo("REMOTE SEARCH - remote peer " + targetPeer.hash + ":" + targetPeer.getName() + " contributed " + links + " links for word hash " + wordhashes); yacyCore.seedDB.mySeed.incRI(links); yacyCore.seedDB.mySeed.incRU(links); } @@ -120,10 +120,9 @@ public class yacySearch extends Thread { } if (ranking.size() < seedcount) seedcount = ranking.size(); yacySeed[] result = new yacySeed[seedcount]; - Iterator e = ranking.scores(true); + Iterator e = ranking.scores(true); // lower are better c = 0; - while ((e.hasNext()) && (c < result.length)) - result[c++] = yacyCore.seedDB.getConnected((String) e.next()); + while ((e.hasNext()) && (c < result.length)) result[c++] = yacyCore.seedDB.getConnected((String) e.next()); //System.out.println("DEBUG yacySearch.selectPeers = " + seedcount + " seeds:"); for (int i = 0; i < seedcount; i++) System.out.println(" #" + i + ":" + result[i]); // debug return result;