boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

intermission-feature usage in case of local and remote search

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@510 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 20 years ago
parent 6be19d76c9
commit 248c24b60a
7 changed files with 38 additions and 75 deletions
Show Stats Download Patch File Download Diff File

2
build.properties
Unescape View File

@ -3,7 +3,7 @@ javacSource=1.4
javacTarget=1.4
# Release Configuration
releaseVersion=0.394
releaseVersion=0.395
releaseFile=yacy_dev_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz
#releaseFile=yacy_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz
releaseDir=yacy_dev_v${releaseVersion}_${DSTAMP}_${releaseNr}

13
source/de/anomic/plasma/plasmaSwitchboard.java
Unescape View File

@ -261,7 +261,6 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
// start indexing management
log.logSystem("Starting Indexing Management");
urlPool = new plasmaURLPool(plasmaPath, ramLURL, ramNURL, ramEURL);
wordIndex = new plasmaWordIndex(plasmaPath, ramRWI, log);
int wordCacheMax = Integer.parseInt((String) getConfig("wordCacheMax", "10000"));
@ -1265,6 +1264,10 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
public serverObjects searchFromLocal(Set querywords, String order1, String order2, int count, boolean global, long time /*milliseconds*/, String urlmask) {
// tell all threads to do nothing for a specific time
wordIndex.intermission(time);
intermissionAllThreads(time);
serverObjects prop = new serverObjects();
try {
char[] order = new char[2];
@ -1282,7 +1285,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
Set queryhashes = plasmaSearch.words2hashes(querywords);
// log
log.logInfo("INIT WORD SEARCH: " + gs + " - " + count + " links, " + (time / 1000) + " seconds");
log.logInfo("INIT WORD SEARCH: " + gs + ":" + queryhashes + " - " + count + " links, " + (time / 1000) + " seconds");
long timestamp = System.currentTimeMillis();
// start a presearch, which makes only sense if we idle afterwards.
@ -1431,8 +1434,12 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
}
public serverObjects searchFromRemote(Set hashes, int count, boolean global, long duetime) {
// tell all threads to do nothing for a specific time
wordIndex.intermission(duetime);
intermissionAllThreads(duetime);
if (hashes == null) hashes = new HashSet();
serverObjects prop = new serverObjects();
try {
log.logInfo("INIT HASH SEARCH: " + hashes + " - " + count + " links");

4
source/de/anomic/plasma/plasmaWordIndex.java
Unescape View File

@ -103,6 +103,10 @@ public final class plasmaWordIndex {
return ramCache.removeEntries(wordHash, urlHashes, deleteComplete);
}
public void intermission(long pause) {
this.ramCache.intermission(pause);
}
public void close(int waitingBoundSeconds) {
ramCache.close(waitingBoundSeconds);
}

19
source/de/anomic/plasma/plasmaWordIndexCache.java
Unescape View File

@ -295,6 +295,10 @@ public final class plasmaWordIndexCache implements plasmaWordIndexInterface {
return urlCount;
}
public void intermission(long pause) {
flushThread.intermission(pause);
}
// cache settings
public int maxURLinWordCache() {
@ -334,16 +338,27 @@ public final class plasmaWordIndexCache implements plasmaWordIndexInterface {
private class flush extends Thread {
boolean terminate, pause;
long intermission;
public flush() {
terminate = false;
pause = false;
intermission = 0;
}
public void intermission(long pause) {
this.intermission = System.currentTimeMillis() + pause;
}
public void run() {
String nextHash;
Runtime rt = Runtime.getRuntime();
while (!terminate) {
if (intermission > 0) {
if (this.intermission > System.currentTimeMillis()) {
try {this.sleep(this.intermission - System.currentTimeMillis());} catch (InterruptedException e) {}
}
this.intermission = 0;
}
if (pause) {
try {this.sleep(300);} catch (InterruptedException e) {}
} else {

66
source/de/anomic/plasma/plasmaWordIndexDistribution.java
Unescape View File

@ -127,68 +127,6 @@ public class plasmaWordIndexDistribution {
this.seniorPeerCount = seniorPeerCount;
}
// For testing purposes only ...
// public int performTransferWholeIndex() {
//
// boolean success = true;
// int indexCount = 1000;
// int totalCount = 0;
// String peerHash = "Z-X31fMiBs9h";
// yacySeed seed = (yacySeed) yacyCore.seedDB.getConnected(peerHash);
// String startPointHash = serverCodings.encodeMD5B64("" + System.currentTimeMillis(), true).substring(0, yacySeedDB.commonHashLength);
//
// if ((yacyCore.seedDB == null) || (yacyCore.seedDB.sizeConnected() == 0)) return -1;
//
// while (success) {
// // collect index
// //String startPointHash = yacyCore.seedCache.mySeed.hash;
//
// plasmaWordIndexEntity[] indexEntities = selectTransferIndexes(startPointHash, indexCount);
// if ((indexEntities == null) || (indexEntities.length == 0)) {
// log.logDebug("No index available for index transfer, hash start-point " + startPointHash);
// return -1;
// }
// // count the indexes again, can be smaller as expected
// indexCount = 0; for (int i = 0; i < indexEntities.length; i++) indexCount += indexEntities[i].size();
//
// // iterate over DHT-peers and send away the indexes
// String error;
// String peerNames = "";
//
//
// if ((seed != null) && (indexCount > 0)) {
// error = yacyClient.transferIndex(seed,indexEntities, urlPool.loadedURL);
// if (error == null) {
// log.logInfo("Index transfer of " + indexCount + " words [" + indexEntities[0].wordHash() + " .. " + indexEntities[indexEntities.length-1].wordHash() + "] to peer " + seed.getName() + ":" + seed.hash + " successfull");
// peerNames += ", " + seed.getName();
//
// } else {
// log.logWarning("Index transfer to peer " + seed.getName() + ":" + seed.hash + " failed:'" + error + "', disconnecting peer");
// yacyCore.peerActions.peerDeparture(seed);
// success = false;
// }
// } else {
// success = false;
// }
//
// try {
// if (deleteTransferIndexes(indexEntities)) {
// log.logDebug("Deleted all transferred whole-word indexes locally");
// totalCount += indexCount;;
// startPointHash = indexEntities[indexEntities.length - 1].wordHash();
// } else {
// log.logError("Deleted not all transferred whole-word indexes");
// return -1;
// }
// } catch (IOException ee) {
// log.logError("Deletion of indexes not possible:" + ee.getMessage());
// ee.printStackTrace();
// return -1;
// }
// }
// return totalCount;
// }
public int performTransferIndex(int indexCount, int peerCount, boolean delete) {
if ((yacyCore.seedDB == null) || (yacyCore.seedDB.sizeConnected() == 0)) return -1;
@ -204,7 +142,7 @@ public class plasmaWordIndexDistribution {
indexCount = 0; for (int i = 0; i < indexEntities.length; i++) indexCount += indexEntities[i].size();
// find start point for DHT-selection
String keyhash = indexEntities[indexEntities.length - 1].wordHash();
String keyhash = indexEntities[indexEntities.length - 1].wordHash(); // DHT targets must have greater hashes
// iterate over DHT-peers and send away the indexes
yacySeed seed;
@ -271,7 +209,7 @@ public class plasmaWordIndexDistribution {
Enumeration urlEnum;
plasmaWordIndexEntry indexEntry;
while ((count > 0) && (wordHashIterator.hasNext()) &&
((nexthash = (String) wordHashIterator.next()) != null) && (nexthash.trim().length() > 0)) {
((nexthash = (String) wordHashIterator.next()) != null) && (nexthash.trim().length() > 0)) {
indexEntity = wordIndex.getEntity(nexthash, true);
if (indexEntity.size() == 0) {
indexEntity.deleteComplete();

2
source/de/anomic/server/serverAbstractThread.java
Unescape View File

@ -233,7 +233,7 @@ public abstract class serverAbstractThread extends Thread implements serverThrea
while (running) {
if (this.intermission > 0) {
if (this.intermission > System.currentTimeMillis()) {
ratz(System.currentTimeMillis() - this.intermission);
ratz(this.intermission - System.currentTimeMillis());
}
this.intermission = 0;
}

7
source/de/anomic/yacy/yacySearch.java
Unescape View File

@ -81,7 +81,7 @@ public class yacySearch extends Thread {
public void run() {
this.links = yacyClient.search(set2string(wordhashes), count, global, targetPeer, urlManager, searchManager, blacklist, snippetCache, duetime);
if (links != 0) {
//yacyCore.log.logInfo("REMOTE SEARCH - remote peer '" + targetPeer.get("Name", "anonymous") + "' contributed " + links + " links for word hash " + wordhashes);
//yacyCore.log.logInfo("REMOTE SEARCH - remote peer " + targetPeer.hash + ":" + targetPeer.getName() + " contributed " + links + " links for word hash " + wordhashes);
yacyCore.seedDB.mySeed.incRI(links);
yacyCore.seedDB.mySeed.incRU(links);
}
@ -120,10 +120,9 @@ public class yacySearch extends Thread {
}
if (ranking.size() < seedcount) seedcount = ranking.size();
yacySeed[] result = new yacySeed[seedcount];
Iterator e = ranking.scores(true);
Iterator e = ranking.scores(true); // lower are better
c = 0;
while ((e.hasNext()) && (c < result.length))
result[c++] = yacyCore.seedDB.getConnected((String) e.next());
while ((e.hasNext()) && (c < result.length)) result[c++] = yacyCore.seedDB.getConnected((String) e.next());
//System.out.println("DEBUG yacySearch.selectPeers = " + seedcount + " seeds:"); for (int i = 0; i < seedcount; i++) System.out.println(" #" + i + ":" + result[i]); // debug
return result;

Write Preview
Loading…
Cancel
Save