boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

added CRDistOn = true/false

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@1372 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
borg-0300 19 years ago
parent ee010c36ae
commit c5b6154136
3 changed files with 20 additions and 13 deletions
Show Stats Download Patch File Download Diff File

4
htroot/yacy/transfer.java
Unescape View File

@ -64,7 +64,6 @@ public final class transfer {
plasmaSwitchboard sb = (plasmaSwitchboard) env;
serverObjects prop = new serverObjects();
String process = post.get("process", ""); // permission or store
//String key = post.get("key", ""); // a transmission key from the client
String otherpeer = post.get("iam", ""); // identification of the client (a peer-hash)
@ -80,6 +79,8 @@ public final class transfer {
prop.put("process_path", "");
prop.put("process_maxsize", "0");
if (!sb.rankingOn) { return prop; }
yacySeed otherseed = yacyCore.seedDB.get(otherpeer);
if ((otherseed == null) || (filename.indexOf("..") >= 0)) {
// reject unknown peers: this does not appear fair, but anonymous senders are dangerous
@ -158,5 +159,4 @@ public final class transfer {
return prop;
}
}

10
source/de/anomic/plasma/plasmaSwitchboard.java
Unescape View File

@ -186,6 +186,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
public plasmaCrawlProfile.entry defaultProxyProfile;
public plasmaCrawlProfile.entry defaultRemoteProfile;
public plasmaWordIndexDistribution indexDistribution;
public boolean rankingOn;
public plasmaRankingDistribution rankingOwnDistribution;
public plasmaRankingDistribution rankingOtherDistribution;
public HashMap outgoingCookies, incomingCookies;
@ -454,6 +455,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
// init ranking transmission
/*
CRDistOn = true/false
CRDist0Path = GLOBAL/010_owncr
CRDist0Method = 1
CRDist0Percent = 0
@ -463,6 +465,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
CRDist1Percent = 30
CRDist1Target = kaskelix.de:8080,yacy.dyndns.org:8000,suma-lab.de:8080
**/
rankingOn = getConfig("CRDistOn", "true").equals("true");
rankingOwnDistribution = new plasmaRankingDistribution(log, new File(rankingPath, getConfig("CRDist0Path", plasmaRankingDistribution.CR_OWN)), (int) getConfigLong("CRDist0Method", plasmaRankingDistribution.METHOD_ANYSENIOR), (int) getConfigLong("CRDist0Percent", 0), getConfig("CRDist0Target", ""));
rankingOtherDistribution = new plasmaRankingDistribution(log, new File(rankingPath, getConfig("CRDist1Path", plasmaRankingDistribution.CR_OTHER)), (int) getConfigLong("CRDist1Method", plasmaRankingDistribution.METHOD_MIXEDSENIOR), (int) getConfigLong("CRDist1Percent", 30), getConfig("CRDist1Target", "kaskelix.de:8080,yacy.dyndns.org:8000,suma-lab.de:8080"));
@ -522,6 +525,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
new serverInstantThread(this, "cleanupJob", "cleanupJobSize"), 10000); // all 5 Minutes
deployThread("82_crawlstack", "Crawl URL Stacker", "process that checks url for double-occurrences and for allowance/disallowance by robots.txt", null,
new serverInstantThread(sbStackCrawlThread, "job", "size"), 8000);
deployThread("80_indexing", "Parsing/Indexing", "thread that performes document parsing and indexing", "/IndexCreateIndexingQueue_p.html",
new serverInstantThread(this, "deQueue", "queueSize"), 10000);
for (int i = 1; i < indexing_cluster; i++) {
@ -533,6 +537,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
Long.parseLong(getConfig("80_indexing_busysleep" , "0")),
Long.parseLong(getConfig("80_indexing_memprereq" , "1000000")));
}
deployThread("70_cachemanager", "Proxy Cache Enqueue", "job takes new proxy files from RAM stack, stores them, and hands over to the Indexing Stack", null,
new serverInstantThread(this, "htEntryStoreJob", "htEntrySize"), 10000);
deployThread("62_remotetriggeredcrawl", "Remote Crawl Job", "thread that performes a single crawl/indexing step triggered by a remote peer", null,
@ -616,10 +621,11 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
defaultProxyProfile = profiles.getEntry(getConfig(STR_PROXYPROFILE, ""));
}
if ((profiles.size() == 1) ||
(getConfig(STR_REMOTEPROFILE, "").length() == 0) ||
(profiles.getEntry(getConfig(STR_REMOTEPROFILE, "")) == null)) {
(getConfig(STR_REMOTEPROFILE, "").length() == 0) ||
(profiles.getEntry(getConfig(STR_REMOTEPROFILE, "")) == null)) {
// generate new default entry for remote crawling
defaultRemoteProfile = profiles.newEntry("remote", "", ".*", ".*", 0, 0, true, false, true, true, false, true, true, false);
// defaultRemoteProfile = profiles.newEntry("remote", "", ".*", ".*", 0, 0, true, true, true, true, true, true, true, false);
setConfig(STR_REMOTEPROFILE, defaultRemoteProfile.handle());
} else {
defaultRemoteProfile = profiles.getEntry(getConfig(STR_REMOTEPROFILE, ""));

1
yacy.init
Unescape View File

@ -612,6 +612,7 @@ indexTransfer.maxOpenFiles = 800
# to anonymize the data even against the intermediate peer
# a specific precentage is also sent again to other peers.
# for key-numbers please see de.anomic.plasma.plasmaRankingDistribution
CRDistOn = true
CRDist0Path = GLOBAL/010_owncr
CRDist0Method = 1
CRDist0Percent = 0

Write Preview
Loading…
Cancel
Save