Patch fuer Issue 0000102

and fixes to Patch (private peer status is a property of a peer, not a
status)
pull/1/head
stbrumm 13 years ago committed by Michael Christen
parent 9f1b1b4604
commit d18095dc48

@ -41,18 +41,22 @@ import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch; import de.anomic.server.serverSwitch;
/** draw a banner with information about the peer */ /** draw a banner with information about the peer */
public class Banner { public class Banner
{
public static RasterPlotter respond(final RequestHeader header, final serverObjects post, final serverSwitch env) throws IOException { public static RasterPlotter respond(
final RequestHeader header,
final serverObjects post,
final serverSwitch env) throws IOException {
final Switchboard sb = (Switchboard) env; final Switchboard sb = (Switchboard) env;
final String IMAGE = "htroot/env/grafics/yacy.png"; final String IMAGE = "htroot/env/grafics/yacy.png";
int width = 468; int width = 468;
int height = 60; int height = 60;
String bgcolor = "e7effc"; String bgcolor = "e7effc";
String textcolor = "000000"; String textcolor = "000000";
String bordercolor = "5090d0"; String bordercolor = "5090d0";
if (post != null) { if ( post != null ) {
bgcolor = post.get("bgcolor", bgcolor); bgcolor = post.get("bgcolor", bgcolor);
textcolor = post.get("textcolor", textcolor); textcolor = post.get("textcolor", textcolor);
bordercolor = post.get("bordercolor", bordercolor); bordercolor = post.get("bordercolor", bordercolor);
@ -60,41 +64,41 @@ public class Banner {
height = post.getInt("heigth", height); height = post.getInt("heigth", height);
} }
String name = ""; String name = "";
long links = 0; long links = 0;
long words = 0; long words = 0;
int myppm = 0; int myppm = 0;
double myqph = 0; double myqph = 0;
String type = ""; String type = "";
final String network = env.getConfig(SwitchboardConstants.NETWORK_NAME, "unspecified").toUpperCase(); final String network = env.getConfig(SwitchboardConstants.NETWORK_NAME, "unspecified").toUpperCase();
final int peers = sb.peers.sizeConnected() + 1; // the '+ 1': the own peer is not included in sizeConnected() final int peers = sb.peers.sizeConnected() + 1; // the '+ 1': the own peer is not included in sizeConnected()
long nlinks = sb.peers.countActiveURL(); long nlinks = sb.peers.countActiveURL();
long nwords = sb.peers.countActiveRWI(); long nwords = sb.peers.countActiveRWI();
final double nqpm = sb.peers.countActiveQPM(); final double nqpm = sb.peers.countActiveQPM();
long nppm = sb.peers.countActivePPM(); long nppm = sb.peers.countActivePPM();
double nqph = 0; double nqph = 0;
final Seed seed = sb.peers.mySeed(); final Seed seed = sb.peers.mySeed();
if (seed != null){ if ( seed != null ) {
name = seed.get(Seed.NAME, "-").toUpperCase(); name = seed.get(Seed.NAME, "-").toUpperCase();
links = seed.getLinkCount(); links = seed.getLinkCount();
words = seed.getWordCount(); words = seed.getWordCount();
myppm = seed.getPPM(); myppm = seed.getPPM();
myqph = 60d * seed.getQPM(); myqph = 60d * seed.getQPM();
if (sb.peers.mySeed().isVirgin()) { if ( sb.peers.mySeed().isVirgin() ) {
type = "VIRGIN"; type = "VIRGIN";
nqph = Math.round(6000d * nqpm) / 100d; nqph = Math.round(6000d * nqpm) / 100d;
} else if(sb.peers.mySeed().isJunior()) { } else if ( sb.peers.mySeed().isJunior() ) {
type = "JUNIOR"; type = "JUNIOR";
nqph = Math.round(6000d * nqpm) / 100d; nqph = Math.round(6000d * nqpm) / 100d;
} else if(sb.peers.mySeed().isSenior()) { } else if ( sb.peers.mySeed().isSenior() ) {
type = "SENIOR"; type = "SENIOR";
nlinks = nlinks + links; nlinks = nlinks + links;
nwords = nwords + words; nwords = nwords + words;
nqph = Math.round(6000d * nqpm + 100d * myqph) / 100d; nqph = Math.round(6000d * nqpm + 100d * myqph) / 100d;
nppm = nppm + myppm; nppm = nppm + myppm;
} else if(sb.peers.mySeed().isPrincipal()) { } else if ( sb.peers.mySeed().isPrincipal() ) {
type = "PRINCIPAL"; type = "PRINCIPAL";
nlinks = nlinks + links; nlinks = nlinks + links;
nwords = nwords + words; nwords = nwords + words;
@ -103,13 +107,48 @@ public class Banner {
} }
} }
if (!NetworkGraph.logoIsLoaded()) { if ( !NetworkGraph.logoIsLoaded() ) {
ImageIO.setUseCache(false); // do not write a cache to disc; keep in RAM ImageIO.setUseCache(false); // do not write a cache to disc; keep in RAM
final BufferedImage logo = ImageIO.read(new File(IMAGE)); final BufferedImage logo = ImageIO.read(new File(IMAGE));
return NetworkGraph.getBannerPicture(1000, width, height, bgcolor, textcolor, bordercolor, name, links, words, type, myppm, network, peers, nlinks, nwords, nqph, nppm, logo); return NetworkGraph.getBannerPicture(
1000,
width,
height,
bgcolor,
textcolor,
bordercolor,
name,
links,
words,
type,
myppm,
network,
peers,
nlinks,
nwords,
nqph,
nppm,
logo);
} }
return NetworkGraph.getBannerPicture(1000, width, height, bgcolor, textcolor, bordercolor, name, links, words, type, myppm, network, peers, nlinks, nwords, nqph, nppm); return NetworkGraph.getBannerPicture(
1000,
width,
height,
bgcolor,
textcolor,
bordercolor,
name,
links,
words,
type,
myppm,
network,
peers,
nlinks,
nwords,
nqph,
nppm);
} }
} }

@ -163,23 +163,9 @@
</dt> </dt>
<dd>Your search engine will not contact any other peer, and will reject every request. <dd>Your search engine will not contact any other peer, and will reject every request.
</dd> </dd>
<!-- not yet implemented
<dt> <dt>
<label for="cluster.modePrivatecluster">Private Cluster</label> <label for="publicpeer">Public Peer</label>
<input type="radio" value="privatecluster" id="cluster.modePrivatecluster" name="cluster.mode" <input type="radio" value="publicpeer" id="publicpeer" name="cluster.mode"
#(privateclusterChecked)#::checked="checked" #(/privateclusterChecked)#/>
</dt>
<dd>
Your peer is part of a private cluster without public visibility.<br />
Index data is not distributed, but remote crawl requests are distributed and accepted from your cluster.<br />
Search requests are spread over all peers of the cluster, and answered from all peers of the cluster.<br />
List of ip:port - addresses of the cluster: (comma-separated)<br />
<input type="text" name="cluster.peers.ipport" value="#[cluster.peers.ipport]#" size="80" maxlength="800" />
</dd>
-->
<dt>
<label for="cluster.modePublicpeer">Public Peer</label>
<input type="radio" value="publicpeer" id="cluster.modePublicpeer" name="cluster.mode"
#(publicpeerChecked)#::checked="checked" #(/publicpeerChecked)#/> #(publicpeerChecked)#::checked="checked" #(/publicpeerChecked)#/>
</dt> </dt>
<dd> <dd>
@ -187,8 +173,8 @@
Your peer does not accept any outside index data, but responds on all remote search requests. Your peer does not accept any outside index data, but responds on all remote search requests.
</dd> </dd>
<dt> <dt>
<label for="cluster.modePubliccluster">Public Cluster</label> <label for="publiccluster">Public Cluster</label>
<input type="radio" value="publiccluster" id="cluster.modePubliccluster" name="cluster.mode" <input type="radio" value="publiccluster" id="publiccluster" name="cluster.mode"
#(publicclusterChecked)#::checked="checked" #(/publicclusterChecked)#/> #(publicclusterChecked)#::checked="checked" #(/publicclusterChecked)#/>
</dt> </dt>
<dd> <dd>

@ -40,26 +40,36 @@ import de.anomic.data.WorkTables;
import de.anomic.server.serverObjects; import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch; import de.anomic.server.serverSwitch;
public class ConfigNetwork_p { public class ConfigNetwork_p
{
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) throws FileNotFoundException, IOException { public static serverObjects respond(
final RequestHeader header,
final serverObjects post,
final serverSwitch env) throws FileNotFoundException, IOException {
final Switchboard sb = (Switchboard) env; final Switchboard sb = (Switchboard) env;
final serverObjects prop = new serverObjects(); final serverObjects prop = new serverObjects();
int commit = 0; int commit = 0;
// load all options for network definitions // load all options for network definitions
final File networkBootstrapLocationsFile = new File(new File(sb.getAppPath(), "defaults"), "yacy.networks"); final File networkBootstrapLocationsFile =
new File(new File(sb.getAppPath(), "defaults"), "yacy.networks");
final Set<String> networkBootstrapLocations = FileUtils.loadList(networkBootstrapLocationsFile); final Set<String> networkBootstrapLocations = FileUtils.loadList(networkBootstrapLocationsFile);
if (post != null) { if ( post != null ) {
// store this call as api call // store this call as api call
sb.tables.recordAPICall(post, "ConfigNetwork_p.html", WorkTables.TABLE_API_TYPE_CONFIGURATION, "network settings"); sb.tables.recordAPICall(
post,
if (post.containsKey("changeNetwork")) { "ConfigNetwork_p.html",
final String networkDefinition = post.get("networkDefinition", "defaults/yacy.network.freeworld.unit"); WorkTables.TABLE_API_TYPE_CONFIGURATION,
if (networkDefinition.equals(sb.getConfig("network.unit.definition", ""))) { "network settings");
if ( post.containsKey("changeNetwork") ) {
final String networkDefinition =
post.get("networkDefinition", "defaults/yacy.network.freeworld.unit");
if ( networkDefinition.equals(sb.getConfig("network.unit.definition", "")) ) {
// no change // no change
commit = 3; commit = 3;
} else { } else {
@ -69,55 +79,55 @@ public class ConfigNetwork_p {
} }
} }
if (post.containsKey("save")) { if ( post.containsKey("save") ) {
// DHT control // DHT control
boolean indexDistribute = "on".equals(post.get("indexDistribute", "")); boolean indexDistribute = "on".equals(post.get("indexDistribute", ""));
boolean indexReceive = "on".equals(post.get("indexReceive", "")); boolean indexReceive = "on".equals(post.get("indexReceive", ""));
if (!indexReceive) { if ( !indexReceive ) {
// remove heuristics // remove heuristics
sb.setConfig("heuristic.site", false); sb.setConfig("heuristic.site", false);
sb.setConfig("heuristic.scroogle", false); sb.setConfig("heuristic.scroogle", false);
sb.setConfig("heuristic.blekko", false); sb.setConfig("heuristic.blekko", false);
} }
final boolean robinsonmode = "robinson".equals(post.get("network", "")); final boolean robinsonmode = "robinson".equals(post.get("network", ""));
if (robinsonmode) { if ( robinsonmode ) {
indexDistribute = false; indexDistribute = false;
indexReceive = false; indexReceive = false;
commit = 1; commit = 1;
} else { } else {
if (!indexDistribute && !indexReceive) { if ( !indexDistribute && !indexReceive ) {
prop.put("commitDHTIsRobinson", "1"); prop.put("commitDHTIsRobinson", "1");
commit = 2; commit = 2;
} else if (indexDistribute && indexReceive) { } else if ( indexDistribute && indexReceive ) {
commit = 1; commit = 1;
} else { } else {
if (!indexReceive) { if ( !indexReceive ) {
prop.put("commitDHTNoGlobalSearch", "1"); prop.put("commitDHTNoGlobalSearch", "1");
} }
commit = 1; commit = 1;
} }
} }
if (indexDistribute) { if ( indexDistribute ) {
sb.setConfig(SwitchboardConstants.INDEX_DIST_ALLOW, true); sb.setConfig(SwitchboardConstants.INDEX_DIST_ALLOW, true);
} else { } else {
sb.setConfig(SwitchboardConstants.INDEX_DIST_ALLOW, false); sb.setConfig(SwitchboardConstants.INDEX_DIST_ALLOW, false);
} }
if ("on".equals(post.get("indexDistributeWhileCrawling",""))) { if ( "on".equals(post.get("indexDistributeWhileCrawling", "")) ) {
sb.setConfig(SwitchboardConstants.INDEX_DIST_ALLOW_WHILE_CRAWLING, true); sb.setConfig(SwitchboardConstants.INDEX_DIST_ALLOW_WHILE_CRAWLING, true);
} else { } else {
sb.setConfig(SwitchboardConstants.INDEX_DIST_ALLOW_WHILE_CRAWLING, false); sb.setConfig(SwitchboardConstants.INDEX_DIST_ALLOW_WHILE_CRAWLING, false);
} }
if ("on".equals(post.get("indexDistributeWhileIndexing",""))) { if ( "on".equals(post.get("indexDistributeWhileIndexing", "")) ) {
sb.setConfig(SwitchboardConstants.INDEX_DIST_ALLOW_WHILE_INDEXING, true); sb.setConfig(SwitchboardConstants.INDEX_DIST_ALLOW_WHILE_INDEXING, true);
} else { } else {
sb.setConfig(SwitchboardConstants.INDEX_DIST_ALLOW_WHILE_INDEXING, false); sb.setConfig(SwitchboardConstants.INDEX_DIST_ALLOW_WHILE_INDEXING, false);
} }
if (indexReceive) { if ( indexReceive ) {
sb.setConfig(SwitchboardConstants.INDEX_RECEIVE_ALLOW, true); sb.setConfig(SwitchboardConstants.INDEX_RECEIVE_ALLOW, true);
sb.peers.mySeed().setFlagAcceptRemoteIndex(true); sb.peers.mySeed().setFlagAcceptRemoteIndex(true);
} else { } else {
@ -126,19 +136,24 @@ public class ConfigNetwork_p {
sb.setConfig(SwitchboardConstants.INDEX_RECEIVE_AUTODISABLED, false); sb.setConfig(SwitchboardConstants.INDEX_RECEIVE_AUTODISABLED, false);
} }
if ("on".equals(post.get("indexReceiveBlockBlacklist", ""))) { if ( "on".equals(post.get("indexReceiveBlockBlacklist", "")) ) {
sb.setConfig("indexReceiveBlockBlacklist", true); sb.setConfig("indexReceiveBlockBlacklist", true);
} else { } else {
sb.setConfig("indexReceiveBlockBlacklist", false); sb.setConfig("indexReceiveBlockBlacklist", false);
} }
if (post.containsKey("peertags")) { if ( post.containsKey("peertags") ) {
sb.peers.mySeed().setPeerTags(MapTools.string2set(normalizedList(post.get("peertags")), ",")); sb.peers.mySeed().setPeerTags(
MapTools.string2set(normalizedList(post.get("peertags")), ","));
} }
sb.setConfig("cluster.mode", post.get("cluster.mode", "publicpeer")); sb.setConfig("cluster.mode", post.get(
SwitchboardConstants.CLUSTER_MODE,
SwitchboardConstants.CLUSTER_MODE_PUBLIC_PEER));
sb.setConfig("cluster.peers.ipport", checkIPPortList(post.get("cluster.peers.ipport", ""))); sb.setConfig("cluster.peers.ipport", checkIPPortList(post.get("cluster.peers.ipport", "")));
sb.setConfig("cluster.peers.yacydomain", checkYaCyDomainList(post.get("cluster.peers.yacydomain", ""))); sb.setConfig(
"cluster.peers.yacydomain",
checkYaCyDomainList(post.get("cluster.peers.yacydomain", "")));
// update the cluster hash set // update the cluster hash set
sb.clusterhashes = sb.peers.clusterHashes(sb.getConfig("cluster.peers.yacydomain", "")); sb.clusterhashes = sb.peers.clusterHashes(sb.getConfig("cluster.peers.yacydomain", ""));
@ -150,20 +165,34 @@ public class ConfigNetwork_p {
// write remote crawl request settings // write remote crawl request settings
prop.put("crawlResponse", sb.getConfigBool("crawlResponse", false) ? "1" : "0"); prop.put("crawlResponse", sb.getConfigBool("crawlResponse", false) ? "1" : "0");
final long RTCbusySleep = Math.max(1, env.getConfigInt(SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL_BUSYSLEEP, 100)); final long RTCbusySleep =
Math
.max(1, env.getConfigInt(SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL_BUSYSLEEP, 100));
final int RTCppm = (int) (60000L / RTCbusySleep); final int RTCppm = (int) (60000L / RTCbusySleep);
prop.put("acceptCrawlLimit", RTCppm); prop.put("acceptCrawlLimit", RTCppm);
final boolean indexDistribute = sb.getConfigBool(SwitchboardConstants.INDEX_DIST_ALLOW, true); final boolean indexDistribute = sb.getConfigBool(SwitchboardConstants.INDEX_DIST_ALLOW, true);
final boolean indexReceive = sb.getConfigBool(SwitchboardConstants.INDEX_RECEIVE_ALLOW, true); final boolean indexReceive = sb.getConfigBool(SwitchboardConstants.INDEX_RECEIVE_ALLOW, true);
prop.put("indexDistributeChecked", (indexDistribute) ? "1" : "0"); prop.put("indexDistributeChecked", (indexDistribute) ? "1" : "0");
prop.put("indexDistributeWhileCrawling.on", (sb.getConfigBool(SwitchboardConstants.INDEX_DIST_ALLOW_WHILE_CRAWLING, true)) ? "1" : "0"); prop.put(
prop.put("indexDistributeWhileCrawling.off", (sb.getConfigBool(SwitchboardConstants.INDEX_DIST_ALLOW_WHILE_CRAWLING, true)) ? "0" : "1"); "indexDistributeWhileCrawling.on",
prop.put("indexDistributeWhileIndexing.on", (sb.getConfigBool(SwitchboardConstants.INDEX_DIST_ALLOW_WHILE_INDEXING, true)) ? "1" : "0"); (sb.getConfigBool(SwitchboardConstants.INDEX_DIST_ALLOW_WHILE_CRAWLING, true)) ? "1" : "0");
prop.put("indexDistributeWhileIndexing.off", (sb.getConfigBool(SwitchboardConstants.INDEX_DIST_ALLOW_WHILE_INDEXING, true)) ? "0" : "1"); prop.put(
"indexDistributeWhileCrawling.off",
(sb.getConfigBool(SwitchboardConstants.INDEX_DIST_ALLOW_WHILE_CRAWLING, true)) ? "0" : "1");
prop.put(
"indexDistributeWhileIndexing.on",
(sb.getConfigBool(SwitchboardConstants.INDEX_DIST_ALLOW_WHILE_INDEXING, true)) ? "1" : "0");
prop.put(
"indexDistributeWhileIndexing.off",
(sb.getConfigBool(SwitchboardConstants.INDEX_DIST_ALLOW_WHILE_INDEXING, true)) ? "0" : "1");
prop.put("indexReceiveChecked", (indexReceive) ? "1" : "0"); prop.put("indexReceiveChecked", (indexReceive) ? "1" : "0");
prop.put("indexReceiveBlockBlacklistChecked.on", (sb.getConfigBool("indexReceiveBlockBlacklist", true)) ? "1" : "0"); prop.put(
prop.put("indexReceiveBlockBlacklistChecked.off", (sb.getConfigBool("indexReceiveBlockBlacklist", true)) ? "0" : "1"); "indexReceiveBlockBlacklistChecked.on",
(sb.getConfigBool("indexReceiveBlockBlacklist", true)) ? "1" : "0");
prop.put(
"indexReceiveBlockBlacklistChecked.off",
(sb.getConfigBool("indexReceiveBlockBlacklist", true)) ? "0" : "1");
prop.putHTML("peertags", MapTools.set2string(sb.peers.mySeed().getPeerTags(), ",", false)); prop.putHTML("peertags", MapTools.set2string(sb.peers.mySeed().getPeerTags(), ",", false));
// set seed information directly // set seed information directly
@ -176,20 +205,25 @@ public class ConfigNetwork_p {
prop.putHTML("cluster.peers.ipport", sb.getConfig("cluster.peers.ipport", "")); prop.putHTML("cluster.peers.ipport", sb.getConfig("cluster.peers.ipport", ""));
prop.putHTML("cluster.peers.yacydomain", sb.getConfig("cluster.peers.yacydomain", "")); prop.putHTML("cluster.peers.yacydomain", sb.getConfig("cluster.peers.yacydomain", ""));
StringBuilder hashes = new StringBuilder(); StringBuilder hashes = new StringBuilder();
for (final byte[] h : sb.clusterhashes.keySet()) { for ( final byte[] h : sb.clusterhashes.keySet() ) {
hashes.append(", ").append(ASCII.String(h)); hashes.append(", ").append(ASCII.String(h));
} }
if (hashes.length() > 2) { if ( hashes.length() > 2 ) {
hashes = hashes.delete(0, 2); hashes = hashes.delete(0, 2);
} }
prop.put("cluster.peers.yacydomain.hashes", hashes.toString()); prop.put("cluster.peers.yacydomain.hashes", hashes.toString());
// set p2p mode flags // set p2p mode flags
prop.put("privatepeerChecked", ("privatepeer".equals(sb.getConfig("cluster.mode", ""))) ? "1" : "0"); prop.put(
prop.put("privateclusterChecked", ("privatecluster".equals(sb.getConfig("cluster.mode", ""))) ? "1" : "0"); "privatepeerChecked",
prop.put("publicclusterChecked", ("publiccluster".equals(sb.getConfig("cluster.mode", ""))) ? "1" : "0"); (SwitchboardConstants.CLUSTER_MODE_PRIVATE_PEER.equals(sb.getConfig(SwitchboardConstants.CLUSTER_MODE, ""))) ? "1" : "0");
prop.put("publicpeerChecked", ("publicpeer".equals(sb.getConfig("cluster.mode", ""))) ? "1" : "0"); prop.put(
"publicclusterChecked",
(SwitchboardConstants.CLUSTER_MODE_PUBLIC_CLUSTER.equals(sb.getConfig(SwitchboardConstants.CLUSTER_MODE, ""))) ? "1" : "0");
prop.put(
"publicpeerChecked",
(SwitchboardConstants.CLUSTER_MODE_PUBLIC_PEER.equals(sb.getConfig(SwitchboardConstants.CLUSTER_MODE, ""))) ? "1" : "0");
// set network configuration // set network configuration
prop.putHTML("network.unit.definition", sb.getConfig("network.unit.definition", "")); prop.putHTML("network.unit.definition", sb.getConfig("network.unit.definition", ""));
@ -199,7 +233,7 @@ public class ConfigNetwork_p {
prop.putHTML("network.unit.dht", sb.getConfig("network.unit.dht", "")); prop.putHTML("network.unit.dht", sb.getConfig("network.unit.dht", ""));
networkBootstrapLocations.remove(sb.getConfig("network.unit.definition", "")); networkBootstrapLocations.remove(sb.getConfig("network.unit.definition", ""));
int c = 0; int c = 0;
for (final String s: networkBootstrapLocations) { for ( final String s : networkBootstrapLocations ) {
prop.put("networks_" + c++ + "_network", s); prop.put("networks_" + c++ + "_network", s);
} }
prop.put("networks", c); prop.put("networks", c);
@ -211,10 +245,10 @@ public class ConfigNetwork_p {
input = input.replace(' ', ','); input = input.replace(' ', ',');
input = input.replace(' ', ';'); input = input.replace(' ', ';');
input = input.replaceAll(",,", ","); input = input.replaceAll(",,", ",");
if (input.length() > 0 && input.charAt(0) == ',') { if ( input.length() > 0 && input.charAt(0) == ',' ) {
input = input.substring(1); input = input.substring(1);
} }
if (input.endsWith(",")) { if ( input.endsWith(",") ) {
input = input.substring(0, input.length() - 1); input = input.substring(0, input.length() - 1);
} }
return input; return input;
@ -223,14 +257,16 @@ public class ConfigNetwork_p {
private static String checkYaCyDomainList(final String input) { private static String checkYaCyDomainList(final String input) {
final String[] array = normalizedList(input).split(","); final String[] array = normalizedList(input).split(",");
final StringBuilder output = new StringBuilder(); final StringBuilder output = new StringBuilder();
for (final String element : array) { for ( final String element : array ) {
if ((element.endsWith(".yacyh")) || (element.endsWith(".yacy")) || if ( (element.endsWith(".yacyh"))
(element.indexOf(".yacyh=",0) > 0) || (element.indexOf(".yacy=",0) > 0)) { || (element.endsWith(".yacy"))
|| (element.indexOf(".yacyh=", 0) > 0)
|| (element.indexOf(".yacy=", 0) > 0) ) {
output.append(",").append(element); output.append(",").append(element);
} }
} }
if (output.length() == 0) { if ( output.length() == 0 ) {
return input; return input;
} }
return output.delete(0, 1).toString(); return output.delete(0, 1).toString();
@ -239,12 +275,12 @@ public class ConfigNetwork_p {
private static String checkIPPortList(final String input) { private static String checkIPPortList(final String input) {
final String[] array = normalizedList(input).split(","); final String[] array = normalizedList(input).split(",");
final StringBuilder output = new StringBuilder(); final StringBuilder output = new StringBuilder();
for (final String element :array) { for ( final String element : array ) {
if (element.indexOf(':',0) >= 9) { if ( element.indexOf(':', 0) >= 9 ) {
output.append(",").append(element); output.append(",").append(element);
} }
} }
if (input.length() == 0) { if ( input.length() == 0 ) {
return input; return input;
} }
return output.delete(0, 1).toString(); return output.delete(0, 1).toString();

@ -69,6 +69,12 @@
<dd class="hint">You have not published your peer seed yet. This happens automatically, just wait. <dd class="hint">You have not published your peer seed yet. This happens automatically, just wait.
</dd> </dd>
#(/urgentStatusVirgin)# #(/urgentStatusVirgin)#
#(hintStatusPrivate)#::
<dt class="hintIcon"><img src="env/grafics/bad.png" width="32" height="32" alt="idea"/></dt>
<dd class="hint">Your network configuration is in private mode. Your peer seed will not be published.
</dd>
#(/hintStatusPrivate)#
<!-- warnings --> <!-- warnings -->

@ -46,67 +46,73 @@ import de.anomic.server.serverCore;
import de.anomic.server.serverObjects; import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch; import de.anomic.server.serverSwitch;
public class Status { public class Status
{
private static final String SEEDSERVER = "seedServer"; private static final String SEEDSERVER = "seedServer";
private static final String PEERSTATUS = "peerStatus"; private static final String PEERSTATUS = "peerStatus";
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { public static serverObjects respond(
final RequestHeader header,
final serverObjects post,
final serverSwitch env) {
// return variable that accumulates replacements // return variable that accumulates replacements
final serverObjects prop = new serverObjects(); final serverObjects prop = new serverObjects();
final Switchboard sb = (Switchboard) env; final Switchboard sb = (Switchboard) env;
// check if the basic configuration was accessed before and forward // check if the basic configuration was accessed before and forward
prop.put("forwardToConfigBasic", 0); prop.put("forwardToConfigBasic", 0);
if ((post == null || !post.containsKey("noforward")) && if ( (post == null || !post.containsKey("noforward"))
sb.getConfig("server.servlets.submitted", "").indexOf("ConfigBasic.html",0) < 0 && && sb.getConfig("server.servlets.submitted", "").indexOf("ConfigBasic.html", 0) < 0
Seed.isDefaultPeerName(sb.peers.mySeed().getName())) { && Seed.isDefaultPeerName(sb.peers.mySeed().getName()) ) {
// forward to ConfigBasic // forward to ConfigBasic
prop.put("forwardToConfigBasic", 1); prop.put("forwardToConfigBasic", 1);
} }
if (post != null) post.remove("noforward"); if ( post != null ) {
post.remove("noforward");
}
if (post != null && post.size() > 0) { if ( post != null && post.size() > 0 ) {
if (sb.adminAuthenticated(header) < 2) { if ( sb.adminAuthenticated(header) < 2 ) {
prop.put("AUTHENTICATE","admin log-in"); prop.put("AUTHENTICATE", "admin log-in");
return prop; return prop;
} }
boolean redirect = false; boolean redirect = false;
if (post.containsKey("login")) { if ( post.containsKey("login") ) {
prop.put("LOCATION",""); prop.put("LOCATION", "");
return prop; return prop;
} else if (post.containsKey("pauseCrawlJob")) { } else if ( post.containsKey("pauseCrawlJob") ) {
final String jobType = post.get("jobType"); final String jobType = post.get("jobType");
if ("localCrawl".equals(jobType)) { if ( "localCrawl".equals(jobType) ) {
sb.pauseCrawlJob(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL); sb.pauseCrawlJob(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL);
} else if ("remoteTriggeredCrawl".equals(jobType)) { } else if ( "remoteTriggeredCrawl".equals(jobType) ) {
sb.pauseCrawlJob(SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL); sb.pauseCrawlJob(SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL);
} }
redirect = true; redirect = true;
} else if (post.containsKey("continueCrawlJob")) { } else if ( post.containsKey("continueCrawlJob") ) {
final String jobType = post.get("jobType"); final String jobType = post.get("jobType");
if ("localCrawl".equals(jobType)) { if ( "localCrawl".equals(jobType) ) {
sb.continueCrawlJob(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL); sb.continueCrawlJob(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL);
} else if ("remoteTriggeredCrawl".equals(jobType)) { } else if ( "remoteTriggeredCrawl".equals(jobType) ) {
sb.continueCrawlJob(SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL); sb.continueCrawlJob(SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL);
} }
redirect = true; redirect = true;
} else if (post.containsKey("ResetTraffic")) { } else if ( post.containsKey("ResetTraffic") ) {
ByteCount.resetCount(); ByteCount.resetCount();
redirect = true; redirect = true;
} else if (post.containsKey("popup")) { } else if ( post.containsKey("popup") ) {
final boolean trigger_enabled = post.getBoolean("popup", false); final boolean trigger_enabled = post.getBoolean("popup", false);
sb.setConfig("browserPopUpTrigger", trigger_enabled); sb.setConfig("browserPopUpTrigger", trigger_enabled);
redirect = true; redirect = true;
} else if (post.containsKey("tray")) { } else if ( post.containsKey("tray") ) {
final boolean trigger_enabled = post.getBoolean("tray", false); final boolean trigger_enabled = post.getBoolean("tray", false);
sb.setConfig("trayIcon", trigger_enabled); sb.setConfig("trayIcon", trigger_enabled);
redirect = true; redirect = true;
} }
if (redirect) { if ( redirect ) {
prop.put("LOCATION",""); prop.put("LOCATION", "");
return prop; return prop;
} }
} }
@ -114,7 +120,7 @@ public class Status {
sb.updateMySeed(); sb.updateMySeed();
final boolean adminaccess = sb.adminAuthenticated(header) >= 2; final boolean adminaccess = sb.adminAuthenticated(header) >= 2;
if (adminaccess) { if ( adminaccess ) {
prop.put("showPrivateTable", "1"); prop.put("showPrivateTable", "1");
prop.put("privateStatusTable", "Status_p.inc"); prop.put("privateStatusTable", "Status_p.inc");
} else { } else {
@ -123,26 +129,28 @@ public class Status {
} }
// password protection // password protection
if ((sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, "").length() == 0) && (!sb.getConfigBool("adminAccountForLocalhost", false))) { if ( (sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, "").length() == 0)
&& (!sb.getConfigBool("adminAccountForLocalhost", false)) ) {
prop.put("protection", "0"); // not protected prop.put("protection", "0"); // not protected
prop.put("urgentSetPassword", "1"); prop.put("urgentSetPassword", "1");
} else { } else {
prop.put("protection", "1"); // protected prop.put("protection", "1"); // protected
} }
if (sb.getConfigBool("adminAccountForLocalhost", false)) { if ( sb.getConfigBool("adminAccountForLocalhost", false) ) {
prop.put("unrestrictedLocalAccess", 1); prop.put("unrestrictedLocalAccess", 1);
} }
// resource observer status // resource observer status
if (adminaccess) { if ( adminaccess ) {
if (!sb.observer.getDiskAvailable()){ if ( !sb.observer.getDiskAvailable() ) {
final String minFree = Formatter.bytesToString(sb.observer.getMinFreeDiskSpace()); final String minFree = Formatter.bytesToString(sb.observer.getMinFreeDiskSpace());
prop.put("warningDiskSpaceLow", "1"); prop.put("warningDiskSpaceLow", "1");
prop.put("warningDiskSpaceLow_minSpace", minFree); prop.put("warningDiskSpaceLow_minSpace", minFree);
} }
if (!sb.observer.getMemoryAvailable()){ if ( !sb.observer.getMemoryAvailable() ) {
final String minFree = Formatter.bytesToString(sb.observer.getMinFreeMemory() * 1024L * 1024L); final String minFree =
Formatter.bytesToString(sb.observer.getMinFreeMemory() * 1024L * 1024L);
prop.put("warningMemoryLow", "1"); prop.put("warningMemoryLow", "1");
prop.put("warningMemoryLow_minSpace", minFree); prop.put("warningMemoryLow_minSpace", minFree);
} }
@ -151,25 +159,26 @@ public class Status {
// version information // version information
//final String versionstring = yacyVersion.combined2prettyVersion(sb.getConfig("version","0.1")); //final String versionstring = yacyVersion.combined2prettyVersion(sb.getConfig("version","0.1"));
final String versionstring = yacyBuildProperties.getVersion() + "/" + yacyBuildProperties.getSVNRevision(); final String versionstring =
yacyBuildProperties.getVersion() + "/" + yacyBuildProperties.getSVNRevision();
prop.put("versionpp", versionstring); prop.put("versionpp", versionstring);
// place some more hints // place some more hints
if ((adminaccess) && (sb.getThread(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL).getJobCount() == 0)) { if ( (adminaccess) && (sb.getThread(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL).getJobCount() == 0) ) {
prop.put("hintCrawlStart", "1"); prop.put("hintCrawlStart", "1");
} }
if ((adminaccess) && (sb.getThread(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL).getJobCount() > 500)) { if ( (adminaccess) && (sb.getThread(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL).getJobCount() > 500) ) {
prop.put("hintCrawlMonitor", "1"); prop.put("hintCrawlMonitor", "1");
} }
// hostname and port // hostname and port
final String extendedPortString = sb.getConfig("port", "8090"); final String extendedPortString = sb.getConfig("port", "8090");
final int pos = extendedPortString.indexOf(':',0); final int pos = extendedPortString.indexOf(':', 0);
prop.put("port",serverCore.getPortNr(extendedPortString)); prop.put("port", serverCore.getPortNr(extendedPortString));
if (pos != -1) { if ( pos != -1 ) {
prop.put("extPortFormat", "1"); prop.put("extPortFormat", "1");
prop.putHTML("extPortFormat_extPort",extendedPortString); prop.putHTML("extPortFormat_extPort", extendedPortString);
} else { } else {
prop.put("extPortFormat", "0"); prop.put("extPortFormat", "0");
} }
@ -177,9 +186,9 @@ public class Status {
prop.put("host", hostIP != null ? hostIP.getHostAddress() : "Unkown IP"); prop.put("host", hostIP != null ? hostIP.getHostAddress() : "Unkown IP");
// ssl support // ssl support
prop.put("sslSupport",sb.getConfig("keyStore", "").length() == 0 ? "0" : "1"); prop.put("sslSupport", sb.getConfig("keyStore", "").length() == 0 ? "0" : "1");
if (sb.getConfigBool("remoteProxyUse", false)) { if ( sb.getConfigBool("remoteProxyUse", false) ) {
prop.put("remoteProxy", "1"); prop.put("remoteProxy", "1");
prop.putXML("remoteProxy_host", sb.getConfig("remoteProxyHost", "<unknown>")); prop.putXML("remoteProxy_host", sb.getConfig("remoteProxyHost", "<unknown>"));
prop.putXML("remoteProxy_port", sb.getConfig("remoteProxyPort", "<unknown>")); prop.putXML("remoteProxy_port", sb.getConfig("remoteProxyPort", "<unknown>"));
@ -191,22 +200,24 @@ public class Status {
// peer information // peer information
String thisHash = ""; String thisHash = "";
final String thisName = sb.peers.mySeed().getName(); final String thisName = sb.peers.mySeed().getName();
if (sb.peers.mySeed() == null) { if ( sb.peers.mySeed() == null ) {
thisHash = "not assigned"; thisHash = "not assigned";
prop.put("peerAddress", "0"); // not assigned prop.put("peerAddress", "0"); // not assigned
prop.put("peerStatistics", "0"); // unknown prop.put("peerStatistics", "0"); // unknown
} else { } else {
final long uptime = 60000 * sb.peers.mySeed().getLong(Seed.UPTIME, 0L); final long uptime = 60000 * sb.peers.mySeed().getLong(Seed.UPTIME, 0L);
prop.put("peerStatistics", "1"); prop.put("peerStatistics", "1");
prop.put("peerStatistics_uptime", PeerActions.formatInterval(uptime)); prop.put("peerStatistics_uptime", PeerActions.formatInterval(uptime));
prop.putNum("peerStatistics_pagesperminute", sb.peers.mySeed().getPPM()); prop.putNum("peerStatistics_pagesperminute", sb.peers.mySeed().getPPM());
prop.putNum("peerStatistics_queriesperhour", Math.round(6000d * sb.peers.mySeed().getQPM()) / 100d); prop.putNum(
"peerStatistics_queriesperhour",
Math.round(6000d * sb.peers.mySeed().getQPM()) / 100d);
prop.putNum("peerStatistics_links", sb.peers.mySeed().getLinkCount()); prop.putNum("peerStatistics_links", sb.peers.mySeed().getLinkCount());
prop.put("peerStatistics_words", Formatter.number(sb.peers.mySeed().getWordCount())); prop.put("peerStatistics_words", Formatter.number(sb.peers.mySeed().getWordCount()));
prop.putNum("peerStatistics_disconnects", sb.peers.peerActions.disconnects); prop.putNum("peerStatistics_disconnects", sb.peers.peerActions.disconnects);
prop.put("peerStatistics_connects", Formatter.number(sb.peers.mySeed().get(Seed.CCOUNT, "0"))); prop.put("peerStatistics_connects", Formatter.number(sb.peers.mySeed().get(Seed.CCOUNT, "0")));
thisHash = sb.peers.mySeed().hash; thisHash = sb.peers.mySeed().hash;
if (sb.peers.mySeed().getPublicAddress() == null) { if ( sb.peers.mySeed().getPublicAddress() == null ) {
prop.put("peerAddress", "0"); // not assigned + instructions prop.put("peerAddress", "0"); // not assigned + instructions
prop.put("warningGoOnline", "1"); prop.put("warningGoOnline", "1");
} else { } else {
@ -215,17 +226,25 @@ public class Status {
prop.putXML("peerAddress_peername", sb.peers.mySeed().getName().toLowerCase()); prop.putXML("peerAddress_peername", sb.peers.mySeed().getName().toLowerCase());
} }
} }
final String peerStatus = ((sb.peers.mySeed() == null) ? Seed.PEERTYPE_VIRGIN : sb.peers.mySeed().get(Seed.PEERTYPE, Seed.PEERTYPE_VIRGIN)); final String peerStatus =
if (Seed.PEERTYPE_VIRGIN.equals(peerStatus) && "freeworld".equals(sb.getConfig(SwitchboardConstants.NETWORK_NAME, ""))) { ((sb.peers.mySeed() == null) ? Seed.PEERTYPE_VIRGIN : sb.peers.mySeed().get(
Seed.PEERTYPE,
Seed.PEERTYPE_VIRGIN));
if ( Seed.PEERTYPE_VIRGIN.equals(peerStatus)
&& "freeworld".equals(sb.getConfig(SwitchboardConstants.NETWORK_NAME, ""))
&& !SwitchboardConstants.CLUSTER_MODE_PRIVATE_PEER.equals(sb.getConfig(SwitchboardConstants.CLUSTER_MODE, ""))) {
prop.put(PEERSTATUS, "0"); prop.put(PEERSTATUS, "0");
prop.put("urgentStatusVirgin", "1"); prop.put("urgentStatusVirgin", "1");
} else if (Seed.PEERTYPE_JUNIOR.equals(peerStatus) && "freeworld".equals(sb.getConfig(SwitchboardConstants.NETWORK_NAME, ""))) { } else if ( Seed.PEERTYPE_JUNIOR.equals(peerStatus)
&& "freeworld".equals(sb.getConfig(SwitchboardConstants.NETWORK_NAME, ""))
&& !SwitchboardConstants.CLUSTER_MODE_PRIVATE_PEER.equals(sb.getConfig(SwitchboardConstants.CLUSTER_MODE, ""))) {
prop.put(PEERSTATUS, "1"); prop.put(PEERSTATUS, "1");
prop.put("warningStatusJunior", "1"); prop.put("warningStatusJunior", "1");
} else if (Seed.PEERTYPE_SENIOR.equals(peerStatus)) { } else if ( Seed.PEERTYPE_SENIOR.equals(peerStatus) ) {
prop.put(PEERSTATUS, "2"); prop.put(PEERSTATUS, "2");
prop.put("hintStatusSenior", "1"); prop.put("hintStatusSenior", "1");
} else if (Seed.PEERTYPE_PRINCIPAL.equals(peerStatus)) { } else if ( Seed.PEERTYPE_PRINCIPAL.equals(peerStatus) ) {
prop.put(PEERSTATUS, "3"); prop.put(PEERSTATUS, "3");
prop.put("hintStatusPrincipal", "1"); prop.put("hintStatusPrincipal", "1");
prop.put("hintStatusPrincipal_seedURL", sb.peers.mySeed().get(Seed.SEEDLISTURL, "?")); prop.put("hintStatusPrincipal_seedURL", sb.peers.mySeed().get(Seed.SEEDLISTURL, "?"));
@ -234,50 +253,52 @@ public class Status {
prop.put("hash", thisHash); prop.put("hash", thisHash);
final String seedUploadMethod = sb.getConfig("seedUploadMethod", ""); final String seedUploadMethod = sb.getConfig("seedUploadMethod", "");
if (!"none".equalsIgnoreCase(seedUploadMethod) || if ( !"none".equalsIgnoreCase(seedUploadMethod)
("".equals(seedUploadMethod) && (sb.getConfig("seedFTPPassword", "").length() > 0 || || ("".equals(seedUploadMethod) && (sb.getConfig("seedFTPPassword", "").length() > 0 || sb
sb.getConfig("seedFilePath", "").length() > 0))) { .getConfig("seedFilePath", "")
if ("".equals(seedUploadMethod)) { .length() > 0)) ) {
if (sb.getConfig("seedFTPPassword", "").length() > 0) { if ( "".equals(seedUploadMethod) ) {
sb.setConfig("seedUploadMethod","Ftp"); if ( sb.getConfig("seedFTPPassword", "").length() > 0 ) {
sb.setConfig("seedUploadMethod", "Ftp");
} }
if (sb.getConfig("seedFilePath", "").length() > 0) { if ( sb.getConfig("seedFilePath", "").length() > 0 ) {
sb.setConfig("seedUploadMethod","File"); sb.setConfig("seedUploadMethod", "File");
} }
} }
if ("ftp".equalsIgnoreCase(seedUploadMethod)) { if ( "ftp".equalsIgnoreCase(seedUploadMethod) ) {
prop.put(SEEDSERVER, "1"); // enabled prop.put(SEEDSERVER, "1"); // enabled
prop.putHTML("seedServer_seedServer", sb.getConfig("seedFTPServer", "")); prop.putHTML("seedServer_seedServer", sb.getConfig("seedFTPServer", ""));
} else if ("scp".equalsIgnoreCase(seedUploadMethod)) { } else if ( "scp".equalsIgnoreCase(seedUploadMethod) ) {
prop.put(SEEDSERVER, "1"); // enabled prop.put(SEEDSERVER, "1"); // enabled
prop.putHTML("seedServer_seedServer", sb.getConfig("seedScpServer", "")); prop.putHTML("seedServer_seedServer", sb.getConfig("seedScpServer", ""));
} else if ("file".equalsIgnoreCase(seedUploadMethod)) { } else if ( "file".equalsIgnoreCase(seedUploadMethod) ) {
prop.put(SEEDSERVER, "2"); // enabled prop.put(SEEDSERVER, "2"); // enabled
prop.putHTML("seedServer_seedFile", sb.getConfig("seedFilePath", "")); prop.putHTML("seedServer_seedFile", sb.getConfig("seedFilePath", ""));
} }
prop.put("seedServer_lastUpload", prop.put(
PeerActions.formatInterval(System.currentTimeMillis() - sb.peers.lastSeedUpload_timeStamp)); "seedServer_lastUpload",
PeerActions.formatInterval(System.currentTimeMillis() - sb.peers.lastSeedUpload_timeStamp));
} else { } else {
prop.put(SEEDSERVER, "0"); // disabled prop.put(SEEDSERVER, "0"); // disabled
} }
if (sb.peers != null && sb.peers.sizeConnected() > 0){ if ( sb.peers != null && sb.peers.sizeConnected() > 0 ) {
prop.put("otherPeers", "1"); prop.put("otherPeers", "1");
prop.putNum("otherPeers_num", sb.peers.sizeConnected()); prop.putNum("otherPeers_num", sb.peers.sizeConnected());
}else{ } else {
prop.put("otherPeers", "0"); // not online prop.put("otherPeers", "0"); // not online
} }
if (!sb.getConfigBool("browserPopUpTrigger", false)) { if ( !sb.getConfigBool("browserPopUpTrigger", false) ) {
prop.put("popup", "0"); prop.put("popup", "0");
} else { } else {
prop.put("popup", "1"); prop.put("popup", "1");
} }
if (!OS.isWindows) { if ( !OS.isWindows ) {
prop.put("tray", "2"); prop.put("tray", "2");
} else if (!sb.getConfigBool("trayIcon", false)) { } else if ( !sb.getConfigBool("trayIcon", false) ) {
prop.put("tray", "0"); prop.put("tray", "0");
} else { } else {
prop.put("tray", "1"); prop.put("tray", "1");
@ -305,18 +326,26 @@ public class Status {
final int loaderPercent = (loaderMaxCount == 0) ? 0 : loaderJobCount * 100 / loaderMaxCount; final int loaderPercent = (loaderMaxCount == 0) ? 0 : loaderJobCount * 100 / loaderMaxCount;
prop.putNum("loaderQueueSize", loaderJobCount); prop.putNum("loaderQueueSize", loaderJobCount);
prop.putNum("loaderQueueMax", loaderMaxCount); prop.putNum("loaderQueueMax", loaderMaxCount);
prop.put("loaderQueuePercent", (loaderPercent>100) ? 100 : loaderPercent); prop.put("loaderQueuePercent", (loaderPercent > 100) ? 100 : loaderPercent);
prop.putNum("localCrawlQueueSize", sb.getThread(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL).getJobCount()); prop.putNum("localCrawlQueueSize", sb
prop.put("localCrawlPaused",sb.crawlJobIsPaused(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL) ? "1" : "0"); .getThread(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL)
.getJobCount());
prop.putNum("remoteTriggeredCrawlQueueSize", sb.getThread(SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL).getJobCount()); prop.put("localCrawlPaused", sb.crawlJobIsPaused(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL)
prop.put("remoteTriggeredCrawlPaused",sb.crawlJobIsPaused(SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL) ? "1" : "0"); ? "1"
: "0");
prop.putNum(
"remoteTriggeredCrawlQueueSize",
sb.getThread(SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL).getJobCount());
prop.put(
"remoteTriggeredCrawlPaused",
sb.crawlJobIsPaused(SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL) ? "1" : "0");
prop.putNum("stackCrawlQueueSize", sb.crawlStacker.size()); prop.putNum("stackCrawlQueueSize", sb.crawlStacker.size());
// return rewrite properties // return rewrite properties
prop.put("date",(new Date()).toString()); prop.put("date", (new Date()).toString());
return prop; return prop;
} }
} }

File diff suppressed because it is too large Load Diff

@ -142,11 +142,19 @@ public class CrawlQueues {
* @return if the hash exists, the name of the database is returned, otherwise null is returned * @return if the hash exists, the name of the database is returned, otherwise null is returned
*/ */
public String urlExists(final byte[] hash) { public String urlExists(final byte[] hash) {
if (this.delegatedURL.exists(hash)) return "delegated"; if (this.delegatedURL.exists(hash)) {
if (this.errorURL.exists(hash)) return "errors"; return "delegated";
if (this.noticeURL.existsInStack(hash)) return "crawler"; }
if (this.errorURL.exists(hash)) {
return "errors";
}
if (this.noticeURL.existsInStack(hash)) {
return "crawler";
}
for (final Loader worker: this.workers.values()) { for (final Loader worker: this.workers.values()) {
if (Base64Order.enhancedCoder.equal(worker.request.url().hash(), hash)) return "worker"; if (Base64Order.enhancedCoder.equal(worker.request.url().hash(), hash)) {
return "worker";
}
} }
return null; return null;
} }
@ -159,16 +167,26 @@ public class CrawlQueues {
public DigestURI getURL(final byte[] urlhash) { public DigestURI getURL(final byte[] urlhash) {
assert urlhash != null; assert urlhash != null;
if (urlhash == null || urlhash.length == 0) return null; if (urlhash == null || urlhash.length == 0) {
return null;
}
ZURL.Entry ee = this.delegatedURL.get(urlhash); ZURL.Entry ee = this.delegatedURL.get(urlhash);
if (ee != null) return ee.url(); if (ee != null) {
return ee.url();
}
ee = this.errorURL.get(urlhash); ee = this.errorURL.get(urlhash);
if (ee != null) return ee.url(); if (ee != null) {
return ee.url();
}
for (final Loader w: this.workers.values()) { for (final Loader w: this.workers.values()) {
if (Base64Order.enhancedCoder.equal(w.request.url().hash(), urlhash)) return w.request.url(); if (Base64Order.enhancedCoder.equal(w.request.url().hash(), urlhash)) {
return w.request.url();
}
} }
final Request ne = this.noticeURL.get(urlhash); final Request ne = this.noticeURL.get(urlhash);
if (ne != null) return ne.url(); if (ne != null) {
return ne.url();
}
return null; return null;
} }
@ -176,7 +194,9 @@ public class CrawlQueues {
// wait for all workers to finish // wait for all workers to finish
final int timeout = (int) this.sb.getConfigLong("crawler.clientTimeout", 10000); final int timeout = (int) this.sb.getConfigLong("crawler.clientTimeout", 10000);
for (final Loader w: this.workers.values()) { for (final Loader w: this.workers.values()) {
if (w.age() > timeout) w.interrupt(); if (w.age() > timeout) {
w.interrupt();
}
} }
} }
@ -185,7 +205,9 @@ public class CrawlQueues {
final Request[] e = new Request[this.workers.size()]; final Request[] e = new Request[this.workers.size()];
int i = 0; int i = 0;
for (final Loader w: this.workers.values()) { for (final Loader w: this.workers.values()) {
if (i >= e.length) break; if (i >= e.length) {
break;
}
e[i++] = w.request; e[i++] = w.request;
} }
return e; return e;
@ -199,8 +221,7 @@ public class CrawlQueues {
public boolean coreCrawlJob() { public boolean coreCrawlJob() {
final boolean robinsonPrivateCase = (this.sb.isRobinsonMode() && final boolean robinsonPrivateCase = (this.sb.isRobinsonMode() &&
!this.sb.getConfig(SwitchboardConstants.CLUSTER_MODE, "").equals(SwitchboardConstants.CLUSTER_MODE_PUBLIC_CLUSTER) && !this.sb.getConfig(SwitchboardConstants.CLUSTER_MODE, "").equals(SwitchboardConstants.CLUSTER_MODE_PUBLIC_CLUSTER));
!this.sb.getConfig(SwitchboardConstants.CLUSTER_MODE, "").equals(SwitchboardConstants.CLUSTER_MODE_PRIVATE_CLUSTER));
if ((robinsonPrivateCase || coreCrawlJobSize() <= 20) && limitCrawlJobSize() > 0) { if ((robinsonPrivateCase || coreCrawlJobSize() <= 20) && limitCrawlJobSize() > 0) {
// move some tasks to the core crawl job so we have something to do // move some tasks to the core crawl job so we have something to do
@ -216,12 +237,16 @@ public class CrawlQueues {
final String queueCheckCore = loadIsPossible(NoticedURL.StackType.CORE); final String queueCheckCore = loadIsPossible(NoticedURL.StackType.CORE);
final String queueCheckNoload = loadIsPossible(NoticedURL.StackType.NOLOAD); final String queueCheckNoload = loadIsPossible(NoticedURL.StackType.NOLOAD);
if (queueCheckCore != null && queueCheckNoload != null) { if (queueCheckCore != null && queueCheckNoload != null) {
if (this.log.isFine()) this.log.logFine("omitting de-queue/local: " + queueCheckCore + ":" + queueCheckNoload); if (this.log.isFine()) {
this.log.logFine("omitting de-queue/local: " + queueCheckCore + ":" + queueCheckNoload);
}
return false; return false;
} }
if (isPaused(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL)) { if (isPaused(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL)) {
if (this.log.isFine()) this.log.logFine("omitting de-queue/local: paused"); if (this.log.isFine()) {
this.log.logFine("omitting de-queue/local: paused");
}
return false; return false;
} }
@ -238,7 +263,9 @@ public class CrawlQueues {
if (this.noticeURL.stackSize(NoticedURL.StackType.NOLOAD) > 0) { if (this.noticeURL.stackSize(NoticedURL.StackType.NOLOAD) > 0) {
// get one entry that will not be loaded, just indexed // get one entry that will not be loaded, just indexed
urlEntry = this.noticeURL.pop(NoticedURL.StackType.NOLOAD, true, this.sb.crawler); urlEntry = this.noticeURL.pop(NoticedURL.StackType.NOLOAD, true, this.sb.crawler);
if (urlEntry == null) continue; if (urlEntry == null) {
continue;
}
final String profileHandle = urlEntry.profileHandle(); final String profileHandle = urlEntry.profileHandle();
if (profileHandle == null) { if (profileHandle == null) {
this.log.logSevere(stats + ": NULL PROFILE HANDLE '" + urlEntry.profileHandle() + "' for URL " + urlEntry.url()); this.log.logSevere(stats + ": NULL PROFILE HANDLE '" + urlEntry.profileHandle() + "' for URL " + urlEntry.url());
@ -259,7 +286,9 @@ public class CrawlQueues {
} }
urlEntry = this.noticeURL.pop(NoticedURL.StackType.CORE, true, this.sb.crawler); urlEntry = this.noticeURL.pop(NoticedURL.StackType.CORE, true, this.sb.crawler);
if (urlEntry == null) continue; if (urlEntry == null) {
continue;
}
final String profileHandle = urlEntry.profileHandle(); final String profileHandle = urlEntry.profileHandle();
// System.out.println("DEBUG plasmaSwitchboard.processCrawling: // System.out.println("DEBUG plasmaSwitchboard.processCrawling:
// profileHandle = " + profileHandle + ", urlEntry.url = " + urlEntry.url()); // profileHandle = " + profileHandle + ", urlEntry.url = " + urlEntry.url());
@ -271,7 +300,9 @@ public class CrawlQueues {
return true; return true;
} catch (final IOException e) { } catch (final IOException e) {
this.log.logSevere(stats + ": CANNOT FETCH ENTRY: " + e.getMessage(), e); this.log.logSevere(stats + ": CANNOT FETCH ENTRY: " + e.getMessage(), e);
if (e.getMessage().indexOf("hash is null",0) > 0) this.noticeURL.clear(NoticedURL.StackType.CORE); if (e.getMessage().indexOf("hash is null",0) > 0) {
this.noticeURL.clear(NoticedURL.StackType.CORE);
}
} }
} }
return true; return true;
@ -293,7 +324,7 @@ public class CrawlQueues {
final DigestURI url = urlEntry.url(); final DigestURI url = urlEntry.url();
final String urlProtocol = url.getProtocol(); final String urlProtocol = url.getProtocol();
if (this.sb.loader.isSupportedProtocol(urlProtocol)) { if (this.sb.loader.isSupportedProtocol(urlProtocol)) {
if (this.log.isFine()) if (this.log.isFine()) {
this.log.logFine(stats + ": URL=" + urlEntry.url() this.log.logFine(stats + ": URL=" + urlEntry.url()
+ ", initiator=" + ((urlEntry.initiator() == null) ? "" : ASCII.String(urlEntry.initiator())) + ", initiator=" + ((urlEntry.initiator() == null) ? "" : ASCII.String(urlEntry.initiator()))
+ ", crawlOrder=" + ((profile.remoteIndexing()) ? "true" : "false") + ", crawlOrder=" + ((profile.remoteIndexing()) ? "true" : "false")
@ -302,6 +333,7 @@ public class CrawlQueues {
+ ", must-match=" + profile.urlMustMatchPattern().toString() + ", must-match=" + profile.urlMustMatchPattern().toString()
+ ", must-not-match=" + profile.urlMustNotMatchPattern().toString() + ", must-not-match=" + profile.urlMustNotMatchPattern().toString()
+ ", permission=" + ((this.sb.peers == null) ? "undefined" : (((this.sb.peers.mySeed().isSenior()) || (this.sb.peers.mySeed().isPrincipal())) ? "true" : "false"))); + ", permission=" + ((this.sb.peers == null) ? "undefined" : (((this.sb.peers.mySeed().isSenior()) || (this.sb.peers.mySeed().isPrincipal())) ? "true" : "false")));
}
// work off one Crawl stack entry // work off one Crawl stack entry
if (urlEntry == null || urlEntry.url() == null) { if (urlEntry == null || urlEntry.url() == null) {
@ -387,23 +419,31 @@ public class CrawlQueues {
} }
// check again // check again
if (this.workers.size() >= this.sb.getConfigLong(SwitchboardConstants.CRAWLER_THREADS_ACTIVE_MAX, 20)) { if (this.workers.size() >= this.sb.getConfigLong(SwitchboardConstants.CRAWLER_THREADS_ACTIVE_MAX, 20)) {
if (this.log.isFine()) this.log.logFine("remoteCrawlLoaderJob: too many processes in loader queue, dismissed (" + "cacheLoader=" + this.workers.size() + "), httpClients = " + ConnectionInfo.getCount()); if (this.log.isFine()) {
this.log.logFine("remoteCrawlLoaderJob: too many processes in loader queue, dismissed (" + "cacheLoader=" + this.workers.size() + "), httpClients = " + ConnectionInfo.getCount());
}
return false; return false;
} }
final String cautionCause = this.sb.onlineCaution(); final String cautionCause = this.sb.onlineCaution();
if (cautionCause != null) { if (cautionCause != null) {
if (this.log.isFine()) this.log.logFine("remoteCrawlLoaderJob: online caution for " + cautionCause + ", omitting processing"); if (this.log.isFine()) {
this.log.logFine("remoteCrawlLoaderJob: online caution for " + cautionCause + ", omitting processing");
}
return false; return false;
} }
if (remoteTriggeredCrawlJobSize() > 200) { if (remoteTriggeredCrawlJobSize() > 200) {
if (this.log.isFine()) this.log.logFine("remoteCrawlLoaderJob: the remote-triggered crawl job queue is filled, omitting processing"); if (this.log.isFine()) {
this.log.logFine("remoteCrawlLoaderJob: the remote-triggered crawl job queue is filled, omitting processing");
}
return false; return false;
} }
if (coreCrawlJobSize() > 0 /*&& sb.indexingStorageProcessor.queueSize() > 0*/) { if (coreCrawlJobSize() > 0 /*&& sb.indexingStorageProcessor.queueSize() > 0*/) {
if (this.log.isFine()) this.log.logFine("remoteCrawlLoaderJob: a local crawl is running, omitting processing"); if (this.log.isFine()) {
this.log.logFine("remoteCrawlLoaderJob: a local crawl is running, omitting processing");
}
return false; return false;
} }
@ -414,27 +454,37 @@ public class CrawlQueues {
final Iterator<Seed> e = PeerSelection.getProvidesRemoteCrawlURLs(this.sb.peers); final Iterator<Seed> e = PeerSelection.getProvidesRemoteCrawlURLs(this.sb.peers);
while (e.hasNext()) { while (e.hasNext()) {
seed = e.next(); seed = e.next();
if (seed != null) this.remoteCrawlProviderHashes.add(seed.hash); if (seed != null) {
this.remoteCrawlProviderHashes.add(seed.hash);
}
} }
} }
} }
if (this.remoteCrawlProviderHashes.isEmpty()) return false; if (this.remoteCrawlProviderHashes.isEmpty()) {
return false;
}
// take one entry from the provider list and load the entries from the remote peer // take one entry from the provider list and load the entries from the remote peer
seed = null; seed = null;
String hash = null; String hash = null;
while (seed == null && !this.remoteCrawlProviderHashes.isEmpty()) { while (seed == null && !this.remoteCrawlProviderHashes.isEmpty()) {
hash = this.remoteCrawlProviderHashes.remove(this.remoteCrawlProviderHashes.size() - 1); hash = this.remoteCrawlProviderHashes.remove(this.remoteCrawlProviderHashes.size() - 1);
if (hash == null) continue; if (hash == null) {
continue;
}
seed = this.sb.peers.get(hash); seed = this.sb.peers.get(hash);
if (seed == null) continue; if (seed == null) {
continue;
}
// check if the peer is inside our cluster // check if the peer is inside our cluster
if ((this.sb.isRobinsonMode()) && (!this.sb.isInMyCluster(seed))) { if ((this.sb.isRobinsonMode()) && (!this.sb.isInMyCluster(seed))) {
seed = null; seed = null;
continue; continue;
} }
} }
if (seed == null) return false; if (seed == null) {
return false;
}
// we know a peer which should provide remote crawl entries. load them now. // we know a peer which should provide remote crawl entries. load them now.
final RSSFeed feed = Protocol.queryRemoteCrawlURLs(this.sb.peers, seed, 60, 8000); final RSSFeed feed = Protocol.queryRemoteCrawlURLs(this.sb.peers, seed, 60, 8000);
@ -467,7 +517,9 @@ public class CrawlQueues {
final String urlRejectReason = this.sb.crawlStacker.urlInAcceptedDomain(url); final String urlRejectReason = this.sb.crawlStacker.urlInAcceptedDomain(url);
if (urlRejectReason == null) { if (urlRejectReason == null) {
// stack url // stack url
if (this.sb.getLog().isFinest()) this.sb.getLog().logFinest("crawlOrder: stack: url='" + url + "'"); if (this.sb.getLog().isFinest()) {
this.sb.getLog().logFinest("crawlOrder: stack: url='" + url + "'");
}
this.sb.crawlStacker.enqueueEntry(new Request( this.sb.crawlStacker.enqueueEntry(new Request(
ASCII.getBytes(hash), ASCII.getBytes(hash),
url, url,
@ -510,12 +562,16 @@ public class CrawlQueues {
// or there is no global crawl on the stack // or there is no global crawl on the stack
final String queueCheck = loadIsPossible(NoticedURL.StackType.REMOTE); final String queueCheck = loadIsPossible(NoticedURL.StackType.REMOTE);
if (queueCheck != null) { if (queueCheck != null) {
if (this.log.isFinest()) this.log.logFinest("omitting de-queue/remote: " + queueCheck); if (this.log.isFinest()) {
this.log.logFinest("omitting de-queue/remote: " + queueCheck);
}
return false; return false;
} }
if (isPaused(SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL)) { if (isPaused(SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL)) {
if (this.log.isFinest()) this.log.logFinest("omitting de-queue/remote: paused"); if (this.log.isFinest()) {
this.log.logFinest("omitting de-queue/remote: paused");
}
return false; return false;
} }
@ -532,7 +588,9 @@ public class CrawlQueues {
return true; return true;
} catch (final IOException e) { } catch (final IOException e) {
this.log.logSevere(stats + ": CANNOT FETCH ENTRY: " + e.getMessage(), e); this.log.logSevere(stats + ": CANNOT FETCH ENTRY: " + e.getMessage(), e);
if (e.getMessage().indexOf("hash is null",0) > 0) this.noticeURL.clear(NoticedURL.StackType.REMOTE); if (e.getMessage().indexOf("hash is null",0) > 0) {
this.noticeURL.clear(NoticedURL.StackType.REMOTE);
}
return true; return true;
} }
} }
@ -599,7 +657,9 @@ public class CrawlQueues {
final Response response = CrawlQueues.this.sb.loader.load(this.request, e == null ? CacheStrategy.IFEXIST : e.cacheStrategy(), true); final Response response = CrawlQueues.this.sb.loader.load(this.request, e == null ? CacheStrategy.IFEXIST : e.cacheStrategy(), true);
if (response == null) { if (response == null) {
this.request.setStatus("error", WorkflowJob.STATUS_FINISHED); this.request.setStatus("error", WorkflowJob.STATUS_FINISHED);
if (CrawlQueues.this.log.isFine()) CrawlQueues.this.log.logFine("problem loading " + this.request.url().toString() + ": no content (possibly caused by cache policy)"); if (CrawlQueues.this.log.isFine()) {
CrawlQueues.this.log.logFine("problem loading " + this.request.url().toString() + ": no content (possibly caused by cache policy)");
}
result = "no content (possibly caused by cache policy)"; result = "no content (possibly caused by cache policy)";
} else { } else {
this.request.setStatus("loaded", WorkflowJob.STATUS_RUNNING); this.request.setStatus("loaded", WorkflowJob.STATUS_RUNNING);
@ -609,7 +669,9 @@ public class CrawlQueues {
} }
} catch (final IOException e) { } catch (final IOException e) {
this.request.setStatus("error", WorkflowJob.STATUS_FINISHED); this.request.setStatus("error", WorkflowJob.STATUS_FINISHED);
if (CrawlQueues.this.log.isFine()) CrawlQueues.this.log.logFine("problem loading " + this.request.url().toString() + ": " + e.getMessage()); if (CrawlQueues.this.log.isFine()) {
CrawlQueues.this.log.logFine("problem loading " + this.request.url().toString() + ": " + e.getMessage());
}
result = "load error - " + e.getMessage(); result = "load error - " + e.getMessage();
} }

@ -61,16 +61,18 @@ import net.yacy.peers.operation.yacySeedUploadFtp;
import net.yacy.peers.operation.yacySeedUploadScp; import net.yacy.peers.operation.yacySeedUploadScp;
import net.yacy.peers.operation.yacySeedUploader; import net.yacy.peers.operation.yacySeedUploader;
import net.yacy.search.Switchboard; import net.yacy.search.Switchboard;
import net.yacy.search.SwitchboardConstants;
import de.anomic.server.serverCore; import de.anomic.server.serverCore;
public class Network { public class Network
{
// statics // statics
public static final ThreadGroup publishThreadGroup = new ThreadGroup("publishThreadGroup"); public static final ThreadGroup publishThreadGroup = new ThreadGroup("publishThreadGroup");
public static final HashMap<String, String> seedUploadMethods = new HashMap<String, String>(); public static final HashMap<String, String> seedUploadMethods = new HashMap<String, String>();
public static final Log log = new Log("YACY"); public static final Log log = new Log("YACY");
public static long lastOnlineTime = 0; public static long lastOnlineTime = 0;
/** pseudo-random key derived from a time-interval while YaCy startup*/ /** pseudo-random key derived from a time-interval while YaCy startup */
public static long speedKey = 0; public static long speedKey = 0;
public static long magic = System.currentTimeMillis(); public static long magic = System.currentTimeMillis();
public static final Map<String, Accessible> amIAccessibleDB = new ConcurrentHashMap<String, Accessible>(); // Holds PeerHash / yacyAccessible Relations public static final Map<String, Accessible> amIAccessibleDB = new ConcurrentHashMap<String, Accessible>(); // Holds PeerHash / yacyAccessible Relations
@ -105,10 +107,10 @@ public class Network {
// ensure that correct IP is used // ensure that correct IP is used
final String staticIP = sb.getConfig("staticIP", ""); final String staticIP = sb.getConfig("staticIP", "");
if (staticIP.length() != 0 && Seed.isProperIP(staticIP) == null) { if ( staticIP.length() != 0 && Seed.isProperIP(staticIP) == null ) {
serverCore.useStaticIP = true; serverCore.useStaticIP = true;
sb.peers.mySeed().setIP(staticIP); sb.peers.mySeed().setIP(staticIP);
log.logInfo("staticIP set to "+ staticIP); log.logInfo("staticIP set to " + staticIP);
} else { } else {
serverCore.useStaticIP = false; serverCore.useStaticIP = false;
} }
@ -127,7 +129,9 @@ public class Network {
} }
public final void publishSeedList() { public final void publishSeedList() {
if (log.isFine()) log.logFine("yacyCore.publishSeedList: Triggered Seed Publish"); if ( log.isFine() ) {
log.logFine("yacyCore.publishSeedList: Triggered Seed Publish");
}
/* /*
if (oldIPStamp.equals((String) seedDB.mySeed.get(yacySeed.IP, "127.0.0.1"))) if (oldIPStamp.equals((String) seedDB.mySeed.get(yacySeed.IP, "127.0.0.1")))
@ -138,45 +142,48 @@ public class Network {
yacyCore.log.logDebug("***DEBUG publishSeedList: I can reach myself"); yacyCore.log.logDebug("***DEBUG publishSeedList: I can reach myself");
*/ */
if ((this.sb.peers.lastSeedUpload_myIP.equals(this.sb.peers.mySeed().getIP())) && if ( (this.sb.peers.lastSeedUpload_myIP.equals(this.sb.peers.mySeed().getIP()))
(this.sb.peers.lastSeedUpload_seedDBSize == this.sb.peers.sizeConnected()) && && (this.sb.peers.lastSeedUpload_seedDBSize == this.sb.peers.sizeConnected())
(canReachMyself()) && && (canReachMyself())
(System.currentTimeMillis() - this.sb.peers.lastSeedUpload_timeStamp < 1000 * 60 * 60 * 24) && && (System.currentTimeMillis() - this.sb.peers.lastSeedUpload_timeStamp < 1000 * 60 * 60 * 24)
(this.sb.peers.mySeed().isPrincipal()) && (this.sb.peers.mySeed().isPrincipal()) ) {
) { if ( log.isFine() ) {
if (log.isFine()) log.logFine("yacyCore.publishSeedList: not necessary to publish: oldIP is equal, sizeConnected is equal and I can reach myself under the old IP."); log
.logFine("yacyCore.publishSeedList: not necessary to publish: oldIP is equal, sizeConnected is equal and I can reach myself under the old IP.");
}
return; return;
} }
// getting the seed upload method that should be used ... // getting the seed upload method that should be used ...
final String seedUploadMethod = this.sb.getConfig("seedUploadMethod", ""); final String seedUploadMethod = this.sb.getConfig("seedUploadMethod", "");
if ( if ( (!seedUploadMethod.equalsIgnoreCase("none"))
(!seedUploadMethod.equalsIgnoreCase("none")) || || ((seedUploadMethod.equals("")) && (this.sb.getConfig("seedFTPPassword", "").length() > 0))
((seedUploadMethod.equals("")) && (this.sb.getConfig("seedFTPPassword", "").length() > 0)) || || ((seedUploadMethod.equals("")) && (this.sb.getConfig("seedFilePath", "").length() > 0)) ) {
((seedUploadMethod.equals("")) && (this.sb.getConfig("seedFilePath", "").length() > 0)) if ( seedUploadMethod.equals("") ) {
) { if ( this.sb.getConfig("seedFTPPassword", "").length() > 0 ) {
if (seedUploadMethod.equals("")) {
if (this.sb.getConfig("seedFTPPassword", "").length() > 0) {
this.sb.setConfig("seedUploadMethod", "Ftp"); this.sb.setConfig("seedUploadMethod", "Ftp");
} }
if (this.sb.getConfig("seedFilePath", "").length() > 0) { if ( this.sb.getConfig("seedFilePath", "").length() > 0 ) {
this.sb.setConfig("seedUploadMethod", "File"); this.sb.setConfig("seedUploadMethod", "File");
} }
} }
// we want to be a principal... // we want to be a principal...
saveSeedList(this.sb); saveSeedList(this.sb);
} else { } else {
if (seedUploadMethod.equals("")) { if ( seedUploadMethod.equals("") ) {
this.sb.setConfig("seedUploadMethod", "none"); this.sb.setConfig("seedUploadMethod", "none");
} }
if (log.isFine()) log.logFine("yacyCore.publishSeedList: No uploading method configured"); if ( log.isFine() ) {
log.logFine("yacyCore.publishSeedList: No uploading method configured");
}
return; return;
} }
} }
public final void peerPing() { public final void peerPing() {
if ((this.sb.isRobinsonMode()) && (this.sb.getConfig("cluster.mode", "").equals("privatepeer"))) { if ( (this.sb.isRobinsonMode())
&& (this.sb.getConfig(SwitchboardConstants.CLUSTER_MODE, "").equals(SwitchboardConstants.CLUSTER_MODE_PRIVATE_PEER)) ) {
// in case this peer is a privat peer we omit the peer ping // in case this peer is a privat peer we omit the peer ping
// all other robinson peer types do a peer ping: // all other robinson peer types do a peer ping:
// the privatecluster does the ping to the other cluster members // the privatecluster does the ping to the other cluster members
@ -189,42 +196,54 @@ public class Network {
this.sb.updateMySeed(); this.sb.updateMySeed();
// publish own seed to other peer, this can every peer, but makes only sense for senior peers // publish own seed to other peer, this can every peer, but makes only sense for senior peers
if (this.sb.peers.sizeConnected() == 0) { if ( this.sb.peers.sizeConnected() == 0 ) {
// reload the seed lists // reload the seed lists
this.sb.loadSeedLists(); this.sb.loadSeedLists();
log.logInfo("re-initialized seed list. received " + this.sb.peers.sizeConnected() + " new peer(s)"); log.logInfo("re-initialized seed list. received "
+ this.sb.peers.sizeConnected()
+ " new peer(s)");
} }
final int newSeeds = publishMySeed(false); final int newSeeds = publishMySeed(false);
if (newSeeds > 0) { if ( newSeeds > 0 ) {
log.logInfo("received " + newSeeds + " new peer(s), know a total of " + this.sb.peers.sizeConnected() + " different peers"); log.logInfo("received "
+ newSeeds
+ " new peer(s), know a total of "
+ this.sb.peers.sizeConnected()
+ " different peers");
} }
} }
private boolean canReachMyself() { // TODO: check if this method is necessary - depending on the used router it will not work private boolean canReachMyself() { // TODO: check if this method is necessary - depending on the used router it will not work
// returns true if we can reach ourself under our known peer address // returns true if we can reach ourself under our known peer address
// if we cannot reach ourself, we call a forced publishMySeed and return false // if we cannot reach ourself, we call a forced publishMySeed and return false
final long[] callback = Protocol.queryUrlCount(this.sb.peers.mySeed()); final long[] callback = Protocol.queryUrlCount(this.sb.peers.mySeed());
if (callback[0] >= 0 && callback[1] == magic) { if ( callback[0] >= 0 && callback[1] == magic ) {
this.sb.peers.mySeed().setLastSeenUTC(); this.sb.peers.mySeed().setLastSeenUTC();
return true; return true;
} }
log.logInfo("re-connect own seed"); log.logInfo("re-connect own seed");
final String oldAddress = this.sb.peers.mySeed().getPublicAddress(); final String oldAddress = this.sb.peers.mySeed().getPublicAddress();
/*final int newSeeds =*/ publishMySeed(true); /*final int newSeeds =*/publishMySeed(true);
return (oldAddress != null && oldAddress.equals(this.sb.peers.mySeed().getPublicAddress())); return (oldAddress != null && oldAddress.equals(this.sb.peers.mySeed().getPublicAddress()));
} }
// use our own formatter to prevent concurrency locks with other processes // use our own formatter to prevent concurrency locks with other processes
private final static GenericFormatter my_SHORT_SECOND_FORMATTER = new GenericFormatter(GenericFormatter.FORMAT_SHORT_SECOND, GenericFormatter.time_second); private final static GenericFormatter my_SHORT_SECOND_FORMATTER = new GenericFormatter(
GenericFormatter.FORMAT_SHORT_SECOND,
GenericFormatter.time_second);
protected class publishThread extends Thread { protected class publishThread extends Thread
{
int added; int added;
private final Seed seed; private final Seed seed;
private final Semaphore sync; private final Semaphore sync;
private final List<Thread> syncList; private final List<Thread> syncList;
public publishThread(final ThreadGroup tg, final Seed seed, public publishThread(
final Semaphore sync, final List<Thread> syncList) throws InterruptedException { final ThreadGroup tg,
final Seed seed,
final Semaphore sync,
final List<Thread> syncList) throws InterruptedException {
super(tg, "PublishSeed_" + seed.getName()); super(tg, "PublishSeed_" + seed.getName());
this.sync = sync; this.sync = sync;
@ -235,50 +254,106 @@ public class Network {
this.added = 0; this.added = 0;
} }
@Override
public final void run() { public final void run() {
try { try {
this.added = Protocol.hello(Network.this.sb.peers.mySeed(), Network.this.sb.peers.peerActions, this.seed.getClusterAddress(), this.seed.hash, this.seed.getName()); this.added =
if (this.added < 0) { Protocol.hello(
Network.this.sb.peers.mySeed(),
Network.this.sb.peers.peerActions,
this.seed.getClusterAddress(),
this.seed.hash,
this.seed.getName());
if ( this.added < 0 ) {
// no or wrong response, delete that address // no or wrong response, delete that address
final String cause = "peer ping to peer resulted in error response (added < 0)"; final String cause = "peer ping to peer resulted in error response (added < 0)";
log.logInfo("publish: disconnected " + this.seed.get(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR) + " peer '" + this.seed.getName() + "' from " + this.seed.getPublicAddress() + ": " + cause); log.logInfo("publish: disconnected "
+ this.seed.get(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR)
+ " peer '"
+ this.seed.getName()
+ "' from "
+ this.seed.getPublicAddress()
+ ": "
+ cause);
Network.this.sb.peers.peerActions.peerDeparture(this.seed, cause); Network.this.sb.peers.peerActions.peerDeparture(this.seed, cause);
} else { } else {
// success! we have published our peer to a senior peer // success! we have published our peer to a senior peer
// update latest news from the other peer // update latest news from the other peer
log.logInfo("publish: handshaked " + this.seed.get(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR) + " peer '" + this.seed.getName() + "' at " + this.seed.getPublicAddress()); log.logInfo("publish: handshaked "
+ this.seed.get(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR)
+ " peer '"
+ this.seed.getName()
+ "' at "
+ this.seed.getPublicAddress());
// check if seed's lastSeen has been updated // check if seed's lastSeen has been updated
final Seed newSeed = Network.this.sb.peers.getConnected(this.seed.hash); final Seed newSeed = Network.this.sb.peers.getConnected(this.seed.hash);
if (newSeed != null) { if ( newSeed != null ) {
if (!newSeed.isOnline()) { if ( !newSeed.isOnline() ) {
if (log.isFine()) log.logFine("publish: recently handshaked " + this.seed.get(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR) + if ( log.isFine() ) {
" peer '" + this.seed.getName() + "' at " + this.seed.getPublicAddress() + " is not online." + log.logFine("publish: recently handshaked "
" Removing Peer from connected"); + this.seed.get(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR)
+ " peer '"
+ this.seed.getName()
+ "' at "
+ this.seed.getPublicAddress()
+ " is not online."
+ " Removing Peer from connected");
}
Network.this.sb.peers.peerActions.peerDeparture(newSeed, "peer not online"); Network.this.sb.peers.peerActions.peerDeparture(newSeed, "peer not online");
} else } else if ( newSeed.getLastSeenUTC() < (System.currentTimeMillis() - 10000) ) {
if (newSeed.getLastSeenUTC() < (System.currentTimeMillis() - 10000)) {
// update last seed date // update last seed date
if (newSeed.getLastSeenUTC() >= this.seed.getLastSeenUTC()) { if ( newSeed.getLastSeenUTC() >= this.seed.getLastSeenUTC() ) {
if (log.isFine()) log.logFine("publish: recently handshaked " + this.seed.get(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR) + if ( log.isFine() ) {
" peer '" + this.seed.getName() + "' at " + this.seed.getPublicAddress() + " with old LastSeen: '" + log
my_SHORT_SECOND_FORMATTER.format(new Date(newSeed.getLastSeenUTC())) + "'"); .logFine("publish: recently handshaked "
+ this.seed.get(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR)
+ " peer '"
+ this.seed.getName()
+ "' at "
+ this.seed.getPublicAddress()
+ " with old LastSeen: '"
+ my_SHORT_SECOND_FORMATTER.format(new Date(newSeed
.getLastSeenUTC())) + "'");
}
newSeed.setLastSeenUTC(); newSeed.setLastSeenUTC();
Network.this.sb.peers.peerActions.peerArrival(newSeed, true); Network.this.sb.peers.peerActions.peerArrival(newSeed, true);
} else { } else {
if (log.isFine()) log.logFine("publish: recently handshaked " + this.seed.get(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR) + if ( log.isFine() ) {
" peer '" + this.seed.getName() + "' at " + this.seed.getPublicAddress() + " with old LastSeen: '" + log
my_SHORT_SECOND_FORMATTER.format(new Date(newSeed.getLastSeenUTC())) + "', this is more recent: '" + .logFine("publish: recently handshaked "
my_SHORT_SECOND_FORMATTER.format(new Date(this.seed.getLastSeenUTC())) + "'"); + this.seed.get(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR)
+ " peer '"
+ this.seed.getName()
+ "' at "
+ this.seed.getPublicAddress()
+ " with old LastSeen: '"
+ my_SHORT_SECOND_FORMATTER.format(new Date(newSeed
.getLastSeenUTC()))
+ "', this is more recent: '"
+ my_SHORT_SECOND_FORMATTER.format(new Date(this.seed
.getLastSeenUTC()))
+ "'");
}
this.seed.setLastSeenUTC(); this.seed.setLastSeenUTC();
Network.this.sb.peers.peerActions.peerArrival(this.seed, true); Network.this.sb.peers.peerActions.peerArrival(this.seed, true);
} }
} }
} else { } else {
if (log.isFine()) log.logFine("publish: recently handshaked " + this.seed.get(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR) + " peer '" + this.seed.getName() + "' at " + this.seed.getPublicAddress() + " not in connectedDB"); if ( log.isFine() ) {
log.logFine("publish: recently handshaked "
+ this.seed.get(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR)
+ " peer '"
+ this.seed.getName()
+ "' at "
+ this.seed.getPublicAddress()
+ " not in connectedDB");
}
} }
} }
} catch (final Exception e) { } catch ( final Exception e ) {
log.logSevere("publishThread: error with target seed " + this.seed.toString() + ": " + e.getMessage(), e); log.logSevere(
"publishThread: error with target seed " + this.seed.toString() + ": " + e.getMessage(),
e);
} finally { } finally {
this.syncList.add(this); this.syncList.add(this);
this.sync.release(); this.sync.release();
@ -310,41 +385,54 @@ public class Network {
int attempts = this.sb.peers.sizeConnected(); int attempts = this.sb.peers.sizeConnected();
// getting a list of peers to contact // getting a list of peers to contact
if (this.sb.peers.mySeed().get(Seed.PEERTYPE, Seed.PEERTYPE_VIRGIN).equals(Seed.PEERTYPE_VIRGIN)) { if ( this.sb.peers.mySeed().get(Seed.PEERTYPE, Seed.PEERTYPE_VIRGIN).equals(Seed.PEERTYPE_VIRGIN) ) {
if (attempts > PING_INITIAL) { attempts = PING_INITIAL; } if ( attempts > PING_INITIAL ) {
attempts = PING_INITIAL;
}
final Map<byte[], String> ch = Switchboard.getSwitchboard().clusterhashes; final Map<byte[], String> ch = Switchboard.getSwitchboard().clusterhashes;
seeds = PeerSelection.seedsByAge(this.sb.peers, true, attempts - ((ch == null) ? 0 : ch.size())); // best for fast connection seeds =
PeerSelection.seedsByAge(this.sb.peers, true, attempts - ((ch == null) ? 0 : ch.size())); // best for fast connection
// add also all peers from cluster if this is a public robinson cluster // add also all peers from cluster if this is a public robinson cluster
if (ch != null) { if ( ch != null ) {
final Iterator<Map.Entry<byte[], String>> i = ch.entrySet().iterator(); final Iterator<Map.Entry<byte[], String>> i = ch.entrySet().iterator();
String hash; String hash;
Map.Entry<byte[], String> entry; Map.Entry<byte[], String> entry;
Seed seed; Seed seed;
while (i.hasNext()) { while ( i.hasNext() ) {
entry = i.next(); entry = i.next();
hash = ASCII.String(entry.getKey()); hash = ASCII.String(entry.getKey());
seed = seeds.get(hash); seed = seeds.get(hash);
if (seed == null) { if ( seed == null ) {
seed = this.sb.peers.get(hash); seed = this.sb.peers.get(hash);
if (seed == null) continue; if ( seed == null ) {
continue;
}
} }
seed.setAlternativeAddress(entry.getValue()); seed.setAlternativeAddress(entry.getValue());
seeds.put(hash, seed); seeds.put(hash, seed);
} }
} }
} else { } else {
int diff = PING_MIN_DBSIZE - amIAccessibleDB.size(); int diff = PING_MIN_DBSIZE - amIAccessibleDB.size();
if (diff > PING_MIN_RUNNING) { if ( diff > PING_MIN_RUNNING ) {
diff = Math.min(diff, PING_MAX_RUNNING); diff = Math.min(diff, PING_MAX_RUNNING);
if (attempts > diff) { attempts = diff; } if ( attempts > diff ) {
attempts = diff;
}
} else { } else {
if (attempts > PING_MIN_RUNNING) { attempts = PING_MIN_RUNNING; } if ( attempts > PING_MIN_RUNNING ) {
attempts = PING_MIN_RUNNING;
}
} }
seeds = PeerSelection.seedsByAge(this.sb.peers, false, attempts); // best for seed list maintenance/cleaning seeds = PeerSelection.seedsByAge(this.sb.peers, false, attempts); // best for seed list maintenance/cleaning
} }
if (seeds == null || seeds.isEmpty()) { return 0; } if ( seeds == null || seeds.isEmpty() ) {
if (seeds.size() < attempts) { attempts = seeds.size(); } return 0;
}
if ( seeds.size() < attempts ) {
attempts = seeds.size();
}
// This will try to get Peers that are not currently in amIAccessibleDB // This will try to get Peers that are not currently in amIAccessibleDB
final Iterator<Seed> si = seeds.values().iterator(); final Iterator<Seed> si = seeds.values().iterator();
@ -353,12 +441,12 @@ public class Network {
// include a YaCyNews record to my seed // include a YaCyNews record to my seed
try { try {
final NewsDB.Record record = this.sb.peers.newsPool.myPublication(); final NewsDB.Record record = this.sb.peers.newsPool.myPublication();
if (record == null) { if ( record == null ) {
this.sb.peers.mySeed().put("news", ""); this.sb.peers.mySeed().put("news", "");
} else { } else {
this.sb.peers.mySeed().put("news", de.anomic.tools.crypt.simpleEncode(record.toString())); this.sb.peers.mySeed().put("news", de.anomic.tools.crypt.simpleEncode(record.toString()));
} }
} catch (final Exception e) { } catch ( final Exception e ) {
log.logSevere("publishMySeed: problem with news encoding", e); log.logSevere("publishMySeed: problem with news encoding", e);
} }
this.sb.peers.mySeed().setUnusedFlags(); this.sb.peers.mySeed().setUnusedFlags();
@ -371,20 +459,25 @@ public class Network {
// going through the peer list and starting a new publisher thread for each peer // going through the peer list and starting a new publisher thread for each peer
int i = 0; int i = 0;
while (si.hasNext()) { while ( si.hasNext() ) {
seed = si.next(); seed = si.next();
if (seed == null) { if ( seed == null ) {
sync.acquire(); sync.acquire();
continue; continue;
} }
i++; i++;
final String address = seed.getClusterAddress(); final String address = seed.getClusterAddress();
if (log.isFine()) log.logFine("HELLO #" + i + " to peer '" + seed.get(Seed.NAME, "") + "' at " + address); // debug if ( log.isFine() ) {
log.logFine("HELLO #" + i + " to peer '" + seed.get(Seed.NAME, "") + "' at " + address); // debug
}
final String seederror = seed.isProper(false); final String seederror = seed.isProper(false);
if ((address == null) || (seederror != null)) { if ( (address == null) || (seederror != null) ) {
// we don't like that address, delete it // we don't like that address, delete it
this.sb.peers.peerActions.peerDeparture(seed, "peer ping to peer resulted in address = " + address + "; seederror = " + seederror); this.sb.peers.peerActions.peerDeparture(seed, "peer ping to peer resulted in address = "
+ address
+ "; seederror = "
+ seederror);
sync.acquire(); sync.acquire();
} else { } else {
// starting a new publisher thread // starting a new publisher thread
@ -394,13 +487,13 @@ public class Network {
} }
// receiving the result of all started publisher threads // receiving the result of all started publisher threads
for (int j = 0; j < contactedSeedCount; j++) { for ( int j = 0; j < contactedSeedCount; j++ ) {
// waiting for the next thread to finish // waiting for the next thread to finish
sync.acquire(); sync.acquire();
// if this is true something is wrong ... // if this is true something is wrong ...
if (syncList.isEmpty()) { if ( syncList.isEmpty() ) {
log.logWarning("PeerPing: syncList.isEmpty()==true"); log.logWarning("PeerPing: syncList.isEmpty()==true");
continue; continue;
//return 0; //return 0;
@ -410,9 +503,9 @@ public class Network {
final publishThread t = (publishThread) syncList.remove(0); final publishThread t = (publishThread) syncList.remove(0);
// getting the amount of new reported seeds // getting the amount of new reported seeds
if (t.added >= 0) { if ( t.added >= 0 ) {
if (newSeeds == -1) { if ( newSeeds == -1 ) {
newSeeds = t.added; newSeeds = t.added;
} else { } else {
newSeeds += t.added; newSeeds += t.added;
} }
@ -423,33 +516,41 @@ public class Network {
int notaccessible = 0; int notaccessible = 0;
final long cutofftime = System.currentTimeMillis() - PING_MAX_DBAGE; final long cutofftime = System.currentTimeMillis() - PING_MAX_DBAGE;
final int dbSize; final int dbSize;
synchronized (amIAccessibleDB) { synchronized ( amIAccessibleDB ) {
dbSize = amIAccessibleDB.size(); dbSize = amIAccessibleDB.size();
final Iterator<String> ai = amIAccessibleDB.keySet().iterator(); final Iterator<String> ai = amIAccessibleDB.keySet().iterator();
while (ai.hasNext()) { while ( ai.hasNext() ) {
final Accessible ya = amIAccessibleDB.get(ai.next()); final Accessible ya = amIAccessibleDB.get(ai.next());
if (ya.lastUpdated < cutofftime) { if ( ya.lastUpdated < cutofftime ) {
ai.remove(); ai.remove();
} else { } else {
if (ya.IWasAccessed) { if ( ya.IWasAccessed ) {
accessible++; accessible++;
} else { } else {
notaccessible++; notaccessible++;
} }
} }
} }
if (log.isFine()) log.logFine("DBSize before -> after Cleanup: " + dbSize + " -> " + amIAccessibleDB.size()); if ( log.isFine() ) {
log
.logFine("DBSize before -> after Cleanup: "
+ dbSize
+ " -> "
+ amIAccessibleDB.size());
}
} }
log.logInfo("PeerPing: I am accessible for " + accessible + log.logInfo("PeerPing: I am accessible for "
" peer(s), not accessible for " + notaccessible + " peer(s)."); + accessible
+ " peer(s), not accessible for "
+ notaccessible
+ " peer(s).");
if ((accessible + notaccessible) > 0) { if ( (accessible + notaccessible) > 0 ) {
final String newPeerType; final String newPeerType;
// At least one other Peer told us our type // At least one other Peer told us our type
if ((accessible >= PING_MIN_PEERSEEN) || if ( (accessible >= PING_MIN_PEERSEEN) || (accessible >= notaccessible) ) {
(accessible >= notaccessible)) {
// We can be reached from a majority of other Peers // We can be reached from a majority of other Peers
if (this.sb.peers.mySeed().isPrincipal()) { if ( this.sb.peers.mySeed().isPrincipal() ) {
newPeerType = Seed.PEERTYPE_PRINCIPAL; newPeerType = Seed.PEERTYPE_PRINCIPAL;
} else { } else {
newPeerType = Seed.PEERTYPE_SENIOR; newPeerType = Seed.PEERTYPE_SENIOR;
@ -458,10 +559,14 @@ public class Network {
// We cannot be reached from the outside // We cannot be reached from the outside
newPeerType = Seed.PEERTYPE_JUNIOR; newPeerType = Seed.PEERTYPE_JUNIOR;
} }
if (this.sb.peers.mySeed().orVirgin().equals(newPeerType)) { if ( this.sb.peers.mySeed().orVirgin().equals(newPeerType) ) {
log.logInfo("PeerPing: myType is " + this.sb.peers.mySeed().orVirgin()); log.logInfo("PeerPing: myType is " + this.sb.peers.mySeed().orVirgin());
} else { } else {
log.logInfo("PeerPing: changing myType from '" + this.sb.peers.mySeed().orVirgin() + "' to '" + newPeerType + "'"); log.logInfo("PeerPing: changing myType from '"
+ this.sb.peers.mySeed().orVirgin()
+ "' to '"
+ newPeerType
+ "'");
this.sb.peers.mySeed().put(Seed.PEERTYPE, newPeerType); this.sb.peers.mySeed().put(Seed.PEERTYPE, newPeerType);
} }
} else { } else {
@ -474,24 +579,33 @@ public class Network {
this.sb.peers.saveMySeed(); this.sb.peers.saveMySeed();
// if we have an address, we do nothing // if we have an address, we do nothing
if (this.sb.peers.mySeed().isProper(true) == null && !force) { return 0; } if ( this.sb.peers.mySeed().isProper(true) == null && !force ) {
if (newSeeds > 0) return newSeeds; return 0;
}
if ( newSeeds > 0 ) {
return newSeeds;
}
// still no success: ask own NAT or internet responder // still no success: ask own NAT or internet responder
//final boolean DI604use = switchboard.getConfig("DI604use", "false").equals("true"); //final boolean DI604use = switchboard.getConfig("DI604use", "false").equals("true");
//final String DI604pw = switchboard.getConfig("DI604pw", ""); //final String DI604pw = switchboard.getConfig("DI604pw", "");
final String ip = this.sb.getConfig("staticIP", ""); final String ip = this.sb.getConfig("staticIP", "");
//if (ip.equals("")) ip = natLib.retrieveIP(DI604use, DI604pw); //if (ip.equals("")) ip = natLib.retrieveIP(DI604use, DI604pw);
// yacyCore.log.logDebug("DEBUG: new IP=" + ip); // yacyCore.log.logDebug("DEBUG: new IP=" + ip);
if (Seed.isProperIP(ip) == null) this.sb.peers.mySeed().setIP(ip); if ( Seed.isProperIP(ip) == null ) {
if (this.sb.peers.mySeed().get(Seed.PEERTYPE, Seed.PEERTYPE_JUNIOR).equals(Seed.PEERTYPE_JUNIOR)) // ??????????????? this.sb.peers.mySeed().setIP(ip);
}
if ( this.sb.peers.mySeed().get(Seed.PEERTYPE, Seed.PEERTYPE_JUNIOR).equals(Seed.PEERTYPE_JUNIOR) ) {
this.sb.peers.mySeed().put(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR); // to start bootstraping, we need to be recognised as PEERTYPE_SENIOR peer this.sb.peers.mySeed().put(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR); // to start bootstraping, we need to be recognised as PEERTYPE_SENIOR peer
log.logInfo("publish: no recipient found, our address is " + }
((this.sb.peers.mySeed().getPublicAddress() == null) ? "unknown" : this.sb.peers.mySeed().getPublicAddress())); log.logInfo("publish: no recipient found, our address is "
+ ((this.sb.peers.mySeed().getPublicAddress() == null) ? "unknown" : this.sb.peers
.mySeed()
.getPublicAddress()));
this.sb.peers.saveMySeed(); this.sb.peers.saveMySeed();
return 0; return 0;
} catch (final InterruptedException e) { } catch ( final InterruptedException e ) {
try { try {
log.logInfo("publish: Interruption detected while publishing my seed."); log.logInfo("publish: Interruption detected while publishing my seed.");
@ -499,32 +613,50 @@ public class Network {
Thread.interrupted(); Thread.interrupted();
// interrupt all already started publishThreads // interrupt all already started publishThreads
log.logInfo("publish: Signaling shutdown to " + Network.publishThreadGroup.activeCount() + " remaining publishing threads ..."); log.logInfo("publish: Signaling shutdown to "
+ Network.publishThreadGroup.activeCount()
+ " remaining publishing threads ...");
Network.publishThreadGroup.interrupt(); Network.publishThreadGroup.interrupt();
// waiting some time for the publishThreads to finish execution // waiting some time for the publishThreads to finish execution
try { Thread.sleep(500); } catch (final InterruptedException ex) {} try {
Thread.sleep(500);
} catch ( final InterruptedException ex ) {
}
// getting the amount of remaining publishing threads // getting the amount of remaining publishing threads
int threadCount = Network.publishThreadGroup.activeCount(); int threadCount = Network.publishThreadGroup.activeCount();
final Thread[] threadList = new Thread[threadCount]; final Thread[] threadList = new Thread[threadCount];
threadCount = Network.publishThreadGroup.enumerate(threadList); threadCount = Network.publishThreadGroup.enumerate(threadList);
// we need to use a timeout here because of missing interruptable session threads ... // we need to use a timeout here because of missing interruptable session threads ...
if (log.isFine()) log.logFine("publish: Waiting for " + Network.publishThreadGroup.activeCount() + " remaining publishing threads to finish shutdown ..."); if ( log.isFine() ) {
for (int currentThreadIdx = 0; currentThreadIdx < threadCount; currentThreadIdx++) { log.logFine("publish: Waiting for "
+ Network.publishThreadGroup.activeCount()
+ " remaining publishing threads to finish shutdown ...");
}
for ( int currentThreadIdx = 0; currentThreadIdx < threadCount; currentThreadIdx++ ) {
final Thread currentThread = threadList[currentThreadIdx]; final Thread currentThread = threadList[currentThreadIdx];
if (currentThread.isAlive()) { if ( currentThread.isAlive() ) {
if (log.isFine()) log.logFine("publish: Waiting for remaining publishing thread '" + currentThread.getName() + "' to finish shutdown"); if ( log.isFine() ) {
try { currentThread.join(500); } catch (final InterruptedException ex) {} log.logFine("publish: Waiting for remaining publishing thread '"
+ currentThread.getName()
+ "' to finish shutdown");
}
try {
currentThread.join(500);
} catch ( final InterruptedException ex ) {
}
} }
} }
log.logInfo("publish: Shutdown off all remaining publishing thread finished."); log.logInfo("publish: Shutdown off all remaining publishing thread finished.");
} catch (final Exception ee) { } catch ( final Exception ee ) {
log.logWarning("publish: Unexpected error while trying to shutdown all remaining publishing threads.", e); log.logWarning(
"publish: Unexpected error while trying to shutdown all remaining publishing threads.",
e);
} }
return 0; return 0;
@ -533,25 +665,27 @@ public class Network {
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
public static HashMap<String, String> getSeedUploadMethods() { public static HashMap<String, String> getSeedUploadMethods() {
synchronized (Network.seedUploadMethods) { synchronized ( Network.seedUploadMethods ) {
return (HashMap<String, String>) Network.seedUploadMethods.clone(); return (HashMap<String, String>) Network.seedUploadMethods.clone();
} }
} }
public static yacySeedUploader getSeedUploader(final String methodname) { public static yacySeedUploader getSeedUploader(final String methodname) {
String className = null; String className = null;
synchronized (Network.seedUploadMethods) { synchronized ( Network.seedUploadMethods ) {
if (Network.seedUploadMethods.containsKey(methodname)) { if ( Network.seedUploadMethods.containsKey(methodname) ) {
className = Network.seedUploadMethods.get(methodname); className = Network.seedUploadMethods.get(methodname);
} }
} }
if (className == null) { return null; } if ( className == null ) {
return null;
}
try { try {
final Class<?> uploaderClass = Class.forName(className); final Class<?> uploaderClass = Class.forName(className);
final Object uploader = uploaderClass.newInstance(); final Object uploader = uploaderClass.newInstance();
return (yacySeedUploader) uploader; return (yacySeedUploader) uploader;
} catch (final Exception e) { } catch ( final Exception e ) {
return null; return null;
} }
} }
@ -559,19 +693,32 @@ public class Network {
public static void loadSeedUploadMethods() { public static void loadSeedUploadMethods() {
yacySeedUploader uploader; yacySeedUploader uploader;
uploader = new yacySeedUploadFile(); uploader = new yacySeedUploadFile();
Network.seedUploadMethods.put(uploader.getClass().getSimpleName().substring("yacySeedUpload".length()), uploader.getClass().getCanonicalName()); Network.seedUploadMethods.put(uploader
.getClass()
.getSimpleName()
.substring("yacySeedUpload".length()), uploader.getClass().getCanonicalName());
uploader = new yacySeedUploadFtp(); uploader = new yacySeedUploadFtp();
Network.seedUploadMethods.put(uploader.getClass().getSimpleName().substring("yacySeedUpload".length()), uploader.getClass().getCanonicalName()); Network.seedUploadMethods.put(uploader
.getClass()
.getSimpleName()
.substring("yacySeedUpload".length()), uploader.getClass().getCanonicalName());
uploader = new yacySeedUploadScp(); uploader = new yacySeedUploadScp();
Network.seedUploadMethods.put(uploader.getClass().getSimpleName().substring("yacySeedUpload".length()), uploader.getClass().getCanonicalName()); Network.seedUploadMethods.put(uploader
.getClass()
.getSimpleName()
.substring("yacySeedUpload".length()), uploader.getClass().getCanonicalName());
} }
public static boolean changeSeedUploadMethod(final String method) { public static boolean changeSeedUploadMethod(final String method) {
if (method == null || method.length() == 0) { return false; } if ( method == null || method.length() == 0 ) {
return false;
}
if (method.equalsIgnoreCase("none")) { return true; } if ( method.equalsIgnoreCase("none") ) {
return true;
}
synchronized (Network.seedUploadMethods) { synchronized ( Network.seedUploadMethods ) {
return Network.seedUploadMethods.containsKey(method); return Network.seedUploadMethods.containsKey(method);
} }
} }
@ -582,7 +729,7 @@ public class Network {
String logt; String logt;
// be shure that we have something to say // be shure that we have something to say
if (sb.peers.mySeed().getPublicAddress() == null) { if ( sb.peers.mySeed().getPublicAddress() == null ) {
final String errorMsg = "We have no valid IP address until now"; final String errorMsg = "We have no valid IP address until now";
log.logWarning("SaveSeedList: " + errorMsg); log.logWarning("SaveSeedList: " + errorMsg);
return errorMsg; return errorMsg;
@ -592,27 +739,30 @@ public class Network {
String seedUploadMethod = sb.getConfig("seedUploadMethod", ""); String seedUploadMethod = sb.getConfig("seedUploadMethod", "");
// for backward compatiblity .... // for backward compatiblity ....
if (seedUploadMethod.equalsIgnoreCase("Ftp") || if ( seedUploadMethod.equalsIgnoreCase("Ftp")
(seedUploadMethod.equals("") && || (seedUploadMethod.equals("") && sb.getConfig("seedFTPPassword", "").length() > 0) ) {
sb.getConfig("seedFTPPassword", "").length() > 0)) {
seedUploadMethod = "Ftp"; seedUploadMethod = "Ftp";
sb.setConfig("seedUploadMethod", seedUploadMethod); sb.setConfig("seedUploadMethod", seedUploadMethod);
} else if (seedUploadMethod.equalsIgnoreCase("File") || } else if ( seedUploadMethod.equalsIgnoreCase("File")
(seedUploadMethod.equals("") && || (seedUploadMethod.equals("") && sb.getConfig("seedFilePath", "").length() > 0) ) {
sb.getConfig("seedFilePath", "").length() > 0)) {
seedUploadMethod = "File"; seedUploadMethod = "File";
sb.setConfig("seedUploadMethod", seedUploadMethod); sb.setConfig("seedUploadMethod", seedUploadMethod);
} }
// determine the seed uploader that should be used ... // determine the seed uploader that should be used ...
if (seedUploadMethod.equalsIgnoreCase("none")) { return "no uploader specified"; } if ( seedUploadMethod.equalsIgnoreCase("none") ) {
return "no uploader specified";
}
final yacySeedUploader uploader = getSeedUploader(seedUploadMethod); final yacySeedUploader uploader = getSeedUploader(seedUploadMethod);
if (uploader == null) { if ( uploader == null ) {
final String errorMsg = "Unable to get the proper uploader-class for seed uploading method '" + seedUploadMethod + "'."; final String errorMsg =
"Unable to get the proper uploader-class for seed uploading method '"
+ seedUploadMethod
+ "'.";
log.logWarning("SaveSeedList: " + errorMsg); log.logWarning("SaveSeedList: " + errorMsg);
return errorMsg; return errorMsg;
} }
@ -621,35 +771,51 @@ public class Network {
DigestURI seedURL; DigestURI seedURL;
try { try {
final String seedURLStr = sb.peers.mySeed().get(Seed.SEEDLISTURL, ""); final String seedURLStr = sb.peers.mySeed().get(Seed.SEEDLISTURL, "");
if (seedURLStr.length() == 0) { throw new MalformedURLException("The seed-file url must not be empty."); } if ( seedURLStr.length() == 0 ) {
if (!( throw new MalformedURLException("The seed-file url must not be empty.");
seedURLStr.toLowerCase().startsWith("http://") || }
seedURLStr.toLowerCase().startsWith("https://") if ( !(seedURLStr.toLowerCase().startsWith("http://") || seedURLStr.toLowerCase().startsWith(
)) { "https://")) ) {
throw new MalformedURLException("Unsupported protocol."); throw new MalformedURLException("Unsupported protocol.");
} }
seedURL = new DigestURI(seedURLStr); seedURL = new DigestURI(seedURLStr);
} catch (final MalformedURLException e) { } catch ( final MalformedURLException e ) {
final String errorMsg = "Malformed seed file URL '" + sb.peers.mySeed().get(Seed.SEEDLISTURL, "") + "'. " + e.getMessage(); final String errorMsg =
"Malformed seed file URL '"
+ sb.peers.mySeed().get(Seed.SEEDLISTURL, "")
+ "'. "
+ e.getMessage();
log.logWarning("SaveSeedList: " + errorMsg); log.logWarning("SaveSeedList: " + errorMsg);
return errorMsg; return errorMsg;
} }
// upload the seed-list using the configured uploader class // upload the seed-list using the configured uploader class
String prevStatus = sb.peers.mySeed().get(Seed.PEERTYPE, Seed.PEERTYPE_JUNIOR); String prevStatus = sb.peers.mySeed().get(Seed.PEERTYPE, Seed.PEERTYPE_JUNIOR);
if (prevStatus.equals(Seed.PEERTYPE_PRINCIPAL)) { prevStatus = Seed.PEERTYPE_SENIOR; } if ( prevStatus.equals(Seed.PEERTYPE_PRINCIPAL) ) {
prevStatus = Seed.PEERTYPE_SENIOR;
}
try { try {
sb.peers.mySeed().put(Seed.PEERTYPE, Seed.PEERTYPE_PRINCIPAL); // this information shall also be uploaded sb.peers.mySeed().put(Seed.PEERTYPE, Seed.PEERTYPE_PRINCIPAL); // this information shall also be uploaded
if (log.isFine()) log.logFine("SaveSeedList: Using seed uploading method '" + seedUploadMethod + "' for seed-list uploading." + if ( log.isFine() ) {
"\n\tPrevious peerType is '" + sb.peers.mySeed().get(Seed.PEERTYPE, Seed.PEERTYPE_JUNIOR) + "'."); log.logFine("SaveSeedList: Using seed uploading method '"
+ seedUploadMethod
+ "' for seed-list uploading."
+ "\n\tPrevious peerType is '"
+ sb.peers.mySeed().get(Seed.PEERTYPE, Seed.PEERTYPE_JUNIOR)
+ "'.");
}
logt = sb.peers.uploadSeedList(uploader, sb, sb.peers, seedURL); logt = sb.peers.uploadSeedList(uploader, sb, sb.peers, seedURL);
if (logt != null) { if ( logt != null ) {
if (logt.indexOf("Error",0) >= 0) { if ( logt.indexOf("Error", 0) >= 0 ) {
sb.peers.mySeed().put(Seed.PEERTYPE, prevStatus); sb.peers.mySeed().put(Seed.PEERTYPE, prevStatus);
final String errorMsg = "SaveSeedList: seed upload failed using " + uploader.getClass().getName() + " (error): " + logt.substring(logt.indexOf("Error",0) + 6); final String errorMsg =
"SaveSeedList: seed upload failed using "
+ uploader.getClass().getName()
+ " (error): "
+ logt.substring(logt.indexOf("Error", 0) + 6);
log.logSevere(errorMsg); log.logSevere(errorMsg);
return errorMsg; return errorMsg;
} }
@ -659,7 +825,7 @@ public class Network {
// finally, set the principal status // finally, set the principal status
sb.setConfig("yacyStatus", Seed.PEERTYPE_PRINCIPAL); sb.setConfig("yacyStatus", Seed.PEERTYPE_PRINCIPAL);
return null; return null;
} catch (final Exception e) { } catch ( final Exception e ) {
sb.peers.mySeed().put(Seed.PEERTYPE, prevStatus); sb.peers.mySeed().put(Seed.PEERTYPE, prevStatus);
sb.setConfig("yacyStatus", prevStatus); sb.setConfig("yacyStatus", prevStatus);
final String errorMsg = "SaveSeedList: Seed upload failed (IO error): " + e.getMessage(); final String errorMsg = "SaveSeedList: Seed upload failed (IO error): " + e.getMessage();

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

@ -278,8 +278,8 @@ public final class SwitchboardConstants {
public static final String CLUSTER_MODE = "cluster.mode"; public static final String CLUSTER_MODE = "cluster.mode";
public static final String CLUSTER_MODE_PUBLIC_CLUSTER = "publiccluster"; public static final String CLUSTER_MODE_PUBLIC_CLUSTER = "publiccluster";
public static final String CLUSTER_MODE_PRIVATE_CLUSTER = "privatecluster";
public static final String CLUSTER_MODE_PUBLIC_PEER = "publicpeer"; public static final String CLUSTER_MODE_PUBLIC_PEER = "publicpeer";
public static final String CLUSTER_MODE_PRIVATE_PEER = "privatepeer";
public static final String CLUSTER_PEERS_IPPORT = "cluster.peers.ipport"; public static final String CLUSTER_PEERS_IPPORT = "cluster.peers.ipport";
public static final String DHT_BURST_ROBINSON = "network.unit.dht.burst.robinson"; public static final String DHT_BURST_ROBINSON = "network.unit.dht.burst.robinson";

Loading…
Cancel
Save