Merge branch 'master' of git@gitorious.org:yacy/rc1.git

pull/1/head
orbiter 11 years ago
commit 856da2712b

@ -1262,4 +1262,5 @@ crawler.userAgent.clienttimeout = 10000
# interface decorations
decoration.audio = false
decoration.grafics.linkstructure = true
decoration.hostanalysis = false

@ -115,6 +115,25 @@ function updatepage(str) {
</fieldset>
#(/hosts)#
#(hostanalysis)#::
<fieldset><legend>Host Analysis</legend>
#{facets}#
<table class="sortable" border="0" style="float:left">
<tr class="TableCellDark">
<td align="left" colspan="5" nowrap class="pending">#[facetname]#</td>
<td align="right" colspan="5" nowrap class="listingok">#</td>
</tr>
#{facet}#
<tr class="TableCellLight">
<td align="left" colspan="5" nowrap class="pending">#[key]#</td>
<td align="right" colspan="5" nowrap class="listingok"><a href="#[a]#" target="_blank" class="forceNoExternalIcon">#[count]#</a></td>
</tr>
#{/facet}#
</table>&nbsp;&nbsp;
#{/facets}#
</fieldset>
#(/hostanalysis)#
#(files)#::
<fieldset><legend>Browser for <a href="#[path]#" target="_blank">#[path]#</a></legend>
<p>documents stored for host: #[hostsize]#; documents stored for subpath: #[subpathloadsize]#; unloaded documents detected in subpath: #[subpathdetectedsize]# <!-- #(complete)#;<a href="HostBrowser.html?admin=#[admin]#&complete=true&path=#[path]#">get complete list</a>::<a href="HostBrowser.html?admin=#[admin]#&path=#[path]#">directory view</a>#(/complete)#-->

@ -29,6 +29,7 @@ import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.TreeMap;
import java.util.concurrent.BlockingQueue;
@ -41,6 +42,7 @@ import net.yacy.cora.document.encoding.UTF8;
import net.yacy.cora.document.id.DigestURL;
import net.yacy.cora.document.id.MultiProtocolURL;
import net.yacy.cora.federate.solr.FailType;
import net.yacy.cora.federate.solr.SolrType;
import net.yacy.cora.federate.solr.connector.AbstractSolrConnector;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.cora.sorting.ClusteredScoreMap;
@ -89,6 +91,7 @@ public class HostBrowser {
prop.put("result", "");
prop.put("hosts", 0);
prop.put("files", 0);
prop.put("hostanalysis", 0);
prop.put("admin", "false");
boolean admin = false;
@ -248,37 +251,71 @@ public class HostBrowser {
}
if (path.length() > 0) {
boolean delete = false;
boolean reload404 = false;
if (authorized && post.containsKey("delete")) {
// delete the complete path!! That includes everything that matches with this prefix.
delete = true;
}
if (authorized && post.containsKey("reload404")) {
// try to re-load all urls that have load errors and matches with this prefix.
reload404 = true;
}
int facetcount=post.getInt("facetcount", 0);
boolean complete = post.getBoolean("complete");
if (complete) { // we want only root paths for complete lists
p = path.indexOf('/', 10);
if (p > 0) path = path.substring(0, p + 1);
}
prop.put("files_complete", complete ? 1 : 0);
prop.put("files_complete_admin", admin ? "true" : "false");
prop.putHTML("files_complete_path", path);
p = path.substring(0, path.length() - 1).lastIndexOf('/');
if (p < 8) {
prop.put("files_root", 1);
} else {
prop.put("files_root", 0);
prop.putHTML("files_root_path", path.substring(0, p + 1));
prop.put("files_root_admin", admin ? "true" : "false");
}
try {
// generate file list from path
DigestURL uri = new DigestURL(path);
String host = uri.getHost();
// write host analysis if path after host is empty
if (uri.getPath().length() <= 1 && host != null && host.length() > 0 && sb.getConfigBool("decoration.hostanalysis", false)) {
//how many documents per crawldepth_i; get crawldepth_i facet for host
ArrayList<String> ff = new ArrayList<>();
for (CollectionSchema csf: CollectionSchema.values()) {
if (csf.getType() != SolrType.num_integer || csf.isMultiValued()) continue;
String facetfield = csf.getSolrFieldName();
if (!fulltext.getDefaultConfiguration().contains(facetfield)) continue;
ff.add(csf.getSolrFieldName());
}
String[] facetfields = ff.toArray(new String[ff.size()]);
Map<String, ReversibleScoreMap<String>> facets = fulltext.getDefaultConnector().getFacets(CollectionSchema.host_s.getSolrFieldName() + ":\"" + host + "\"", 100, facetfields);
int fc = 0;
for (String facetfield: facetfields) {
prop.put("hostanalysis_facets_" + fc + "_facetname", facetfield);
ReversibleScoreMap<String> facetfieldmap = facets.get(facetfield);
TreeMap<Integer, Integer> statMap = new TreeMap<>();
for (String k: facetfieldmap) statMap.put(Integer.parseInt(k), facetfieldmap.get(k));
int c = 0; for (Entry<Integer, Integer> entry: statMap.entrySet()) {
prop.put("hostanalysis_facets_" + fc + "_facet_" + c + "_key", entry.getKey());
prop.put("hostanalysis_facets_" + fc + "_facet_" + c + "_count", entry.getValue());
prop.put("hostanalysis_facets_" + fc + "_facet_" + c + "_a", "http://localhost:" + sb.getConfigInt("port", 8090) + "/solr/collection1/select?q=host_s:" + host + " AND " + facetfield + ":" + entry.getKey() + "&defType=edismax&start=0&rows=1000&fl=sku,crawldepth_i");
c++;
}
prop.put("hostanalysis_facets_" + fc + "_facet", c);
fc++;
}
prop.put("hostanalysis_facets", fc);
prop.put("hostanalysis", 1);
}
// write file list for subpath
boolean delete = false;
boolean reload404 = false;
if (authorized && post.containsKey("delete")) {
// delete the complete path!! That includes everything that matches with this prefix.
delete = true;
}
if (authorized && post.containsKey("reload404")) {
// try to re-load all urls that have load errors and matches with this prefix.
reload404 = true;
}
int facetcount=post.getInt("facetcount", 0);
boolean complete = post.getBoolean("complete");
if (complete) { // we want only root paths for complete lists
p = path.indexOf('/', 10);
if (p > 0) path = path.substring(0, p + 1);
}
prop.put("files_complete", complete ? 1 : 0);
prop.put("files_complete_admin", admin ? "true" : "false");
prop.putHTML("files_complete_path", path);
p = path.substring(0, path.length() - 1).lastIndexOf('/');
if (p < 8) {
prop.put("files_root", 1);
} else {
prop.put("files_root", 0);
prop.putHTML("files_root_path", path.substring(0, p + 1));
prop.put("files_root_admin", admin ? "true" : "false");
}
// generate file list from path
prop.putHTML("outbound_host", host);
if (authorized) prop.putHTML("outbound_admin_host", host); //used for WebStructurePicture_p link
prop.putHTML("inbound_host", host);

@ -62,6 +62,7 @@
<li><a href="Network.html?page=1&maxCount=1000" class="MenuItemLink">Active&nbsp;Principal&nbsp;and&nbsp;Senior&nbsp;Peers</a></li>
<li><a href="Network.html?page=2&maxCount=1000" class="MenuItemLink">Passive&nbsp;Senior&nbsp;Peers</a></li>
<li><a href="Network.html?page=3&maxCount=1000" class="MenuItemLink">Junior&nbsp;(fragment)&nbsp;Peers</a></li>
<li><a href="Network.html?page=5" class="MenuItemLink">Network History</a></li>
</ul>
</div>
#(/menu)#
@ -85,6 +86,8 @@ document.getElementById("apilink").setAttribute("href", "Network.xml?" + window.
<h2>Junior Peers (a fragment) in '#[networkName]#' Network</h2>
::
<h2>Manually contacting Peer</h2>
::
<h2>Network History</h2>
#(/page)#
#(regexerror)#
::
@ -319,8 +322,8 @@ document.getElementById("apilink").setAttribute("href", "Network.xml?" + window.
</tr>
</table>
</div>
::
::
::
#(comment)#
::
<p style="color:red;">
@ -369,6 +372,29 @@ document.getElementById("apilink").setAttribute("href", "Network.xml?" + window.
<td><input type="button" value="contact current peer from this peer" onclick="top.location.href='http://' + document.getElementById('otherPeerAddress').value + '/Network.html?peerHash=#[my-hash]#&peerIP=#[my-ip]#&peerPort=#[my-port]#&page=4&addPeer=add+Peer'"/></td>
</tr>
</table>
::
<!--
possible column values:
aM activeLastMonth
aW activeLastWeek
aD activeLastDay
aH activeLastHour
cC countConnected (Active Senior)
cD countDisconnected (Passive Senior)
cP countPotential (Junior)
cR count of the RWI entries
cI size of the index (number of documents)
maxtime: number of hours in the past
scale: number of hours per scale unit in the bottom line
-->
<h3><b>Count of Connected Senior Peers</b> in the last two days, scale = 1h</h3>
<img src="/NetworkHistory.png?columns=cC&scale=1&maxtime=24" />
<h3><b>Count of all Active Peers Per Day</b> in the last week, scale = 1d</h3>
<img src="/NetworkHistory.png?columns=aD&scale=24&maxtime=168" />
<h3><b>Count of all Active Peers Per Week</b> in the last 30d, scale = 7d</h3>
<img src="/NetworkHistory.png?columns=aW&scale=168&maxtime=720" />
<h3><b>Count of all Active Peers Per Month</b> in the last 365d, scale = 30d</h3>
<img src="/NetworkHistory.png?columns=aM&scale=720&maxtime=8760" />
#(/table)#
#%env/templates/footer.template%#
</body>

@ -199,7 +199,7 @@ public class Network {
prop.put("table", 2); // triggers overview
prop.put("page", 0);
} else if (post != null && post.getInt("page", 1) == 4) {
prop.put("table", 4); // triggers overview
prop.put("table", 4); // triggers "Manually contacting Peer"
prop.put("page", 4);
if (sb.peers.mySeed() != null) {
@ -258,6 +258,9 @@ public class Network {
prop.put("table_comment",0);
}
} else if (post != null && post.getInt("page", 1) == 5) {
prop.put("page", 5); // triggers stats page
prop.put("table", 5);
} else {
// generate table
final int page = (post == null ? 1 : post.getInt("page", 1));

@ -24,8 +24,10 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.Date;
import java.util.HashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import net.yacy.cora.date.GenericFormatter;
import net.yacy.cora.document.encoding.ASCII;
@ -49,7 +51,7 @@ public class NetworkHistory {
final int maxtime = post.getInt("maxtime", 48); // hours
final int bottomscale = post.getInt("scale", 1); // 1h
final String[] columns = post.get("columns", "cC").split("\\|"); // new String[]{"aM", "aW", "aD", "aH", "cC", "cD", "cP", "cR", "cI"};
final String[] columnsx = post.get("columns", "cC").split("\\|"); // new String[]{"aM", "aW", "aD", "aH", "cC", "cD", "cP", "cR", "cI"};
/*
aM activeLastMonth
aW activeLastWeek
@ -61,11 +63,12 @@ public class NetworkHistory {
cR count of the RWI entries
cI size of the index (number of documents)
*/
final Set<String> columns = new LinkedHashSet<>();
for (String col: columnsx) columns.add(col);
// scan the database and put in values
List<Map<String, Long>> rows = new ArrayList<>(maxtime * 2);
long now = System.currentTimeMillis();
long timelimit = now - maxtime * 60 * 60 * 1000;
long timelimit = now - maxtime * 3600000L;
try {
// BEncodedHeap statTable = sb.tables.getHeap("stats");
// Iterator<byte[]> i = statTable.keys(false, false);
@ -97,7 +100,7 @@ public class NetworkHistory {
if (v != null) maxpeers = Math.max(maxpeers, (int) v.longValue());
}
}
final int leftborder = 40;
final int leftborder = 30;
final int rightborder = 10;
final int width = post.getInt("width", 768 + leftborder + rightborder);
final int hspace = width - leftborder - rightborder;
@ -105,10 +108,24 @@ public class NetworkHistory {
final int topborder = 20;
final int bottomborder = 20;
final int vspace = height - topborder - bottomborder;
final int leftscale = (maxpeers / 100) * 10;
ChartPlotter chart = new ChartPlotter(width, height, 0xFFFFFFl, 0x000000l, 0xAAAAAAl, leftborder, rightborder, topborder, bottomborder, "YACY NETWORK HISTORY", "IN THE LAST 48 HOURS");
final int leftscale = maxpeers / 10;
String timestr = maxtime + " HOURS";
if (maxtime > 24 && maxtime % 24 == 0) timestr = (maxtime / 24) + " DAYS";
if (maxtime == 168) timestr = "WEEK";
if (maxtime > 168 && maxtime % 168 == 0) timestr = (maxtime / 168) + " WEEKS";
String headline = "YACY NETWORK HISTORY";
if (columns.contains("aM")) headline += ", ACTIVE PEERS WITHIN THE LAST MONTH";
if (columns.contains("aW")) headline += ", ACTIVE PEERS WITHIN THE LAST WEEK";
if (columns.contains("aD")) headline += ", ACTIVE PEERS WITHIN THE LAST DAY";
if (columns.contains("aH")) headline += ", ACTIVE PEERS WITHIN THE LAST HOUR";
if (columns.contains("cC")) headline += ", ACTIVE SENIOR PEERS";
if (columns.contains("cD")) headline += ", PASSIVE SENIOR PEERS";
if (columns.contains("cP")) headline += ", POTENTIAL JUNIOR PEERS";
if (columns.contains("cI")) headline = "YACY INDEX SIZE HISTORY: NUMBER OF DOCUMENTS";
if (columns.contains("cR")) headline = "YACY INDEX SIZE HISTORY: NUMBER OF RWI ENTRIES";
ChartPlotter chart = new ChartPlotter(width, height, 0xFFFFFFl, 0x000000l, 0xAAAAAAl, leftborder, rightborder, topborder, bottomborder, headline, "IN THE LAST " + timestr);
chart.declareDimension(ChartPlotter.DIMENSION_BOTTOM, bottomscale, hspace / (maxtime / bottomscale), -maxtime, 0x000000l, 0xCCCCCCl, "TIME/HOURS");
chart.declareDimension(ChartPlotter.DIMENSION_LEFT, leftscale, vspace * leftscale / maxpeers, 0, 0x008800l, null , "PEERS");
chart.declareDimension(ChartPlotter.DIMENSION_LEFT, leftscale, vspace * leftscale / maxpeers, 0, 0x008800l, null , columns.contains("cI") ? "DOCUMENTS" : columns.contains("cR") ? "RWIs" : "PEERS");
// write the data
float x0, x1;

@ -52,6 +52,7 @@
<iframe src="rssTerminal.html?set=PEERNEWS,REMOTESEARCH,LOCALSEARCH,REMOTEINDEXING,LOCALINDEXING,DHTRECEIVE,DHTSEND,PROXY&amp;width=600px&amp;height=180px&amp;maxlines=20&amp;maxwidth=120"
style="width:600px;height:180px;margin:0px;border:1px solid black;" scrolling="no" name="newsframe"></iframe><br />
<img src="PerformanceGraph.png?nomem=" id="graph" alt="PerformanceGraph" width="660" height="240"/><br />
<img src="/NetworkHistory.png?columns=cI&scale=1&maxtime=24&width=660" width="660" height="240"/><br />
<img src="Banner.png?textcolor=000000&amp;bgcolor=e7effc&amp;bordercolor=5090d0" alt="banner" width="468" height="60"/>
</div>

@ -190,7 +190,7 @@ public class Tables_p {
}
// adding the peer address
prop.put("address", sb.peers.mySeed().getPublicAddress(sb.peers.mySeed().getIP()));
//prop.put("address", sb.peers.mySeed().getPublicAddress(sb.peers.mySeed().getIP()));
// return rewrite properties
return prop;

@ -153,9 +153,7 @@ public class Domains {
// fill a cache of local host names
for (final InetAddress a: myHostAddresses) {
String hostaddressP = a.getHostAddress();
int p = hostaddressP.indexOf('%');
if (p > 0) hostaddressP = hostaddressP.substring(0, p);
String hostaddressP = chopZoneID(a.getHostAddress());
Set<String> hns = new LinkedHashSet<>();
// generate alternative representations of IPv6 addresses which are needed to check access on the interface (i.e. localhost check)
if (hostaddressP.indexOf("::") < 0) {
@ -1073,7 +1071,7 @@ public class Domains {
* @return list of all intranet addresses
*/
public static Set<InetAddress> myIntranetIPs() {
if (myHostAddresses.size() < 1) try {Thread.sleep(1000);} catch (final InterruptedException e) {}
if (localHostAddresses.size() < 1) try {Thread.sleep(1000);} catch (final InterruptedException e) {}
return localHostAddresses;
}
@ -1107,7 +1105,7 @@ public class Domains {
public static boolean isThisHostIP(final InetAddress clientAddress) {
if (clientAddress == null) return false;
if (clientAddress.isAnyLocalAddress() || clientAddress.isLoopbackAddress()) return true;
return myHostAddresses.contains(clientAddress);
return myHostAddresses.contains(clientAddress); // includes localHostAddresses
}
public static int getDomainID(final String host, final InetAddress hostaddress) {

@ -49,6 +49,7 @@ import java.net.InetAddress;
import java.net.MalformedURLException;
import java.net.URL;
import java.text.ParseException;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
import java.util.HashSet;
@ -56,6 +57,7 @@ import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.Random;
import java.util.Set;
import java.util.StringTokenizer;
import java.util.TreeMap;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
@ -321,6 +323,18 @@ public class Seed implements Cloneable, Comparable<Seed>, Comparator<Seed>
return name.startsWith("_anon");
}
private Set<String> getIPv6Entries() {
String ip6s = this.dna.get(Seed.IP6);
final Set<String> set = Collections.synchronizedSet(new HashSet<String>());
if (ip6s == null) return set;
final StringTokenizer st = new StringTokenizer(ip6s, "|");
while (st.hasMoreTokens()) {
set.add(Domains.chopZoneID(st.nextToken().trim()));
}
return set;
}
/**
* try to get the public IP<br>
*
@ -329,29 +343,11 @@ public class Seed implements Cloneable, Comparable<Seed>, Comparator<Seed>
@Deprecated
public final String getIP() {
final String ipx = this.dna.get(Seed.IP); // may contain both, IPv4 or IPv6
final String ip6 = this.dna.get(Seed.IP6);
Set<String> ip6s = MapTools.string2set(ip6, "|");
if (ip6s == null || ip6s.size() == 0) {
if (ipx != null && !ipx.isEmpty()) return Domains.chopZoneID(ipx);
}
if (ip6s != null && ip6s.size() == 1) {
// We prefer IPv6
for (String s: ip6s) if (s.length() > 0) return Domains.chopZoneID(s);
if (ipx != null && !ipx.isEmpty()) return Domains.chopZoneID(ipx);
}
// if we have more than one IPv6, then chances are high that one of them do not work.
// in that case we prefer the IPv4
if (ipx != null && !ipx.isEmpty()) return Domains.chopZoneID(ipx);
if (ip6s != null) for (String s: ip6s) if (s.length() > 0) return Domains.chopZoneID(s);
// in case that we don't have any address using the dna (i.e. a fresh peer), then use all locally known addresses
for (InetAddress i: Domains.myPublicIPv4()) return Domains.chopZoneID(i.getHostAddress());
for (InetAddress i: Domains.myPublicIPv6()) return Domains.chopZoneID(i.getHostAddress());
// final chance
return Domains.LOCALHOST;
Set<String> ip6s = getIPv6Entries();
if (ip6s != null && ip6s.size() > 0) return ip6s.iterator().next();
return null;
}
/**
@ -364,29 +360,11 @@ public class Seed implements Cloneable, Comparable<Seed>, Comparator<Seed>
public final Set<String> getIPs() {
Set<String> h = new LinkedHashSet<>();
final String ipx = this.dna.get(Seed.IP); // may contain both, IPv4 or IPv6
final String ip6 = this.dna.get(Seed.IP6);
Set<String> ip6s = MapTools.string2set(ip6, "|");
if (ip6s == null || ip6s.size() == 0) {
if (ipx != null && !ipx.isEmpty()) h.add(Domains.chopZoneID(ipx));
} else if (ip6s != null && ip6s.size() == 1) {
// We add IPv6 first because then those addresses appear first
// in the LinkedHashSet and are preferred by methods using only the first one.
for (String s: ip6s) if (s.length() > 0) h.add(Domains.chopZoneID(s));
if (ipx != null && !ipx.isEmpty()) h.add(Domains.chopZoneID(ipx));
} else {
// if we have more than one IPv6, then chances are high that one of them do not work.
// in that case we prefer the IPv4
if (ipx != null && !ipx.isEmpty()) h.add(Domains.chopZoneID(ipx));
if (ip6s != null) for (String s: ip6s) if (s.length() > 0) h.add(Domains.chopZoneID(s));
}
Set<String> ip6s = getIPv6Entries();
// in case that we don't have any address using the dna (i.e. a fresh peer), then use all locally known addresses
if (h.size() == 0) {
for (InetAddress i: Domains.myPublicIPv4()) h.add(Domains.chopZoneID(i.getHostAddress()));
for (InetAddress i: Domains.myPublicIPv6()) h.add(Domains.chopZoneID(i.getHostAddress()));
h.add(Domains.LOCALHOST);
}
// put IPs in h in such a way that we believe the mostfront have more chances to get connected
if (ipx != null && !ipx.isEmpty()) h.add(Domains.chopZoneID(ipx));
h.addAll(ip6s);
return h;
}
@ -396,8 +374,7 @@ public class Seed implements Cloneable, Comparable<Seed>, Comparator<Seed>
*/
public final int countIPs() {
final String ipx = this.dna.get(Seed.IP); // may contain both, IPv4 or IPv6
final String ip6 = this.dna.get(Seed.IP6);
Set<String> ip6s = MapTools.string2set(ip6, "|");
Set<String> ip6s = getIPv6Entries();
if (ip6s == null || ip6s.size() == 0) {
return (ipx == null || ipx.isEmpty()) ? 0 : 1;
@ -412,10 +389,7 @@ public class Seed implements Cloneable, Comparable<Seed>, Comparator<Seed>
*/
public final boolean removeIP(String ip) {
String ipx = Domains.chopZoneID(this.dna.get(Seed.IP)); // may contain both, IPv4 or IPv6
final String ip6 = this.dna.get(Seed.IP6);
Set<String> ip6s = MapTools.string2set(ip6, "|");
Iterator<String> i = ip6s.iterator();
while (i.hasNext()) {String x = i.next(); if (x.indexOf('%') >= 0) i.remove();}
Set<String> ip6s = getIPv6Entries();
if (ip6s == null || ip6s.size() == 0) {
if (ipx != null && !ipx.isEmpty() && ipx.equals(ip)) {

@ -183,12 +183,12 @@ public class serverSwitch {
for (InetAddress i : Domains.myPublicIPv6()) {
String s = i.getHostAddress();
if (Seed.isProperIP(s))
h.add(s);
h.add(Domains.chopZoneID(s));
}
for (InetAddress i : Domains.myPublicIPv4()) {
String s = i.getHostAddress();
if (Seed.isProperIP(s))
h.add(s);
h.add(Domains.chopZoneID(s));
}
return h;
}

@ -178,7 +178,7 @@ public class ChartPlotter extends RasterPlotter {
s += scale;
}
setColor(colorNaming);
PrintTool.print(this, (left) ? x - s1max * 6 - 6 : x + s1max * 6 + 9, this.topborder, 90, name, 1);
PrintTool.print(this, (left) ? Math.max(6, x - s1max * 6 - 6) : x + s1max * 6 + 9, this.height - this.bottomborder, 90, name, -1);
line(x, this.topborder - 4, x, this.height - this.bottomborder + 4, 100);
}

Loading…
Cancel
Save