boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

added time-out function to web analysis

Browse Source 
the default time-out is 1 second

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@3852 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 18 years ago
parent c9702a106f
commit e07458bad4
5 changed files with 35 additions and 26 deletions
Show Stats Download Patch File Download Diff File

24
htroot/WatchWebStructure_p.html
Unescape View File

@ -28,29 +28,29 @@
<dl> <dl>
<dt>host</dt> <dt>host</dt>
<dd> <dd>
<input type="text" value="#[host]#" name="host" id="host" size="20" maxlength="80" /> <input type="text" value="#[host]#" name="host" id="host" size="20" maxlength="80" />
</dd> </dd>
<dt>depth</dt> <dt>depth</dt>
<dd> <dd>
<a href="WatchWebStructure_p.html?host=#[host]#&depth=#[depthd]#&width=#[width]#&height=#[height]#&nodes=#[nodes]#"><img src="env/grafics/minus.gif" /></a> <a href="WatchWebStructure_p.html?host=#[host]#&depth=#[depthd]#&width=#[width]#&height=#[height]#&nodes=#[nodes]#&time=#[time]#"><img src="env/grafics/minus.gif" /></a>
<input type="text" name="depth" value="#[depth]#" size="1" maxlength="1" /> <input type="text" name="depth" value="#[depth]#" size="1" maxlength="1" />
<a href="WatchWebStructure_p.html?host=#[host]#&depth=#[depthi]#&width=#[width]#&height=#[height]#&nodes=#[nodes]#"><img src="env/grafics/plus.gif" /></a> <a href="WatchWebStructure_p.html?host=#[host]#&depth=#[depthi]#&width=#[width]#&height=#[height]#&nodes=#[nodes]#&time=#[time]#"><img src="env/grafics/plus.gif" /></a>
</dd> </dd>
<dt>size</dt> <dt>nodes</dt>
<dd><input type="text" name="width" value="#[width]#" size="4" maxlength="4" />&nbsp;x&nbsp;<input type="text" name="height" value="#[height]#" size="4" maxlength="4" /></dd>
<dt>nodes</dt>
<dd> <dd>
<a href="WatchWebStructure_p.html?host=#[host]#&depth=#[depth]#&width=#[width]#&height=#[height]#&nodes=#[nodesd]#"><img src="env/grafics/minus.gif" /></a> <a href="WatchWebStructure_p.html?host=#[host]#&depth=#[depth]#&width=#[width]#&height=#[height]#&nodes=#[nodesd]#&time=#[time]#"><img src="env/grafics/minus.gif" /></a>
<input type="text" name="nodes" value="#[nodes]#" size="4" maxlength="4" /> <input type="text" name="nodes" value="#[nodes]#" size="4" maxlength="4" />
<a href="WatchWebStructure_p.html?host=#[host]#&depth=#[depth]#&width=#[width]#&height=#[height]#&nodes=#[nodesi]#"><img src="env/grafics/plus.gif" /></a> <a href="WatchWebStructure_p.html?host=#[host]#&depth=#[depth]#&width=#[width]#&height=#[height]#&nodes=#[nodesi]#&time=#[time]#"><img src="env/grafics/plus.gif" /></a>
</dd> </dd>
<dt>size</dt>
<dd><input type="text" name="width" value="#[width]#" size="4" maxlength="4" />&nbsp;x&nbsp;<input type="text" name="height" value="#[height]#" size="4" maxlength="4" /></dd>
<dt>&nbsp;</dt> <dt>&nbsp;</dt>
<dd><input type="submit" value="change" /></dd> <dd><input type="submit" value="change" /></dd>
</dl> </dl>
</form> </form>
</div> </div>
<img src="WebStructurePicture_p.png?host=#[host]#&depth=#[depth]#&width=#[width]#&height=#[height]#&nodes=#[nodes]#" /> <img src="WebStructurePicture_p.png?host=#[host]#&depth=#[depth]#&width=#[width]#&height=#[height]#&nodes=#[nodes]#&time=#[time]#" />
#%env/templates/footer.template%# #%env/templates/footer.template%#
</body> </body>

3
htroot/WatchWebStructure_p.java
Unescape View File

@ -17,6 +17,7 @@ public class WatchWebStructure_p {
int height = 576; int height = 576;
int depth = 3; int depth = 3;
int nodes = 100; // maximum number of host nodes that are painted int nodes = 100; // maximum number of host nodes that are painted
int time = -1;
String host = "auto"; String host = "auto";
if (post != null) { if (post != null) {
@ -24,6 +25,7 @@ public class WatchWebStructure_p {
height = post.getInt("height", 576); height = post.getInt("height", 576);
depth = post.getInt("depth", 3); depth = post.getInt("depth", 3);
nodes = post.getInt("nodes", width * height * 100 / 768 / 576); nodes = post.getInt("nodes", width * height * 100 / 768 / 576);
time = post.getInt("time", -1);
host = post.get("host", "auto"); host = post.get("host", "auto");
} }
@ -50,6 +52,7 @@ public class WatchWebStructure_p {
prop.put("width", width); prop.put("width", width);
prop.put("height", height); prop.put("height", height);
prop.put("nodes", nodes); prop.put("nodes", nodes);
prop.put("time", time);
prop.put("nodesi", Math.min(1000, nodes + 50)); prop.put("nodesi", Math.min(1000, nodes + 50));
prop.put("nodesd", Math.max(0, nodes - 50)); prop.put("nodesd", Math.max(0, nodes - 50));

13
htroot/WebStructurePicture_p.java
Unescape View File

@ -51,6 +51,7 @@ public class WebStructurePicture_p {
int height = 576; int height = 576;
int depth = 3; int depth = 3;
int nodes = 100; // maximum number of host nodes that are painted int nodes = 100; // maximum number of host nodes that are painted
int time = -1;
String host = null; String host = null;
if (post != null) { if (post != null) {
@ -58,6 +59,7 @@ public class WebStructurePicture_p {
height = post.getInt("height", 576); height = post.getInt("height", 576);
depth = post.getInt("depth", 3); depth = post.getInt("depth", 3);
nodes = post.getInt("nodes", width * height * 100 / 768 / 576); nodes = post.getInt("nodes", width * height * 100 / 768 / 576);
time = post.getInt("time", -1);
host = post.get("host", null); host = post.get("host", null);
} }
@ -69,6 +71,9 @@ public class WebStructurePicture_p {
if (depth > 8) depth = 8; if (depth > 8) depth = 8;
if (depth < 0) depth = 0; if (depth < 0) depth = 0;
// calculate target time
long timeout = (time < 0) ? Long.MAX_VALUE : System.currentTimeMillis() + time;
// find start point // find start point
if ((host == null) || (host.length() == 0) || (host.equals("auto"))) { if ((host == null) || (host.length() == 0) || (host.equals("auto"))) {
// find domain with most references // find domain with most references
@ -83,14 +88,14 @@ public class WebStructurePicture_p {
// recursively find domains, up to a specific depth // recursively find domains, up to a specific depth
ymageGraph graph = new ymageGraph(); ymageGraph graph = new ymageGraph();
if (host != null) place(graph, sb.webStructure, hash, host, nodes, 0.0, 0.0, 0, depth); if (host != null) place(graph, sb.webStructure, hash, host, nodes, timeout, 0.0, 0.0, 0, depth);
//graph.print(); //graph.print();
return graph.draw(width, height, 40, 40, 5, 15); return graph.draw(width, height, 40, 40, 5, 15);
} }
private static final int place(ymageGraph graph, plasmaWebStructure structure, String centerhash, String centerhost, int maxnodes, double x, double y, int nextlayer, int maxlayer) { private static final int place(ymageGraph graph, plasmaWebStructure structure, String centerhash, String centerhost, int maxnodes, long timeout, double x, double y, int nextlayer, int maxlayer) {
// returns the number of nodes that had been placed // returns the number of nodes that had been placed
assert centerhost != null; assert centerhost != null;
ymageGraph.coordinate center = graph.getPoint(centerhost); ymageGraph.coordinate center = graph.getPoint(centerhost);
@ -112,7 +117,7 @@ public class WebStructurePicture_p {
int maxtargetrefs = 8, maxthisrefs = 8; int maxtargetrefs = 8, maxthisrefs = 8;
int targetrefs, thisrefs; int targetrefs, thisrefs;
double rr, re; double rr, re;
while ((i.hasNext()) && (maxnodes > 0)) { while ((i.hasNext()) && (maxnodes > 0) && (System.currentTimeMillis() < timeout)) {
entry = (Map.Entry) i.next(); entry = (Map.Entry) i.next();
targethash = (String) entry.getKey(); targethash = (String) entry.getKey();
targethost = structure.resolveDomHash2DomString(targethash); targethost = structure.resolveDomHash2DomString(targethash);
@ -142,7 +147,7 @@ public class WebStructurePicture_p {
targethost = target[1]; targethost = target[1];
ymageGraph.coordinate c = graph.getPoint(targethost); ymageGraph.coordinate c = graph.getPoint(targethost);
assert c != null; assert c != null;
nextnodes = (maxnodes <= 0) ? 0 : place(graph, structure, targethash, targethost, maxnodes, c.x, c.y, nextlayer, maxlayer); nextnodes = ((maxnodes <= 0) || (System.currentTimeMillis() >= timeout)) ? 0 : place(graph, structure, targethash, targethost, maxnodes, timeout, c.x, c.y, nextlayer, maxlayer);
mynodes += nextnodes; mynodes += nextnodes;
maxnodes -= nextnodes; maxnodes -= nextnodes;
graph.setBorder(centerhost, targethost); graph.setBorder(centerhost, targethost);

2
htroot/env/templates/header.template vendored
Unescape View File

@ -25,7 +25,7 @@
<h3>Monitoring</h3> <h3>Monitoring</h3>
<ul class="menu"> <ul class="menu">
<li><a href="/Network.html" accesskey="w" class="MenuItemLink">YaCy Network</a></li> <li><a href="/Network.html" accesskey="w" class="MenuItemLink">YaCy Network</a></li>
<li><a href="/WatchWebStructure_p.html?host=auto&depth=2" class="MenuItemLink lock">Crawl Activity</a></li> <li><a href="/WatchWebStructure_p.html?host=auto&depth=2&time=1000" class="MenuItemLink lock">Crawl Activity</a></li>
<li><a href="/IndexMonitor.html" class="MenuItemLink">Crawl Results</a></li> <li><a href="/IndexMonitor.html" class="MenuItemLink">Crawl Results</a></li>
<li><a href="/SearchStatistics_p.html" class="MenuItemLink lock">Search Statistics</a></li> <li><a href="/SearchStatistics_p.html" class="MenuItemLink lock">Search Statistics</a></li>
<li><a href="/ViewLog_p.html" class="MenuItemLink lock">Server Log</a></li> <li><a href="/ViewLog_p.html" class="MenuItemLink lock">Server Log</a></li>

19
source/de/anomic/plasma/plasmaWebStructure.java
Unescape View File

@ -33,8 +33,9 @@ import java.util.ConcurrentModificationException;
import java.util.Date; import java.util.Date;
import java.util.Iterator; import java.util.Iterator;
import java.util.Map; import java.util.Map;
import java.util.SortedMap; import java.util.HashMap;
import java.util.TreeMap; import java.util.TreeMap;
import java.util.SortedMap;
import de.anomic.kelondro.kelondroBase64Order; import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.net.URL; import de.anomic.net.URL;
@ -155,9 +156,9 @@ public class plasmaWebStructure {
return (refs.length() - 8) / 10; return (refs.length() - 8) / 10;
} }
private static TreeMap refstr2map(String refs) { private static Map refstr2map(String refs) {
if ((refs == null) || (refs.length() <= 8)) return new TreeMap(); if ((refs == null) || (refs.length() <= 8)) return new HashMap();
TreeMap map = new TreeMap(); Map map = new HashMap();
String c; String c;
int refsc = refstr2count(refs); int refsc = refstr2count(refs);
for (int i = 0; i < refsc; i++) { for (int i = 0; i < refsc; i++) {
@ -167,7 +168,7 @@ public class plasmaWebStructure {
return map; return map;
} }
private static String map2refstr(TreeMap map) { private static String map2refstr(Map map) {
StringBuffer s = new StringBuffer(map.size() * 10); StringBuffer s = new StringBuffer(map.size() * 10);
s.append(plasmaURL.shortDayFormatter.format(new Date())); s.append(plasmaURL.shortDayFormatter.format(new Date()));
Iterator i = map.entrySet().iterator(); Iterator i = map.entrySet().iterator();
@ -194,16 +195,16 @@ public class plasmaWebStructure {
return s.toString(); return s.toString();
} }
public TreeMap references(String domhash) { public Map references(String domhash) {
// returns a map with a domhash(String):refcount(Integer) relation // returns a map with a domhash(String):refcount(Integer) relation
assert domhash.length() == 6; assert domhash.length() == 6;
SortedMap tailMap = structure.tailMap(domhash); SortedMap tailMap = structure.tailMap(domhash);
if ((tailMap == null) || (tailMap.size() == 0)) return new TreeMap(); if ((tailMap == null) || (tailMap.size() == 0)) return new HashMap();
String key = (String) tailMap.firstKey(); String key = (String) tailMap.firstKey();
if (key.startsWith(domhash)) { if (key.startsWith(domhash)) {
return refstr2map((String) tailMap.get(key)); return refstr2map((String) tailMap.get(key));
} else { } else {
return new TreeMap(); return new HashMap();
} }
} }
@ -244,7 +245,7 @@ public class plasmaWebStructure {
String domhash = plasmaURL.urlHash(url).substring(6); String domhash = plasmaURL.urlHash(url).substring(6);
// parse the new reference string and join it with the stored references // parse the new reference string and join it with the stored references
TreeMap refs = references(domhash); Map refs = references(domhash);
assert reference.length() % 12 == 0; assert reference.length() % 12 == 0;
String dom; String dom;
int c; int c;

Write Preview
Loading…
Cancel
Save