boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

added time-out function to web analysis

Browse Source 
the default time-out is 1 second

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@3852 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 18 years ago
parent c9702a106f
commit e07458bad4
5 changed files with 35 additions and 26 deletions
Show Stats Download Patch File Download Diff File

24
htroot/WatchWebStructure_p.html
Unescape View File

@ -28,29 +28,29 @@
<dl>
<dt>host</dt>
<dd>
<input type="text" value="#[host]#" name="host" id="host" size="20" maxlength="80" />
<input type="text" value="#[host]#" name="host" id="host" size="20" maxlength="80" />
</dd>
<dt>depth</dt>
<dd>
<a href="WatchWebStructure_p.html?host=#[host]#&depth=#[depthd]#&width=#[width]#&height=#[height]#&nodes=#[nodes]#"><img src="env/grafics/minus.gif" /></a>
<input type="text" name="depth" value="#[depth]#" size="1" maxlength="1" />
<a href="WatchWebStructure_p.html?host=#[host]#&depth=#[depthi]#&width=#[width]#&height=#[height]#&nodes=#[nodes]#"><img src="env/grafics/plus.gif" /></a>
<a href="WatchWebStructure_p.html?host=#[host]#&depth=#[depthd]#&width=#[width]#&height=#[height]#&nodes=#[nodes]#&time=#[time]#"><img src="env/grafics/minus.gif" /></a>
<input type="text" name="depth" value="#[depth]#" size="1" maxlength="1" />
<a href="WatchWebStructure_p.html?host=#[host]#&depth=#[depthi]#&width=#[width]#&height=#[height]#&nodes=#[nodes]#&time=#[time]#"><img src="env/grafics/plus.gif" /></a>
</dd>
<dt>size</dt>
<dd><input type="text" name="width" value="#[width]#" size="4" maxlength="4" />&nbsp;x&nbsp;<input type="text" name="height" value="#[height]#" size="4" maxlength="4" /></dd>
<dt>nodes</dt>
<dt>nodes</dt>
<dd>
<a href="WatchWebStructure_p.html?host=#[host]#&depth=#[depth]#&width=#[width]#&height=#[height]#&nodes=#[nodesd]#"><img src="env/grafics/minus.gif" /></a>
<input type="text" name="nodes" value="#[nodes]#" size="4" maxlength="4" />
<a href="WatchWebStructure_p.html?host=#[host]#&depth=#[depth]#&width=#[width]#&height=#[height]#&nodes=#[nodesi]#"><img src="env/grafics/plus.gif" /></a>
</dd>
<a href="WatchWebStructure_p.html?host=#[host]#&depth=#[depth]#&width=#[width]#&height=#[height]#&nodes=#[nodesd]#&time=#[time]#"><img src="env/grafics/minus.gif" /></a>
<input type="text" name="nodes" value="#[nodes]#" size="4" maxlength="4" />
<a href="WatchWebStructure_p.html?host=#[host]#&depth=#[depth]#&width=#[width]#&height=#[height]#&nodes=#[nodesi]#&time=#[time]#"><img src="env/grafics/plus.gif" /></a>
</dd>
<dt>size</dt>
<dd><input type="text" name="width" value="#[width]#" size="4" maxlength="4" />&nbsp;x&nbsp;<input type="text" name="height" value="#[height]#" size="4" maxlength="4" /></dd>
<dt>&nbsp;</dt>
<dd><input type="submit" value="change" /></dd>
</dl>
</form>
</div>
<img src="WebStructurePicture_p.png?host=#[host]#&depth=#[depth]#&width=#[width]#&height=#[height]#&nodes=#[nodes]#" />
<img src="WebStructurePicture_p.png?host=#[host]#&depth=#[depth]#&width=#[width]#&height=#[height]#&nodes=#[nodes]#&time=#[time]#" />
#%env/templates/footer.template%#
</body>

3
htroot/WatchWebStructure_p.java
Unescape View File

@ -17,6 +17,7 @@ public class WatchWebStructure_p {
int height = 576;
int depth = 3;
int nodes = 100; // maximum number of host nodes that are painted
int time = -1;
String host = "auto";
if (post != null) {
@ -24,6 +25,7 @@ public class WatchWebStructure_p {
height = post.getInt("height", 576);
depth = post.getInt("depth", 3);
nodes = post.getInt("nodes", width * height * 100 / 768 / 576);
time = post.getInt("time", -1);
host = post.get("host", "auto");
}
@ -50,6 +52,7 @@ public class WatchWebStructure_p {
prop.put("width", width);
prop.put("height", height);
prop.put("nodes", nodes);
prop.put("time", time);
prop.put("nodesi", Math.min(1000, nodes + 50));
prop.put("nodesd", Math.max(0, nodes - 50));

13
htroot/WebStructurePicture_p.java
Unescape View File

@ -51,6 +51,7 @@ public class WebStructurePicture_p {
int height = 576;
int depth = 3;
int nodes = 100; // maximum number of host nodes that are painted
int time = -1;
String host = null;
if (post != null) {
@ -58,6 +59,7 @@ public class WebStructurePicture_p {
height = post.getInt("height", 576);
depth = post.getInt("depth", 3);
nodes = post.getInt("nodes", width * height * 100 / 768 / 576);
time = post.getInt("time", -1);
host = post.get("host", null);
}
@ -69,6 +71,9 @@ public class WebStructurePicture_p {
if (depth > 8) depth = 8;
if (depth < 0) depth = 0;
// calculate target time
long timeout = (time < 0) ? Long.MAX_VALUE : System.currentTimeMillis() + time;
// find start point
if ((host == null) || (host.length() == 0) || (host.equals("auto"))) {
// find domain with most references
@ -83,14 +88,14 @@ public class WebStructurePicture_p {
// recursively find domains, up to a specific depth
ymageGraph graph = new ymageGraph();
if (host != null) place(graph, sb.webStructure, hash, host, nodes, 0.0, 0.0, 0, depth);
if (host != null) place(graph, sb.webStructure, hash, host, nodes, timeout, 0.0, 0.0, 0, depth);
//graph.print();
return graph.draw(width, height, 40, 40, 5, 15);
}
private static final int place(ymageGraph graph, plasmaWebStructure structure, String centerhash, String centerhost, int maxnodes, double x, double y, int nextlayer, int maxlayer) {
private static final int place(ymageGraph graph, plasmaWebStructure structure, String centerhash, String centerhost, int maxnodes, long timeout, double x, double y, int nextlayer, int maxlayer) {
// returns the number of nodes that had been placed
assert centerhost != null;
ymageGraph.coordinate center = graph.getPoint(centerhost);
@ -112,7 +117,7 @@ public class WebStructurePicture_p {
int maxtargetrefs = 8, maxthisrefs = 8;
int targetrefs, thisrefs;
double rr, re;
while ((i.hasNext()) && (maxnodes > 0)) {
while ((i.hasNext()) && (maxnodes > 0) && (System.currentTimeMillis() < timeout)) {
entry = (Map.Entry) i.next();
targethash = (String) entry.getKey();
targethost = structure.resolveDomHash2DomString(targethash);
@ -142,7 +147,7 @@ public class WebStructurePicture_p {
targethost = target[1];
ymageGraph.coordinate c = graph.getPoint(targethost);
assert c != null;
nextnodes = (maxnodes <= 0) ? 0 : place(graph, structure, targethash, targethost, maxnodes, c.x, c.y, nextlayer, maxlayer);
nextnodes = ((maxnodes <= 0) || (System.currentTimeMillis() >= timeout)) ? 0 : place(graph, structure, targethash, targethost, maxnodes, timeout, c.x, c.y, nextlayer, maxlayer);
mynodes += nextnodes;
maxnodes -= nextnodes;
graph.setBorder(centerhost, targethost);

2
htroot/env/templates/header.template vendored
Unescape View File

@ -25,7 +25,7 @@
<h3>Monitoring</h3>
<ul class="menu">
<li><a href="/Network.html" accesskey="w" class="MenuItemLink">YaCy Network</a></li>
<li><a href="/WatchWebStructure_p.html?host=auto&depth=2" class="MenuItemLink lock">Crawl Activity</a></li>
<li><a href="/WatchWebStructure_p.html?host=auto&depth=2&time=1000" class="MenuItemLink lock">Crawl Activity</a></li>
<li><a href="/IndexMonitor.html" class="MenuItemLink">Crawl Results</a></li>
<li><a href="/SearchStatistics_p.html" class="MenuItemLink lock">Search Statistics</a></li>
<li><a href="/ViewLog_p.html" class="MenuItemLink lock">Server Log</a></li>

19
source/de/anomic/plasma/plasmaWebStructure.java
Unescape View File

@ -33,8 +33,9 @@ import java.util.ConcurrentModificationException;
import java.util.Date;
import java.util.Iterator;
import java.util.Map;
import java.util.SortedMap;
import java.util.HashMap;
import java.util.TreeMap;
import java.util.SortedMap;
import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.net.URL;
@ -155,9 +156,9 @@ public class plasmaWebStructure {
return (refs.length() - 8) / 10;
}
private static TreeMap refstr2map(String refs) {
if ((refs == null) || (refs.length() <= 8)) return new TreeMap();
TreeMap map = new TreeMap();
private static Map refstr2map(String refs) {
if ((refs == null) || (refs.length() <= 8)) return new HashMap();
Map map = new HashMap();
String c;
int refsc = refstr2count(refs);
for (int i = 0; i < refsc; i++) {
@ -167,7 +168,7 @@ public class plasmaWebStructure {
return map;
}
private static String map2refstr(TreeMap map) {
private static String map2refstr(Map map) {
StringBuffer s = new StringBuffer(map.size() * 10);
s.append(plasmaURL.shortDayFormatter.format(new Date()));
Iterator i = map.entrySet().iterator();
@ -194,16 +195,16 @@ public class plasmaWebStructure {
return s.toString();
}
public TreeMap references(String domhash) {
public Map references(String domhash) {
// returns a map with a domhash(String):refcount(Integer) relation
assert domhash.length() == 6;
SortedMap tailMap = structure.tailMap(domhash);
if ((tailMap == null) || (tailMap.size() == 0)) return new TreeMap();
if ((tailMap == null) || (tailMap.size() == 0)) return new HashMap();
String key = (String) tailMap.firstKey();
if (key.startsWith(domhash)) {
return refstr2map((String) tailMap.get(key));
} else {
return new TreeMap();
return new HashMap();
}
}
@ -244,7 +245,7 @@ public class plasmaWebStructure {
String domhash = plasmaURL.urlHash(url).substring(6);
// parse the new reference string and join it with the stored references
TreeMap refs = references(domhash);
Map refs = references(domhash);
assert reference.length() % 12 == 0;
String dom;
int c;

Write Preview
Loading…
Cancel
Save