see http://localhost:8080/WatchWebStructure_p.html git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@3747 6c8d7289-2bf4-0310-a012-ef5d649a1542pull/1/head
orbiter
18 years ago
parent
33ad0c8246
commit
a585b4d41b
@ -0,0 +1,15 @@
|
||||
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
|
||||
<html xmlns="http://www.w3.org/1999/xhtml">
|
||||
<head>
|
||||
<title>YaCy '#[clientname]#': Web Structure</title>
|
||||
#%env/templates/metas.template%#
|
||||
</head>
|
||||
<body id="WebStructure">
|
||||
#%env/templates/header.template%#
|
||||
#%env/templates/submenuCrawler.template%#
|
||||
<h2>Web Structure</h2>
|
||||
<img src="WebStructurePicture_p.png">
|
||||
|
||||
#%env/templates/footer.template%#
|
||||
</body>
|
||||
</html>
|
||||
@ -0,0 +1,14 @@
|
||||
|
||||
import de.anomic.http.httpHeader;
|
||||
import de.anomic.plasma.plasmaSwitchboard;
|
||||
import de.anomic.server.serverObjects;
|
||||
import de.anomic.server.serverSwitch;
|
||||
|
||||
|
||||
public class WatchWebStructure_p {
|
||||
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) {
|
||||
plasmaSwitchboard sb = (plasmaSwitchboard) env;
|
||||
serverObjects prop = new serverObjects();
|
||||
return prop;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,132 @@
|
||||
// WebStructurePicture.java
|
||||
// (C) 2007 by Michael Peter Christen; mc@anomic.de, Frankfurt a. M., Germany
|
||||
// first published 22.05.2007 on http://yacy.net
|
||||
//
|
||||
// This is a part of YaCy, a peer-to-peer based web search engine
|
||||
//
|
||||
// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
|
||||
// $LastChangedRevision: 1986 $
|
||||
// $LastChangedBy: orbiter $
|
||||
//
|
||||
// LICENSE
|
||||
//
|
||||
// This program is free software; you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation; either version 2 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with this program; if not, write to the Free Software
|
||||
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
|
||||
import java.net.MalformedURLException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Iterator;
|
||||
import java.util.Map;
|
||||
|
||||
import de.anomic.http.httpHeader;
|
||||
import de.anomic.kelondro.kelondroBase64Order;
|
||||
import de.anomic.net.URL;
|
||||
import de.anomic.plasma.plasmaSwitchboard;
|
||||
import de.anomic.plasma.plasmaURL;
|
||||
import de.anomic.plasma.plasmaWebStructure;
|
||||
import de.anomic.server.serverObjects;
|
||||
import de.anomic.server.serverSwitch;
|
||||
import de.anomic.ymage.ymageGraph;
|
||||
import de.anomic.ymage.ymageMatrix;
|
||||
|
||||
public class WebStructurePicture_p {
|
||||
|
||||
private static final double maxlongd = (double) Long.MAX_VALUE;
|
||||
|
||||
public static ymageMatrix respond(httpHeader header, serverObjects post, serverSwitch env) {
|
||||
plasmaSwitchboard sb = (plasmaSwitchboard) env;
|
||||
|
||||
int width = 768;
|
||||
int height = 576;
|
||||
int depth = 3;
|
||||
String host = null;
|
||||
|
||||
if (post != null) {
|
||||
width = post.getInt("width", 768);
|
||||
height = post.getInt("height", 576);
|
||||
depth = post.getInt("depth", 3);
|
||||
host = post.get("host", null);
|
||||
}
|
||||
|
||||
//too small values lead to an error, too big to huge CPU/memory consumption, resulting in possible DOS.
|
||||
if (width < 320 ) width = 320;
|
||||
if (width > 1920) width = 1920;
|
||||
if (height < 240) height = 240;
|
||||
if (height > 1920) height = 1920;
|
||||
if (depth > 8) depth = 8;
|
||||
if (depth < 1) depth = 1;
|
||||
|
||||
// find start point
|
||||
if (host == null) {
|
||||
// find domain with most references
|
||||
host = sb.webStructure.hostWithMaxReferences();
|
||||
}
|
||||
// find start hash
|
||||
String hash = null;
|
||||
try {
|
||||
hash = plasmaURL.urlHash(new URL("http://" + host)).substring(6);
|
||||
} catch (MalformedURLException e) {e.printStackTrace();}
|
||||
assert (sb.webStructure.references(hash) != null);
|
||||
|
||||
// recursively find domains, up to a specific depth
|
||||
ymageGraph graph = new ymageGraph();
|
||||
place(graph, sb.webStructure, hash, host, 0.0, 0.0, 0, depth);
|
||||
//graph.print();
|
||||
|
||||
return graph.draw(width, height, 20, 20, 20, 20);
|
||||
|
||||
}
|
||||
|
||||
private static final void place(ymageGraph graph, plasmaWebStructure structure, String centerhash, String centerhost, double x, double y, int nextlayer, int maxlayer) {
|
||||
// returns the host string
|
||||
assert centerhost != null;
|
||||
ymageGraph.coordinate center = graph.getPoint(centerhost);
|
||||
if (center == null) center = graph.addPoint(centerhost, x, y, nextlayer);
|
||||
if (nextlayer == maxlayer) return;
|
||||
nextlayer++;
|
||||
Map next = structure.references(centerhash);
|
||||
Map.Entry entry;
|
||||
String targethash, targethost;
|
||||
// first set points to next hosts
|
||||
Iterator i = next.entrySet().iterator();
|
||||
ArrayList targets = new ArrayList();
|
||||
while (i.hasNext()) {
|
||||
entry = (Map.Entry) i.next();
|
||||
targethash = (String) entry.getKey();
|
||||
targethost = structure.resolveDomHash2DomString(targethash);
|
||||
if (targethost == null) continue;
|
||||
targets.add(new String[] {targethash, targethost});
|
||||
if (graph.getPoint(targethost) != null) continue;
|
||||
// set a new point. It is placed on a circle around the host point
|
||||
double angle = ((double) kelondroBase64Order.enhancedCoder.cardinal((targethash + "____").getBytes())) / maxlongd * 2 * Math.PI;
|
||||
System.out.println("ANGLE = " + angle);
|
||||
double radius = 1.0 / ((double) (1 << nextlayer));
|
||||
graph.addPoint(targethost, x + radius * Math.cos(angle), y + radius * Math.sin(angle), nextlayer);
|
||||
}
|
||||
// recursively set next hosts
|
||||
i = targets.iterator();
|
||||
String[] target;
|
||||
while (i.hasNext()) {
|
||||
target = (String[]) i.next();
|
||||
targethash = target[0];
|
||||
targethost = target[1];
|
||||
ymageGraph.coordinate c = graph.getPoint(targethost);
|
||||
assert c != null;
|
||||
place(graph, structure, targethash, targethost, c.x, c.y, nextlayer, maxlayer);
|
||||
graph.setBorder(centerhost, targethost);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,7 @@
|
||||
<div class="SubMenu">
|
||||
<h3>Crawler Monitor Menu</h3>
|
||||
<ul class="SubMenu">
|
||||
<li><a href="/WatchCrawler_p.html" class="MenuItemLink lock">Crawler Queues</a></li>
|
||||
<li><a href="/WatchWebStructure_p.html" class="MenuItemLink lock">Web Structure</a></li>
|
||||
</ul>
|
||||
</div>
|
||||
@ -0,0 +1,142 @@
|
||||
package de.anomic.ymage;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.Iterator;
|
||||
import java.util.Map;
|
||||
|
||||
/* this class does not really draw graphes, it is only a container for graph coordinates.
|
||||
* all coordinates are given in a artificial corrdinate system, in the range from
|
||||
* -1 to +1. The lower left point of the graph has the coordinate -1, -1 and the upper
|
||||
* right is 1,1
|
||||
* 0,0 is the center of the graph
|
||||
*/
|
||||
|
||||
public class ymageGraph {
|
||||
|
||||
// a ymageGraph is a set of points and borders between the points
|
||||
// to reference the points, they must all have a nickname
|
||||
|
||||
HashMap points;
|
||||
HashSet borders;
|
||||
double leftmost, rightmost, topmost, bottommost;
|
||||
|
||||
public ymageGraph() {
|
||||
points = new HashMap();
|
||||
borders = new HashSet();
|
||||
leftmost = 1.0;
|
||||
rightmost = -1.0;
|
||||
topmost = -1.0;
|
||||
bottommost = 1.0;
|
||||
}
|
||||
|
||||
public coordinate getPoint(String name) {
|
||||
return (coordinate) points.get(name);
|
||||
}
|
||||
|
||||
public coordinate[] getBorder(String name) {
|
||||
int p = name.indexOf("$");
|
||||
if (p < 0) return null;
|
||||
coordinate from = getPoint(name.substring(0, p));
|
||||
coordinate to = getPoint(name.substring(p + 1));
|
||||
if ((from == null) || (to == null)) return null;
|
||||
return new coordinate[] {from, to};
|
||||
}
|
||||
|
||||
public coordinate addPoint(String name, double x, double y, int layer) {
|
||||
coordinate newc = new coordinate(x, y, layer);
|
||||
coordinate oldc = (coordinate) points.put(name, newc);
|
||||
assert oldc == null; // all add shall be unique
|
||||
if (x > rightmost) rightmost = x;
|
||||
if (x < leftmost) leftmost = x;
|
||||
if (y > topmost) topmost = y;
|
||||
if (y < bottommost) bottommost = y;
|
||||
return newc;
|
||||
}
|
||||
|
||||
public boolean hasBorder(String fromPoint, String toPoint) {
|
||||
return borders.contains(fromPoint + "-" + toPoint);
|
||||
}
|
||||
|
||||
public void setBorder(String fromPoint, String toPoint) {
|
||||
coordinate from = (coordinate) points.get(fromPoint);
|
||||
coordinate to = (coordinate) points.get(toPoint);
|
||||
assert from != null;
|
||||
assert to != null;
|
||||
borders.add(fromPoint + "$" + toPoint);
|
||||
}
|
||||
|
||||
public class coordinate {
|
||||
public double x, y;
|
||||
public int layer;
|
||||
public coordinate(double x, double y, int layer) {
|
||||
assert x >= -1;
|
||||
assert x <= 1;
|
||||
assert y >= -1;
|
||||
assert y <= 1;
|
||||
this.x = x;
|
||||
this.y = y;
|
||||
this.layer = layer;
|
||||
}
|
||||
}
|
||||
|
||||
public void print() {
|
||||
// for debug purpose: print out all coordinates
|
||||
Iterator i = points.entrySet().iterator();
|
||||
Map.Entry entry;
|
||||
String name;
|
||||
coordinate c;
|
||||
while (i.hasNext()) {
|
||||
entry = (Map.Entry) i.next();
|
||||
name = (String) entry.getKey();
|
||||
c = (coordinate) entry.getValue();
|
||||
System.out.println("point(" + c.x + ", " + c.y + ", " + c.layer + ") [" + name + "]");
|
||||
}
|
||||
i = borders.iterator();
|
||||
while (i.hasNext()) {
|
||||
System.out.println("border(" + i.next() + ")");
|
||||
}
|
||||
}
|
||||
|
||||
private static final long color_back = ymageMatrix.SUBTRACTIVE_WHITE;
|
||||
private static final long color_dot = 0x4444AA;
|
||||
private static final long color_line = 0x333333;
|
||||
private static final long color_text = ymageMatrix.SUBTRACTIVE_BLACK;
|
||||
|
||||
public ymageMatrix draw(int width, int height, int leftborder, int rightborder, int topborder, int bottomborder) {
|
||||
ymageMatrix image = new ymageMatrix(width, height, color_back);
|
||||
double xfactor = (width - leftborder - rightborder) / (rightmost - leftmost);
|
||||
double yfactor = (height - topborder - bottomborder) / (topmost - bottommost);
|
||||
image.setMode(ymageMatrix.MODE_SUB);
|
||||
|
||||
Iterator i = points.entrySet().iterator();
|
||||
Map.Entry entry;
|
||||
String name;
|
||||
coordinate c;
|
||||
int x, y;
|
||||
while (i.hasNext()) {
|
||||
entry = (Map.Entry) i.next();
|
||||
name = (String) entry.getKey();
|
||||
c = (coordinate) entry.getValue();
|
||||
x = (int) (leftborder + (c.x - leftmost) * xfactor);
|
||||
y = (int) (height - bottomborder - (c.y - bottommost) * yfactor);
|
||||
image.setColor(color_dot);
|
||||
image.dot(x, y, 5, true);
|
||||
image.setColor(color_text);
|
||||
ymageToolPrint.print(image, x, y + 10, 0, name.toUpperCase(), 0);
|
||||
}
|
||||
i = borders.iterator();
|
||||
coordinate[] border;
|
||||
image.setColor(color_line);
|
||||
while (i.hasNext()) {
|
||||
border = getBorder((String) i.next());
|
||||
if (border == null) continue;
|
||||
image.line(
|
||||
(int) (leftborder + (border[0].x - leftmost) * xfactor),
|
||||
(int) (height - bottomborder - (border[0].y - bottommost) * yfactor),
|
||||
(int) (leftborder + (border[1].x - leftmost) * xfactor),
|
||||
(int) (height - bottomborder - (border[1].y - bottommost) * yfactor));
|
||||
}
|
||||
return image;
|
||||
}
|
||||
}
|
||||
Loading…
Reference in new issue