boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

updated terminal and dynamic webstructure applet: can now change when crawl is running

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4780 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 17 years ago
parent 906c144799
commit f42c8cf69c
8 changed files with 659 additions and 339 deletions
Show Stats Download Patch File Download Diff File

BIN
htroot/processing/domaingraph/applet/domaingraph.jar
View File

Binary file not shown.

285
htroot/processing/domaingraph/applet/domaingraph.java
Unescape View File

@ -12,77 +12,124 @@ import processing.core.*; import traer.physics.*; import traer.animation.*; impo
final float NODE_SIZE = 6; final float NODE_SIZE = 6;
final float EDGE_LENGTH = 50; final float EDGE_LENGTH = 30;
final float EDGE_STRENGTH = 0.01f; final float EDGE_STRENGTH = 0.001f;
final float SPACER_STRENGTH = 10; final float SPACER_STRENGTH = 250;
ParticleSystem physics; ParticleSystem physics;
Smoother3D centroid; Smoother3D centroid;
PFont font; PFont font;
float x = 0.0f;
float y = 0.0f;
float z = 1.0f;
Client myClient; Client myClient;
Particle center0, center1; // two gravity centers to support object ordering for non-quadratic fields Particle center0, center1; // two gravity centers to support object ordering for non-quadratic fields
String parsingHostName = ""; String parsingHostName = "";
String parsingHostID = ""; String parsingHostID = "";
HashMap nodes = new HashMap(); // map that holds host objects HashMap nodes = new HashMap(); // map that holds host objects
String host;
int port;
float a = 0.0f;
long lastUpdate = Long.MAX_VALUE;
boolean initTime = true;
public void setup() { public void setup() {
String[] fontList = PFont.list(); String[] fontList = PFont.list();
//println(fontList); //println(fontList);
font = createFont(fontList[0], 32); //just take any, should be mostly Arial font = createFont(fontList[0], 32); //just take any, should be mostly Arial
textFont(font, 12); textFont(font, 9);
size(660, 400); size(660, 400);
smooth(); smooth();
frameRate( 24 ); frameRate( 12 );
strokeWeight( 2 ); strokeWeight( 1 );
ellipseMode( CENTER ); ellipseMode( CENTER );
physics = new ParticleSystem( 0, 0.25f ); physics = new ParticleSystem( 0, 0.25f );
centroid = new Smoother3D( 0.8f ); centroid = new Smoother3D( 0.8f );
initializePhysics(); initializePhysics();
initRequest(); URL url = null;
try {
url = getDocumentBase();
} catch (NullPointerException e) {}
if (url == null) {
host="localhost";
port=8080;
} else {
host=url.getHost();
port=url.getPort();
}
//println("CodeBase: " + url);
//println("host: " + host);
//println("port: " + port);
initRequest(false);
} }
public void initRequest() { public void initializePhysics() {
myClient = new Client(this, "localhost", 8080); physics.clear();
myClient.write("GET /xml/webstructure.xml HTTP/1.1\n"); center0 = physics.makeParticle(1.0f, -EDGE_LENGTH * 10, 0, 0);
myClient.write("Host: localhost\n\n"); center0.makeFixed();
center1 = physics.makeParticle(1.0f, EDGE_LENGTH * 10, 0, 0);
center1.makeFixed();
centroid.setValue( 0, 0, 1.0f );
} }
public void processRequestResponse() { public void draw() {
if (myClient.available() > 0) { processRequestResponse(20);
String line = myClient.readStringUntil((byte) 10);
//println("Line: " + line); physics.tick( 1.0f );
if (line == null) line = ""; else line = line.trim(); HashSet invisible = invisibleParticles();
if (line.startsWith("<domain")) processDomain(parseProps(line.substring(7, line.length() - 1).trim())); if (physics.numberOfParticles() > 1) updateCentroid(invisible);
if (line.startsWith("<citation")) processCitation(parseProps(line.substring(9, line.length() - 2).trim())); centroid.tick();
}
background( 0 );
translate( width/2 , height/2 );
scale( centroid.z() );
translate( -centroid.x(), -centroid.y() );
drawNetwork(invisible);
} }
public HashMap parseProps(String s) { public void initRequest(boolean update) {
String[] l = s.split(" "); myClient = new Client(this, host, port);
HashMap map = new HashMap(); myClient.write((update) ? "GET /xml/webstructure.xml?latest= HTTP/1.1\n" : "GET /xml/webstructure.xml HTTP/1.1\n");
int p; myClient.write("Host: localhost\n\n");
String z; }
for (int i = 0; i < l.length; i++) {
p = l[i].indexOf("="); public void processRequestResponse(int steps) {
if (p > 0) { if (((myClient == null) || (myClient.available() <= 0)) && (System.currentTimeMillis() - lastUpdate > 10000)) {
z = l[i].substring(p + 1).trim(); initRequest(true);
if (z.charAt(0) == '"') z = z.substring(1); lastUpdate = Long.MAX_VALUE;
if (z.charAt(z.length() - 1) == '"') z = z.substring(0, z.length() - 1); return;
map.put(l[i].substring(0, p), z); }
for (int i = 0; i < steps; i++) {
if (myClient.available() > 0) {
String line = myClient.readStringUntil((byte) 10);
//println("Line: " + line);
if (line == null) line = ""; else line = line.trim();
if (line.startsWith("<domain")) processDomain(parseProps(line.substring(7, line.length() - 1).trim()));
if (line.startsWith("<citation")) processCitation(parseProps(line.substring(9, line.length() - 2).trim()));
lastUpdate = System.currentTimeMillis();
} else {
initTime = false;
} }
} }
return map;
} }
public void processDomain(HashMap props) { public void processDomain(HashMap props) {
//println("Domain: " + props.toString()); //println("Domain: " + props.toString());
parsingHostName = (String) props.get("host"); if (parsingHostName == null) parsingHostName = ""; parsingHostName = (String) props.get("host"); if (parsingHostName == null) parsingHostName = "";
parsingHostID = (String) props.get("id"); if (parsingHostID == null) parsingHostID = ""; parsingHostID = (String) props.get("id"); if (parsingHostID == null) parsingHostID = "";
host h = new host(parsingHostName, physics.makeParticle(1.0f, random(0, EDGE_LENGTH * 30), random(- EDGE_LENGTH * 2, EDGE_LENGTH * 2), 0)); host h = (host) nodes.get(parsingHostID);
if (h != null) {
h.time = System.currentTimeMillis();
return;
}
h = new host(parsingHostName, physics.makeParticle(1.0f, EDGE_LENGTH * 20 * cos(a), -EDGE_LENGTH * 10 * sin(a), 0));
a += TWO_PI/256.0f + TWO_PI / 2; if (a > TWO_PI) a -= TWO_PI;
nodes.put(parsingHostID, h); nodes.put(parsingHostID, h);
addAttraction(h.node); addAttraction(h.node);
} }
@ -97,113 +144,160 @@ public void processCitation(HashMap props) {
} catch (NumberFormatException e) {} } catch (NumberFormatException e) {}
// find the two nodes that have a relation // find the two nodes that have a relation
host h = (host) nodes.get(id); host h = (host) nodes.get(id);
if (h == null) return; // host is not known TODO: store these and create relation later if (h == null) {
return; /*
h = new host(host, physics.makeParticle(1.0, EDGE_LENGTH * 20 * cos(a), -EDGE_LENGTH * 10 * sin(a), 0));
a += TWO_PI/256.0 + TWO_PI / 2; if (a > TWO_PI) a -= TWO_PI;
nodes.put(id, h);
addAttraction(h.node);*/
}
h.time = System.currentTimeMillis();
host p = (host) nodes.get(parsingHostID); // this should be successful host p = (host) nodes.get(parsingHostID); // this should be successful
addRelation(h.node, p.node); // prevent that a spring is made twice
for ( int i = 0; i < physics.numberOfSprings(); ++i ) {
Spring e = physics.getSpring(i);
Particle a = e.getOneEnd();
Particle b = e.getTheOtherEnd();
if (((a == h.node) && (b == p.node)) || ((b == h.node) && (a == p.node))) return;
}
physics.makeSpring(h.node, p.node, EDGE_STRENGTH, EDGE_STRENGTH, EDGE_LENGTH );
} }
public void draw() { public HashMap parseProps(String s) {
processRequestResponse(); String[] l = s.split(" ");
HashMap map = new HashMap();
physics.tick( 1.0f ); int p;
if (physics.numberOfParticles() > 1) updateCentroid(); String z;
centroid.tick(); for (int i = 0; i < l.length; i++) {
p = l[i].indexOf("=");
if (p > 0) {
z = l[i].substring(p + 1).trim();
if (z.charAt(0) == '"') z = z.substring(1);
if (z.charAt(z.length() - 1) == '"') z = z.substring(0, z.length() - 1);
map.put(l[i].substring(0, p), z);
}
}
return map;
}
background( 0 ); public HashSet invisibleParticles() {
translate( width/2 , height/2 ); // get nodes that have no edges
scale( centroid.z() ); HashSet particles = new HashSet();
translate( -centroid.x(), -centroid.y() ); Iterator j = nodes.values().iterator();
host h;
drawNetwork(); long t = 0, n = System.currentTimeMillis();
while (j.hasNext()) {
h = (host) j.next();
t += n - h.time;
particles.add(h.node);
}
t = t / (nodes.size() + 1);
for ( int i = 0; i < physics.numberOfSprings(); ++i ) {
Spring e = physics.getSpring(i);
particles.remove(e.getOneEnd());
particles.remove(e.getTheOtherEnd());
}
// add more nodes if the number is too large
if (nodes.size() > 80) {
j = nodes.values().iterator();
while (j.hasNext()) {
h = (host) j.next();
if (n - h.time > 15000) particles.add(h.node);
if (nodes.size() - particles.size() < 80) break;
}
}
return particles;
} }
public void drawNetwork() { public void drawNetwork(HashSet invisible) {
fill( 100, 255, 100 );
// draw vertices // draw vertices
fill( 120, 255, 120 );
noStroke(); noStroke();
String name; String name;
Iterator j = nodes.values().iterator();
host h; host h;
Iterator j = nodes.values().iterator();
while (j.hasNext()) { while (j.hasNext()) {
h = (host) j.next(); h = (host) j.next();
Particle v = h.node; Particle v = h.node;
if (invisible.contains(v)) continue;
ellipse(v.position().x(), v.position().y(), NODE_SIZE, NODE_SIZE); ellipse(v.position().x(), v.position().y(), NODE_SIZE, NODE_SIZE);
name = h.name; name = h.name;
text(name, v.position().x() - (name.length() * 26 / 10), v.position().y() + 14); text(name, v.position().x() - (name.length() * 26 / 10), v.position().y() + 14);
} }
// draw center // draw center
/* //fill( 255, 0, 0 );
ellipse( center0.position().x(), center0.position().y(), NODE_SIZE * 2, NODE_SIZE * 2 ); //ellipse( center0.position().x(), center0.position().y(), NODE_SIZE * 2, NODE_SIZE * 2 );
name = "Center0"; //ellipse( center1.position().x(), center1.position().y(), NODE_SIZE * 2, NODE_SIZE * 2 );
text(name, center0.position().x() - (name.length() * 26 / 10), center0.position().y() + 14);
ellipse( center1.position().x(), center1.position().y(), NODE_SIZE * 2, NODE_SIZE * 2 );
name = "Center1";
text(name, center1.position().x() - (name.length() * 26 / 10), center1.position().y() + 14);
*/
// draw edges // draw edges
stroke( 160 ); stroke( 200 );
beginShape( LINES );
for ( int i = 0; i < physics.numberOfSprings(); ++i ) { for ( int i = 0; i < physics.numberOfSprings(); ++i ) {
Spring e = physics.getSpring( i ); Spring e = physics.getSpring( i );
Particle a = e.getOneEnd(); Particle a = e.getOneEnd();
if (invisible.contains(a)) continue;
Particle b = e.getTheOtherEnd(); Particle b = e.getTheOtherEnd();
vertex( a.position().x(), a.position().y() ); if (invisible.contains(b)) continue;
vertex( b.position().x(), b.position().y() ); line(a.position().x(), a.position().y(), b.position().x(), b.position().y());
} }
endShape();
} }
public void keyPressed() { public void keyPressed() {
if ( key == 'c' ) { if ( key == 'c' ) initializePhysics();
initializePhysics(); if ( key == 'a' ) x = Math.max(-1.0f, x - 0.1f);
return; if ( key == 'd' ) x = Math.min( 1.0f, x + 0.1f);
} if ( key == 'w' ) y = Math.max(-1.0f, y - 0.1f);
if ( key == 's' ) y = Math.min( 1.0f, y + 0.1f);
if ( key == ' ' ) { if ( key == '-' ) z = Math.max( 1.0f, z - 1.0f);
Particle p = physics.makeParticle(); if ( key == '+' ) z = Math.min(10.0f, z + 1.0f);
addRelation(p, physics.getParticle( (int) random( 0, physics.numberOfParticles()-1) )); if ( key == '0' ) { x = 0.0f; y = 0.0f; z = 1.0f; }
addAttraction(p); if ( key == 't' ) {
HashSet hs = new HashSet();
for (int i = 0; i < physics.numberOfParticles(); ++i ) {
hs.add(physics.getParticle(i));
}
for (int i = 0; i < physics.numberOfSprings(); ++i ) {
hs.remove(physics.getSpring(i).getOneEnd());
hs.remove(physics.getSpring(i).getTheOtherEnd());
}
Iterator i = hs.iterator();
while (i.hasNext()) {
((Particle) i.next()).kill();
}
return; return;
} }
} }
public void updateCentroid() { public void updateCentroid(HashSet invisible) {
float float
xMax = Float.NEGATIVE_INFINITY, xMax = Float.NEGATIVE_INFINITY,
xMin = Float.POSITIVE_INFINITY, xMin = Float.POSITIVE_INFINITY,
yMin = Float.POSITIVE_INFINITY, yMin = Float.POSITIVE_INFINITY,
yMax = Float.NEGATIVE_INFINITY; yMax = Float.NEGATIVE_INFINITY;
for ( int i = 0; i < physics.numberOfParticles(); ++i ) { for (int i = 0; i < physics.numberOfParticles(); ++i) {
Particle p = physics.getParticle( i ); Particle p = physics.getParticle(i);
if ((i >= 2) && ((p == center0) || (p == center1) || (invisible.contains(p)))) continue;
xMax = max( xMax, p.position().x() ); xMax = max( xMax, p.position().x() );
xMin = min( xMin, p.position().x() ); xMin = min( xMin, p.position().x() );
yMin = min( yMin, p.position().y() ); yMin = min( yMin, p.position().y() );
yMax = max( yMax, p.position().y() ); yMax = max( yMax, p.position().y() );
} }
float deltaX = xMax-xMin; float deltaX = xMax-xMin;
float deltaY = yMax-yMin; float deltaY = yMax-yMin;
if ( deltaY > deltaX ) centroid.setTarget(
centroid.setTarget( xMin + 0.5f*deltaX, yMin +0.5f*deltaY, height/(deltaY+50) ); xMin + (x + 1) * 0.5f * deltaX,
else yMin + (y + 1) * 0.5f * deltaY,
centroid.setTarget( xMin + 0.5f*deltaX, yMin +0.5f*deltaY, width/(deltaX+50) ); z * ((deltaY > deltaX) ? height / (deltaY + 50) : width / (deltaX + 50))
} );
public void initializePhysics() {
physics.clear();
center0 = physics.makeParticle(1.0f, 0, 0, 0);
center0.makeFixed();
center1 = physics.makeParticle(1.0f, EDGE_LENGTH * 30, 0, 0);
center1.makeFixed();
centroid.setValue( 0, 0, 1.0f );
} }
public void addAttraction(Particle p) { public void addAttraction(Particle p) {
physics.makeAttraction(center0, p, 5000.0f, 3 * EDGE_LENGTH); physics.makeAttraction(center0, p, SPACER_STRENGTH * 10000.0f, 100 * EDGE_LENGTH);
physics.makeAttraction(center1, p, 5000.0f, 3 * EDGE_LENGTH); physics.makeAttraction(center1, p, SPACER_STRENGTH * 10000.0f, 100 * EDGE_LENGTH);
// spacers // spacers
for ( int i = 0; i < physics.numberOfParticles(); ++i ) { for ( int i = 0; i < physics.numberOfParticles(); ++i ) {
@ -212,17 +306,14 @@ public void addAttraction(Particle p) {
} }
} }
public void addRelation(Particle p, Particle other) {
physics.makeSpring( p, other, EDGE_STRENGTH, EDGE_STRENGTH, EDGE_LENGTH );
//p.moveTo( q.position().x() + random( -1, 1 ), q.position().y() + random( -1, 1 ), 0 );
}
static class host { static class host {
String name; String name;
Particle node; Particle node;
long time;
public host(String name, Particle node) { public host(String name, Particle node) {
this.name = name; this.name = name;
this.node = node; this.node = node;
this.time = System.currentTimeMillis();
} }
} }

285
htroot/processing/domaingraph/applet/domaingraph.pde
Unescape View File

@ -12,77 +12,124 @@ import traer.animation.*;
import processing.net.*; import processing.net.*;
final float NODE_SIZE = 6; final float NODE_SIZE = 6;
final float EDGE_LENGTH = 50; final float EDGE_LENGTH = 30;
final float EDGE_STRENGTH = 0.01; final float EDGE_STRENGTH = 0.001;
final float SPACER_STRENGTH = 10; final float SPACER_STRENGTH = 250;
ParticleSystem physics; ParticleSystem physics;
Smoother3D centroid; Smoother3D centroid;
PFont font; PFont font;
float x = 0.0;
float y = 0.0;
float z = 1.0;
Client myClient; Client myClient;
Particle center0, center1; // two gravity centers to support object ordering for non-quadratic fields Particle center0, center1; // two gravity centers to support object ordering for non-quadratic fields
String parsingHostName = ""; String parsingHostName = "";
String parsingHostID = ""; String parsingHostID = "";
HashMap nodes = new HashMap(); // map that holds host objects HashMap nodes = new HashMap(); // map that holds host objects
String host;
int port;
float a = 0.0;
long lastUpdate = Long.MAX_VALUE;
boolean initTime = true;
void setup() { void setup() {
String[] fontList = PFont.list(); String[] fontList = PFont.list();
//println(fontList); //println(fontList);
font = createFont(fontList[0], 32); //just take any, should be mostly Arial font = createFont(fontList[0], 32); //just take any, should be mostly Arial
textFont(font, 12); textFont(font, 9);
size(660, 400); size(660, 400);
smooth(); smooth();
frameRate( 24 ); frameRate( 12 );
strokeWeight( 2 ); strokeWeight( 1 );
ellipseMode( CENTER ); ellipseMode( CENTER );
physics = new ParticleSystem( 0, 0.25 ); physics = new ParticleSystem( 0, 0.25 );
centroid = new Smoother3D( 0.8 ); centroid = new Smoother3D( 0.8 );
initializePhysics(); initializePhysics();
initRequest(); URL url = null;
try {
url = getDocumentBase();
} catch (NullPointerException e) {}
if (url == null) {
host="localhost";
port=8080;
} else {
host=url.getHost();
port=url.getPort();
}
//println("CodeBase: " + url);
//println("host: " + host);
//println("port: " + port);
initRequest(false);
} }
void initRequest() { void initializePhysics() {
myClient = new Client(this, "localhost", 8080); physics.clear();
myClient.write("GET /xml/webstructure.xml HTTP/1.1\n"); center0 = physics.makeParticle(1.0, -EDGE_LENGTH * 10, 0, 0);
myClient.write("Host: localhost\n\n"); center0.makeFixed();
center1 = physics.makeParticle(1.0, EDGE_LENGTH * 10, 0, 0);
center1.makeFixed();
centroid.setValue( 0, 0, 1.0 );
} }
void processRequestResponse() { void draw() {
if (myClient.available() > 0) { processRequestResponse(20);
String line = myClient.readStringUntil((byte) 10);
//println("Line: " + line); physics.tick( 1.0 );
if (line == null) line = ""; else line = line.trim(); HashSet invisible = invisibleParticles();
if (line.startsWith("<domain")) processDomain(parseProps(line.substring(7, line.length() - 1).trim())); if (physics.numberOfParticles() > 1) updateCentroid(invisible);
if (line.startsWith("<citation")) processCitation(parseProps(line.substring(9, line.length() - 2).trim())); centroid.tick();
}
background( 0 );
translate( width/2 , height/2 );
scale( centroid.z() );
translate( -centroid.x(), -centroid.y() );
drawNetwork(invisible);
} }
HashMap parseProps(String s) { void initRequest(boolean update) {
String[] l = s.split(" "); myClient = new Client(this, host, port);
HashMap map = new HashMap(); myClient.write((update) ? "GET /xml/webstructure.xml?latest= HTTP/1.1\n" : "GET /xml/webstructure.xml HTTP/1.1\n");
int p; myClient.write("Host: localhost\n\n");
String z; }
for (int i = 0; i < l.length; i++) {
p = l[i].indexOf("="); void processRequestResponse(int steps) {
if (p > 0) { if (((myClient == null) || (myClient.available() <= 0)) && (System.currentTimeMillis() - lastUpdate > 10000)) {
z = l[i].substring(p + 1).trim(); initRequest(true);
if (z.charAt(0) == '"') z = z.substring(1); lastUpdate = Long.MAX_VALUE;
if (z.charAt(z.length() - 1) == '"') z = z.substring(0, z.length() - 1); return;
map.put(l[i].substring(0, p), z); }
for (int i = 0; i < steps; i++) {
if (myClient.available() > 0) {
String line = myClient.readStringUntil((byte) 10);
//println("Line: " + line);
if (line == null) line = ""; else line = line.trim();
if (line.startsWith("<domain")) processDomain(parseProps(line.substring(7, line.length() - 1).trim()));
if (line.startsWith("<citation")) processCitation(parseProps(line.substring(9, line.length() - 2).trim()));
lastUpdate = System.currentTimeMillis();
} else {
initTime = false;
} }
} }
return map;
} }
void processDomain(HashMap props) { void processDomain(HashMap props) {
//println("Domain: " + props.toString()); //println("Domain: " + props.toString());
parsingHostName = (String) props.get("host"); if (parsingHostName == null) parsingHostName = ""; parsingHostName = (String) props.get("host"); if (parsingHostName == null) parsingHostName = "";
parsingHostID = (String) props.get("id"); if (parsingHostID == null) parsingHostID = ""; parsingHostID = (String) props.get("id"); if (parsingHostID == null) parsingHostID = "";
host h = new host(parsingHostName, physics.makeParticle(1.0, random(0, EDGE_LENGTH * 30), random(- EDGE_LENGTH * 2, EDGE_LENGTH * 2), 0)); host h = (host) nodes.get(parsingHostID);
if (h != null) {
h.time = System.currentTimeMillis();
return;
}
h = new host(parsingHostName, physics.makeParticle(1.0, EDGE_LENGTH * 20 * cos(a), -EDGE_LENGTH * 10 * sin(a), 0));
a += TWO_PI/256.0 + TWO_PI / 2; if (a > TWO_PI) a -= TWO_PI;
nodes.put(parsingHostID, h); nodes.put(parsingHostID, h);
addAttraction(h.node); addAttraction(h.node);
} }
@ -97,113 +144,160 @@ void processCitation(HashMap props) {
} catch (NumberFormatException e) {} } catch (NumberFormatException e) {}
// find the two nodes that have a relation // find the two nodes that have a relation
host h = (host) nodes.get(id); host h = (host) nodes.get(id);
if (h == null) return; // host is not known TODO: store these and create relation later if (h == null) {
return; /*
h = new host(host, physics.makeParticle(1.0, EDGE_LENGTH * 20 * cos(a), -EDGE_LENGTH * 10 * sin(a), 0));
a += TWO_PI/256.0 + TWO_PI / 2; if (a > TWO_PI) a -= TWO_PI;
nodes.put(id, h);
addAttraction(h.node);*/
}
h.time = System.currentTimeMillis();
host p = (host) nodes.get(parsingHostID); // this should be successful host p = (host) nodes.get(parsingHostID); // this should be successful
addRelation(h.node, p.node); // prevent that a spring is made twice
for ( int i = 0; i < physics.numberOfSprings(); ++i ) {
Spring e = physics.getSpring(i);
Particle a = e.getOneEnd();
Particle b = e.getTheOtherEnd();
if (((a == h.node) && (b == p.node)) || ((b == h.node) && (a == p.node))) return;
}
physics.makeSpring(h.node, p.node, EDGE_STRENGTH, EDGE_STRENGTH, EDGE_LENGTH );
} }
void draw() { HashMap parseProps(String s) {
processRequestResponse(); String[] l = s.split(" ");
HashMap map = new HashMap();
physics.tick( 1.0 ); int p;
if (physics.numberOfParticles() > 1) updateCentroid(); String z;
centroid.tick(); for (int i = 0; i < l.length; i++) {
p = l[i].indexOf("=");
if (p > 0) {
z = l[i].substring(p + 1).trim();
if (z.charAt(0) == '"') z = z.substring(1);
if (z.charAt(z.length() - 1) == '"') z = z.substring(0, z.length() - 1);
map.put(l[i].substring(0, p), z);
}
}
return map;
}
background( 0 ); HashSet invisibleParticles() {
translate( width/2 , height/2 ); // get nodes that have no edges
scale( centroid.z() ); HashSet particles = new HashSet();
translate( -centroid.x(), -centroid.y() ); Iterator j = nodes.values().iterator();
host h;
drawNetwork(); long t = 0, n = System.currentTimeMillis();
while (j.hasNext()) {
h = (host) j.next();
t += n - h.time;
particles.add(h.node);
}
t = t / (nodes.size() + 1);
for ( int i = 0; i < physics.numberOfSprings(); ++i ) {
Spring e = physics.getSpring(i);
particles.remove(e.getOneEnd());
particles.remove(e.getTheOtherEnd());
}
// add more nodes if the number is too large
if (nodes.size() > 80) {
j = nodes.values().iterator();
while (j.hasNext()) {
h = (host) j.next();
if (n - h.time > 15000) particles.add(h.node);
if (nodes.size() - particles.size() < 80) break;
}
}
return particles;
} }
void drawNetwork() { void drawNetwork(HashSet invisible) {
fill( 100, 255, 100 );
// draw vertices // draw vertices
fill( 120, 255, 120 );
noStroke(); noStroke();
String name; String name;
Iterator j = nodes.values().iterator();
host h; host h;
Iterator j = nodes.values().iterator();
while (j.hasNext()) { while (j.hasNext()) {
h = (host) j.next(); h = (host) j.next();
Particle v = h.node; Particle v = h.node;
if (invisible.contains(v)) continue;
ellipse(v.position().x(), v.position().y(), NODE_SIZE, NODE_SIZE); ellipse(v.position().x(), v.position().y(), NODE_SIZE, NODE_SIZE);
name = h.name; name = h.name;
text(name, v.position().x() - (name.length() * 26 / 10), v.position().y() + 14); text(name, v.position().x() - (name.length() * 26 / 10), v.position().y() + 14);
} }
// draw center // draw center
/* //fill( 255, 0, 0 );
ellipse( center0.position().x(), center0.position().y(), NODE_SIZE * 2, NODE_SIZE * 2 ); //ellipse( center0.position().x(), center0.position().y(), NODE_SIZE * 2, NODE_SIZE * 2 );
name = "Center0"; //ellipse( center1.position().x(), center1.position().y(), NODE_SIZE * 2, NODE_SIZE * 2 );
text(name, center0.position().x() - (name.length() * 26 / 10), center0.position().y() + 14);
ellipse( center1.position().x(), center1.position().y(), NODE_SIZE * 2, NODE_SIZE * 2 );
name = "Center1";
text(name, center1.position().x() - (name.length() * 26 / 10), center1.position().y() + 14);
*/
// draw edges // draw edges
stroke( 160 ); stroke( 200 );
beginShape( LINES );
for ( int i = 0; i < physics.numberOfSprings(); ++i ) { for ( int i = 0; i < physics.numberOfSprings(); ++i ) {
Spring e = physics.getSpring( i ); Spring e = physics.getSpring( i );
Particle a = e.getOneEnd(); Particle a = e.getOneEnd();
if (invisible.contains(a)) continue;
Particle b = e.getTheOtherEnd(); Particle b = e.getTheOtherEnd();
vertex( a.position().x(), a.position().y() ); if (invisible.contains(b)) continue;
vertex( b.position().x(), b.position().y() ); line(a.position().x(), a.position().y(), b.position().x(), b.position().y());
} }
endShape();
} }
void keyPressed() { void keyPressed() {
if ( key == 'c' ) { if ( key == 'c' ) initializePhysics();
initializePhysics(); if ( key == 'a' ) x = Math.max(-1.0, x - 0.1);
return; if ( key == 'd' ) x = Math.min( 1.0, x + 0.1);
} if ( key == 'w' ) y = Math.max(-1.0, y - 0.1);
if ( key == 's' ) y = Math.min( 1.0, y + 0.1);
if ( key == ' ' ) { if ( key == '-' ) z = Math.max( 1.0, z - 1.0);
Particle p = physics.makeParticle(); if ( key == '+' ) z = Math.min(10.0, z + 1.0);
addRelation(p, physics.getParticle( (int) random( 0, physics.numberOfParticles()-1) )); if ( key == '0' ) { x = 0.0; y = 0.0; z = 1.0; }
addAttraction(p); if ( key == 't' ) {
HashSet hs = new HashSet();
for (int i = 0; i < physics.numberOfParticles(); ++i ) {
hs.add(physics.getParticle(i));
}
for (int i = 0; i < physics.numberOfSprings(); ++i ) {
hs.remove(physics.getSpring(i).getOneEnd());
hs.remove(physics.getSpring(i).getTheOtherEnd());
}
Iterator i = hs.iterator();
while (i.hasNext()) {
((Particle) i.next()).kill();
}
return; return;
} }
} }
void updateCentroid() { void updateCentroid(HashSet invisible) {
float float
xMax = Float.NEGATIVE_INFINITY, xMax = Float.NEGATIVE_INFINITY,
xMin = Float.POSITIVE_INFINITY, xMin = Float.POSITIVE_INFINITY,
yMin = Float.POSITIVE_INFINITY, yMin = Float.POSITIVE_INFINITY,
yMax = Float.NEGATIVE_INFINITY; yMax = Float.NEGATIVE_INFINITY;
for ( int i = 0; i < physics.numberOfParticles(); ++i ) { for (int i = 0; i < physics.numberOfParticles(); ++i) {
Particle p = physics.getParticle( i ); Particle p = physics.getParticle(i);
if ((i >= 2) && ((p == center0) || (p == center1) || (invisible.contains(p)))) continue;
xMax = max( xMax, p.position().x() ); xMax = max( xMax, p.position().x() );
xMin = min( xMin, p.position().x() ); xMin = min( xMin, p.position().x() );
yMin = min( yMin, p.position().y() ); yMin = min( yMin, p.position().y() );
yMax = max( yMax, p.position().y() ); yMax = max( yMax, p.position().y() );
} }
float deltaX = xMax-xMin; float deltaX = xMax-xMin;
float deltaY = yMax-yMin; float deltaY = yMax-yMin;
if ( deltaY > deltaX ) centroid.setTarget(
centroid.setTarget( xMin + 0.5*deltaX, yMin +0.5*deltaY, height/(deltaY+50) ); xMin + (x + 1) * 0.5 * deltaX,
else yMin + (y + 1) * 0.5 * deltaY,
centroid.setTarget( xMin + 0.5*deltaX, yMin +0.5*deltaY, width/(deltaX+50) ); z * ((deltaY > deltaX) ? height / (deltaY + 50) : width / (deltaX + 50))
} );
void initializePhysics() {
physics.clear();
center0 = physics.makeParticle(1.0, 0, 0, 0);
center0.makeFixed();
center1 = physics.makeParticle(1.0, EDGE_LENGTH * 30, 0, 0);
center1.makeFixed();
centroid.setValue( 0, 0, 1.0 );
} }
void addAttraction(Particle p) { void addAttraction(Particle p) {
physics.makeAttraction(center0, p, 5000.0, 3 * EDGE_LENGTH); physics.makeAttraction(center0, p, SPACER_STRENGTH * 10000.0, 100 * EDGE_LENGTH);
physics.makeAttraction(center1, p, 5000.0, 3 * EDGE_LENGTH); physics.makeAttraction(center1, p, SPACER_STRENGTH * 10000.0, 100 * EDGE_LENGTH);
// spacers // spacers
for ( int i = 0; i < physics.numberOfParticles(); ++i ) { for ( int i = 0; i < physics.numberOfParticles(); ++i ) {
@ -212,16 +306,13 @@ void addAttraction(Particle p) {
} }
} }
void addRelation(Particle p, Particle other) {
physics.makeSpring( p, other, EDGE_STRENGTH, EDGE_STRENGTH, EDGE_LENGTH );
//p.moveTo( q.position().x() + random( -1, 1 ), q.position().y() + random( -1, 1 ), 0 );
}
static class host { static class host {
String name; String name;
Particle node; Particle node;
long time;
public host(String name, Particle node) { public host(String name, Particle node) {
this.name = name; this.name = name;
this.node = node; this.node = node;
this.time = System.currentTimeMillis();
} }
} }

275
htroot/processing/domaingraph/domaingraph.pde
Unescape View File

@ -12,77 +12,124 @@ import traer.animation.*;
import processing.net.*; import processing.net.*;
final float NODE_SIZE = 6; final float NODE_SIZE = 6;
final float EDGE_LENGTH = 50; final float EDGE_LENGTH = 30;
final float EDGE_STRENGTH = 0.001; final float EDGE_STRENGTH = 0.001;
final float SPACER_STRENGTH = 500; final float SPACER_STRENGTH = 250;
ParticleSystem physics; ParticleSystem physics;
Smoother3D centroid; Smoother3D centroid;
PFont font; PFont font;
float x = 0.0;
float y = 0.0;
float z = 1.0;
Client myClient; Client myClient;
Particle center0, center1; // two gravity centers to support object ordering for non-quadratic fields Particle center0, center1; // two gravity centers to support object ordering for non-quadratic fields
String parsingHostName = ""; String parsingHostName = "";
String parsingHostID = ""; String parsingHostID = "";
HashMap nodes = new HashMap(); // map that holds host objects HashMap nodes = new HashMap(); // map that holds host objects
String host;
int port;
float a = 0.0;
long lastUpdate = Long.MAX_VALUE;
boolean initTime = true;
void setup() { void setup() {
String[] fontList = PFont.list(); String[] fontList = PFont.list();
//println(fontList); //println(fontList);
font = createFont(fontList[0], 32); //just take any, should be mostly Arial font = createFont(fontList[0], 32); //just take any, should be mostly Arial
textFont(font, 12); textFont(font, 9);
size(660, 400); size(660, 400);
smooth(); smooth();
frameRate( 24 ); frameRate( 12 );
strokeWeight( 2 ); strokeWeight( 1 );
ellipseMode( CENTER ); ellipseMode( CENTER );
physics = new ParticleSystem( 0, 0.25 ); physics = new ParticleSystem( 0, 0.25 );
centroid = new Smoother3D( 0.8 ); centroid = new Smoother3D( 0.8 );
initializePhysics(); initializePhysics();
initRequest(); URL url = null;
try {
url = getDocumentBase();
} catch (NullPointerException e) {}
if (url == null) {
host="localhost";
port=8080;
} else {
host=url.getHost();
port=url.getPort();
}
//println("CodeBase: " + url);
//println("host: " + host);
//println("port: " + port);
initRequest(false);
} }
void initRequest() { void initializePhysics() {
myClient = new Client(this, "localhost", 8080); physics.clear();
myClient.write("GET /xml/webstructure.xml HTTP/1.1\n"); center0 = physics.makeParticle(1.0, -EDGE_LENGTH * 10, 0, 0);
myClient.write("Host: localhost\n\n"); center0.makeFixed();
center1 = physics.makeParticle(1.0, EDGE_LENGTH * 10, 0, 0);
center1.makeFixed();
centroid.setValue( 0, 0, 1.0 );
} }
void processRequestResponse() { void draw() {
if (myClient.available() > 0) { processRequestResponse(20);
String line = myClient.readStringUntil((byte) 10);
//println("Line: " + line); physics.tick( 1.0 );
if (line == null) line = ""; else line = line.trim(); HashSet invisible = invisibleParticles();
if (line.startsWith("<domain")) processDomain(parseProps(line.substring(7, line.length() - 1).trim())); if (physics.numberOfParticles() > 1) updateCentroid(invisible);
if (line.startsWith("<citation")) processCitation(parseProps(line.substring(9, line.length() - 2).trim())); centroid.tick();
}
background( 0 );
translate( width/2 , height/2 );
scale( centroid.z() );
translate( -centroid.x(), -centroid.y() );
drawNetwork(invisible);
} }
HashMap parseProps(String s) { void initRequest(boolean update) {
String[] l = s.split(" "); myClient = new Client(this, host, port);
HashMap map = new HashMap(); myClient.write((update) ? "GET /xml/webstructure.xml?latest= HTTP/1.1\n" : "GET /xml/webstructure.xml HTTP/1.1\n");
int p; myClient.write("Host: localhost\n\n");
String z; }
for (int i = 0; i < l.length; i++) {
p = l[i].indexOf("="); void processRequestResponse(int steps) {
if (p > 0) { if (((myClient == null) || (myClient.available() <= 0)) && (System.currentTimeMillis() - lastUpdate > 10000)) {
z = l[i].substring(p + 1).trim(); initRequest(true);
if (z.charAt(0) == '"') z = z.substring(1); lastUpdate = Long.MAX_VALUE;
if (z.charAt(z.length() - 1) == '"') z = z.substring(0, z.length() - 1); return;
map.put(l[i].substring(0, p), z); }
for (int i = 0; i < steps; i++) {
if (myClient.available() > 0) {
String line = myClient.readStringUntil((byte) 10);
//println("Line: " + line);
if (line == null) line = ""; else line = line.trim();
if (line.startsWith("<domain")) processDomain(parseProps(line.substring(7, line.length() - 1).trim()));
if (line.startsWith("<citation")) processCitation(parseProps(line.substring(9, line.length() - 2).trim()));
lastUpdate = System.currentTimeMillis();
} else {
initTime = false;
} }
} }
return map;
} }
void processDomain(HashMap props) { void processDomain(HashMap props) {
//println("Domain: " + props.toString()); //println("Domain: " + props.toString());
parsingHostName = (String) props.get("host"); if (parsingHostName == null) parsingHostName = ""; parsingHostName = (String) props.get("host"); if (parsingHostName == null) parsingHostName = "";
parsingHostID = (String) props.get("id"); if (parsingHostID == null) parsingHostID = ""; parsingHostID = (String) props.get("id"); if (parsingHostID == null) parsingHostID = "";
host h = new host(parsingHostName, physics.makeParticle(1.0, random(0, EDGE_LENGTH * 30), random(- EDGE_LENGTH * 2, EDGE_LENGTH * 2), 0)); host h = (host) nodes.get(parsingHostID);
if (h != null) {
h.time = System.currentTimeMillis();
return;
}
h = new host(parsingHostName, physics.makeParticle(1.0, EDGE_LENGTH * 20 * cos(a), -EDGE_LENGTH * 10 * sin(a), 0));
a += TWO_PI/256.0 + TWO_PI / 2; if (a > TWO_PI) a -= TWO_PI;
nodes.put(parsingHostID, h); nodes.put(parsingHostID, h);
addAttraction(h.node); addAttraction(h.node);
} }
@ -97,72 +144,116 @@ void processCitation(HashMap props) {
} catch (NumberFormatException e) {} } catch (NumberFormatException e) {}
// find the two nodes that have a relation // find the two nodes that have a relation
host h = (host) nodes.get(id); host h = (host) nodes.get(id);
if (h == null) return; // host is not known TODO: store these and create relation later if (h == null) {
return; /*
h = new host(host, physics.makeParticle(1.0, EDGE_LENGTH * 20 * cos(a), -EDGE_LENGTH * 10 * sin(a), 0));
a += TWO_PI/256.0 + TWO_PI / 2; if (a > TWO_PI) a -= TWO_PI;
nodes.put(id, h);
addAttraction(h.node);*/
}
h.time = System.currentTimeMillis();
host p = (host) nodes.get(parsingHostID); // this should be successful host p = (host) nodes.get(parsingHostID); // this should be successful
addRelation(h.node, p.node); // prevent that a spring is made twice
for ( int i = 0; i < physics.numberOfSprings(); ++i ) {
Spring e = physics.getSpring(i);
Particle a = e.getOneEnd();
Particle b = e.getTheOtherEnd();
if (((a == h.node) && (b == p.node)) || ((b == h.node) && (a == p.node))) return;
}
physics.makeSpring(h.node, p.node, EDGE_STRENGTH, EDGE_STRENGTH, EDGE_LENGTH );
} }
void draw() { HashMap parseProps(String s) {
processRequestResponse(); String[] l = s.split(" ");
HashMap map = new HashMap();
physics.tick( 1.0 ); int p;
if (physics.numberOfParticles() > 1) updateCentroid(); String z;
centroid.tick(); for (int i = 0; i < l.length; i++) {
p = l[i].indexOf("=");
if (p > 0) {
z = l[i].substring(p + 1).trim();
if (z.charAt(0) == '"') z = z.substring(1);
if (z.charAt(z.length() - 1) == '"') z = z.substring(0, z.length() - 1);
map.put(l[i].substring(0, p), z);
}
}
return map;
}
background( 0 ); HashSet invisibleParticles() {
translate( width/2 , height/2 ); // get nodes that have no edges
scale( centroid.z() ); HashSet particles = new HashSet();
translate( -centroid.x(), -centroid.y() ); Iterator j = nodes.values().iterator();
host h;
drawNetwork(); long t = 0, n = System.currentTimeMillis();
while (j.hasNext()) {
h = (host) j.next();
t += n - h.time;
particles.add(h.node);
}
t = t / (nodes.size() + 1);
for ( int i = 0; i < physics.numberOfSprings(); ++i ) {
Spring e = physics.getSpring(i);
particles.remove(e.getOneEnd());
particles.remove(e.getTheOtherEnd());
}
// add more nodes if the number is too large
if (nodes.size() > 80) {
j = nodes.values().iterator();
while (j.hasNext()) {
h = (host) j.next();
if (n - h.time > 15000) particles.add(h.node);
if (nodes.size() - particles.size() < 80) break;
}
}
return particles;
} }
void drawNetwork() { void drawNetwork(HashSet invisible) {
fill( 100, 255, 100 );
// draw vertices // draw vertices
fill( 120, 255, 120 );
noStroke(); noStroke();
String name; String name;
Iterator j = nodes.values().iterator();
host h; host h;
Iterator j = nodes.values().iterator();
while (j.hasNext()) { while (j.hasNext()) {
h = (host) j.next(); h = (host) j.next();
Particle v = h.node; Particle v = h.node;
if (invisible.contains(v)) continue;
ellipse(v.position().x(), v.position().y(), NODE_SIZE, NODE_SIZE); ellipse(v.position().x(), v.position().y(), NODE_SIZE, NODE_SIZE);
name = h.name; name = h.name;
text(name, v.position().x() - (name.length() * 26 / 10), v.position().y() + 14); text(name, v.position().x() - (name.length() * 26 / 10), v.position().y() + 14);
} }
// draw center // draw center
/* //fill( 255, 0, 0 );
ellipse( center0.position().x(), center0.position().y(), NODE_SIZE * 2, NODE_SIZE * 2 ); //ellipse( center0.position().x(), center0.position().y(), NODE_SIZE * 2, NODE_SIZE * 2 );
name = "Center0"; //ellipse( center1.position().x(), center1.position().y(), NODE_SIZE * 2, NODE_SIZE * 2 );
text(name, center0.position().x() - (name.length() * 26 / 10), center0.position().y() + 14);
ellipse( center1.position().x(), center1.position().y(), NODE_SIZE * 2, NODE_SIZE * 2 );
name = "Center1";
text(name, center1.position().x() - (name.length() * 26 / 10), center1.position().y() + 14);
*/
// draw edges // draw edges
stroke( 160 ); stroke( 200 );
beginShape( LINES );
for ( int i = 0; i < physics.numberOfSprings(); ++i ) { for ( int i = 0; i < physics.numberOfSprings(); ++i ) {
Spring e = physics.getSpring( i ); Spring e = physics.getSpring( i );
Particle a = e.getOneEnd(); Particle a = e.getOneEnd();
if (invisible.contains(a)) continue;
Particle b = e.getTheOtherEnd(); Particle b = e.getTheOtherEnd();
vertex( a.position().x(), a.position().y() ); if (invisible.contains(b)) continue;
vertex( b.position().x(), b.position().y() ); line(a.position().x(), a.position().y(), b.position().x(), b.position().y());
} }
endShape();
} }
void keyPressed() { void keyPressed() {
if ( key == 'c' ) { if ( key == 'c' ) initializePhysics();
initializePhysics(); if ( key == 'a' ) x = Math.max(-1.0, x - 0.1);
return; if ( key == 'd' ) x = Math.min( 1.0, x + 0.1);
} if ( key == 'w' ) y = Math.max(-1.0, y - 0.1);
if ( key == 's' ) y = Math.min( 1.0, y + 0.1);
if ( key == 'd' ) { if ( key == '-' ) z = Math.max( 1.0, z - 1.0);
if ( key == '+' ) z = Math.min(10.0, z + 1.0);
if ( key == '0' ) { x = 0.0; y = 0.0; z = 1.0; }
if ( key == 't' ) {
HashSet hs = new HashSet(); HashSet hs = new HashSet();
for (int i = 0; i < physics.numberOfParticles(); ++i ) { for (int i = 0; i < physics.numberOfParticles(); ++i ) {
hs.add(physics.getParticle(i)); hs.add(physics.getParticle(i));
@ -177,49 +268,36 @@ void keyPressed() {
} }
return; return;
} }
if ( key == ' ' ) {
Particle p = physics.makeParticle();
addRelation(p, physics.getParticle( (int) random( 0, physics.numberOfParticles()-1) ));
addAttraction(p);
return;
}
} }
void updateCentroid() { void updateCentroid(HashSet invisible) {
float float
xMax = Float.NEGATIVE_INFINITY, xMax = Float.NEGATIVE_INFINITY,
xMin = Float.POSITIVE_INFINITY, xMin = Float.POSITIVE_INFINITY,
yMin = Float.POSITIVE_INFINITY, yMin = Float.POSITIVE_INFINITY,
yMax = Float.NEGATIVE_INFINITY; yMax = Float.NEGATIVE_INFINITY;
for ( int i = 0; i < physics.numberOfParticles(); ++i ) { for (int i = 0; i < physics.numberOfParticles(); ++i) {
Particle p = physics.getParticle( i ); Particle p = physics.getParticle(i);
if ((i >= 2) && ((p == center0) || (p == center1) || (invisible.contains(p)))) continue;
xMax = max( xMax, p.position().x() ); xMax = max( xMax, p.position().x() );
xMin = min( xMin, p.position().x() ); xMin = min( xMin, p.position().x() );
yMin = min( yMin, p.position().y() ); yMin = min( yMin, p.position().y() );
yMax = max( yMax, p.position().y() ); yMax = max( yMax, p.position().y() );
} }
float deltaX = xMax-xMin; float deltaX = xMax-xMin;
float deltaY = yMax-yMin; float deltaY = yMax-yMin;
if ( deltaY > deltaX ) centroid.setTarget(
centroid.setTarget( xMin + 0.5*deltaX, yMin +0.5*deltaY, height/(deltaY+50) ); xMin + (x + 1) * 0.5 * deltaX,
else yMin + (y + 1) * 0.5 * deltaY,
centroid.setTarget( xMin + 0.5*deltaX, yMin +0.5*deltaY, width/(deltaX+50) ); z * ((deltaY > deltaX) ? height / (deltaY + 50) : width / (deltaX + 50))
} );
void initializePhysics() {
physics.clear();
center0 = physics.makeParticle(1.0, 0, 0, 0);
center0.makeFixed();
center1 = physics.makeParticle(1.0, EDGE_LENGTH * 30, 0, 0);
center1.makeFixed();
centroid.setValue( 0, 0, 1.0 );
} }
void addAttraction(Particle p) { void addAttraction(Particle p) {
physics.makeAttraction(center0, p, 5000.0, 3 * EDGE_LENGTH); physics.makeAttraction(center0, p, SPACER_STRENGTH * 10000.0, 100 * EDGE_LENGTH);
physics.makeAttraction(center1, p, 5000.0, 3 * EDGE_LENGTH); physics.makeAttraction(center1, p, SPACER_STRENGTH * 10000.0, 100 * EDGE_LENGTH);
// spacers // spacers
for ( int i = 0; i < physics.numberOfParticles(); ++i ) { for ( int i = 0; i < physics.numberOfParticles(); ++i ) {
@ -228,16 +306,13 @@ void addAttraction(Particle p) {
} }
} }
void addRelation(Particle p, Particle other) {
physics.makeSpring( p, other, EDGE_STRENGTH, EDGE_STRENGTH, EDGE_LENGTH );
//p.moveTo( q.position().x() + random( -1, 1 ), q.position().y() + random( -1, 1 ), 0 );
}
static class host { static class host {
String name; String name;
Particle node; Particle node;
long time;
public host(String name, Particle node) { public host(String name, Particle node) {
this.name = name; this.name = name;
this.node = node; this.node = node;
this.time = System.currentTimeMillis();
} }
} }

4
htroot/xml/webstructure.java
Unescape View File

@ -41,7 +41,8 @@ public class webstructure {
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch<?> env) { public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch<?> env) {
serverObjects prop = new serverObjects(); serverObjects prop = new serverObjects();
plasmaSwitchboard sb = (plasmaSwitchboard) env; plasmaSwitchboard sb = (plasmaSwitchboard) env;
Iterator<plasmaWebStructure.structureEntry> i = sb.webStructure.structureEntryIterator(); boolean latest = ((post == null) ? false : post.containsKey("latest"));
Iterator<plasmaWebStructure.structureEntry> i = sb.webStructure.structureEntryIterator(latest);
int c = 0, d; int c = 0, d;
plasmaWebStructure.structureEntry sentry; plasmaWebStructure.structureEntry sentry;
Map.Entry<String, Integer> refentry; Map.Entry<String, Integer> refentry;
@ -71,6 +72,7 @@ public class webstructure {
} }
prop.put("domains", c); prop.put("domains", c);
prop.put("maxref", plasmaWebStructure.maxref); prop.put("maxref", plasmaWebStructure.maxref);
if (latest) sb.webStructure.joinOldNew();
// return rewrite properties // return rewrite properties
return prop; return prop;

2
source/de/anomic/crawler/NoticeURLImporter.java
Unescape View File

@ -21,7 +21,7 @@ public class NoticeURLImporter extends AbstractImporter implements Importer {
private CrawlProfile activeCrawls; private CrawlProfile activeCrawls;
private ImporterManager dbImportManager; private ImporterManager dbImportManager;
public NoticeURLImporter(File crawlerPath, CrawlQueues crawlQueues, CrawlProfile activeCrawls, ImporterManager dbImportManager) throws ImporterException { public NoticeURLImporter(File crawlerPath, CrawlQueues crawlQueues, CrawlProfile activeCrawls, ImporterManager dbImportManager) {
super("NURL"); super("NURL");
this.crawlQueues = crawlQueues; this.crawlQueues = crawlQueues;
this.activeCrawls = activeCrawls; this.activeCrawls = activeCrawls;

2
source/de/anomic/plasma/plasmaSwitchboardQueue.java
Unescape View File

@ -392,7 +392,7 @@ public class plasmaSwitchboardQueue {
} }
public String referrerHash() { public String referrerHash() {
return referrerHash; return (referrerHash == null) ? "" : referrerHash;
} }
public String anchorName() { public String anchorName() {

145
source/de/anomic/plasma/plasmaWebStructure.java
Unescape View File

@ -54,7 +54,7 @@ public class plasmaWebStructure {
private serverLog log; private serverLog log;
private File rankingPath, structureFile; private File rankingPath, structureFile;
private String crlFile, crgFile; private String crlFile, crgFile;
TreeMap<String, String> structure; // <b64hash(6)>','<host> to <date-yyyymmdd(8)>{<target-b64hash(6)><target-count-hex(4)>}* TreeMap<String, String> structure_old, structure_new; // <b64hash(6)>','<host> to <date-yyyymmdd(8)>{<target-b64hash(6)><target-count-hex(4)>}*
public plasmaWebStructure(serverLog log, File rankingPath, String crlFile, String crgFile, File structureFile) { public plasmaWebStructure(serverLog log, File rankingPath, String crlFile, String crgFile, File structureFile) {
this.log = log; this.log = log;
@ -62,27 +62,28 @@ public class plasmaWebStructure {
this.crlFile = crlFile; this.crlFile = crlFile;
this.crgFile = crgFile; this.crgFile = crgFile;
this.crg = new StringBuffer(maxCRGDump); this.crg = new StringBuffer(maxCRGDump);
this.structure = new TreeMap<String, String>(); this.structure_old = new TreeMap<String, String>();
this.structure_new = new TreeMap<String, String>();
this.structureFile = structureFile; this.structureFile = structureFile;
// load web structure // load web structure
Map<String, String> loadedStructure = (this.structureFile.exists()) ? serverFileUtils.loadHashMap(this.structureFile) : new TreeMap<String, String>(); Map<String, String> loadedStructure = (this.structureFile.exists()) ? serverFileUtils.loadHashMap(this.structureFile) : new TreeMap<String, String>();
if (loadedStructure != null) this.structure.putAll(loadedStructure); if (loadedStructure != null) this.structure_old.putAll(loadedStructure);
// delete outdated entries in case the structure is too big // delete out-dated entries in case the structure is too big
if (this.structure.size() > maxhosts) { if (this.structure_old.size() > maxhosts) {
// fill a set with last-modified - dates of the structure // fill a set with last-modified - dates of the structure
TreeSet<String> delset = new TreeSet<String>(); TreeSet<String> delset = new TreeSet<String>();
String key, value; String key, value;
for (Map.Entry<String, String> entry : this.structure.entrySet()) { for (Map.Entry<String, String> entry : this.structure_old.entrySet()) {
key = entry.getKey(); key = entry.getKey();
value = entry.getValue(); value = entry.getValue();
delset.add(value.substring(0, 8) + key); delset.add(value.substring(0, 8) + key);
} }
int delcount = this.structure.size() - (maxhosts * 9 / 10); int delcount = this.structure_old.size() - (maxhosts * 9 / 10);
Iterator<String> j = delset.iterator(); Iterator<String> j = delset.iterator();
while ((delcount > 0) && (j.hasNext())) { while ((delcount > 0) && (j.hasNext())) {
this.structure.remove(j.next().substring(8)); this.structure_old.remove(j.next().substring(8));
delcount--; delcount--;
} }
} }
@ -216,46 +217,78 @@ public class plasmaWebStructure {
public Map<String, Integer> references(String domhash) { public Map<String, Integer> references(String domhash) {
// returns a map with a domhash(String):refcount(Integer) relation // returns a map with a domhash(String):refcount(Integer) relation
assert domhash.length() == 6; assert domhash.length() == 6;
synchronized(structure) { SortedMap<String, String> tailMap;
SortedMap<String, String> tailMap = structure.tailMap(domhash); Map<String, Integer> h = new HashMap<String, Integer>();
if ((tailMap == null) || (tailMap.isEmpty())) return new HashMap<String, Integer>(); synchronized (structure_old) {
String key = tailMap.firstKey(); tailMap = structure_old.tailMap(domhash);
if (key.startsWith(domhash)) { if (!tailMap.isEmpty()) {
return refstr2map(tailMap.get(key)); String key = tailMap.firstKey();
} else { if (key.startsWith(domhash)) {
return new HashMap<String, Integer>(); h = refstr2map(tailMap.get(key));
}
}
}
synchronized (structure_new) {
tailMap = structure_new.tailMap(domhash);
if (!tailMap.isEmpty()) {
String key = tailMap.firstKey();
if (key.startsWith(domhash)) {
h.putAll(refstr2map(tailMap.get(key)));
}
} }
} }
return h;
} }
public int referencesCount(String domhash) { public int referencesCount(String domhash) {
// returns the number of domains that are referenced by this domhash // returns the number of domains that are referenced by this domhash
assert domhash.length() == 6 : "domhash = " + domhash; assert domhash.length() == 6 : "domhash = " + domhash;
synchronized(structure) { SortedMap<String, String> tailMap;
SortedMap<String, String> tailMap = structure.tailMap(domhash); int c = 0;
if ((tailMap == null) || (tailMap.isEmpty())) return 0; synchronized (structure_old) {
String key = tailMap.firstKey(); tailMap = structure_old.tailMap(domhash);
if (key.startsWith(domhash)) { if (!tailMap.isEmpty()) {
return refstr2count(tailMap.get(key)); String key = tailMap.firstKey();
} else { if (key.startsWith(domhash)) {
return 0; c = refstr2count(tailMap.get(key));
}
}
}
synchronized (structure_new) {
tailMap = structure_new.tailMap(domhash);
if (!tailMap.isEmpty()) {
String key = tailMap.firstKey();
if (key.startsWith(domhash)) {
c += refstr2count(tailMap.get(key));
}
} }
} }
return c;
} }
public String resolveDomHash2DomString(String domhash) { public String resolveDomHash2DomString(String domhash) {
// returns the domain as string, null if unknown // returns the domain as string, null if unknown
assert domhash.length() == 6; assert domhash.length() == 6;
synchronized(structure) { SortedMap<String, String> tailMap;
SortedMap<String, String> tailMap = structure.tailMap(domhash); synchronized(structure_old) {
if ((tailMap == null) || (tailMap.isEmpty())) return null; tailMap = structure_old.tailMap(domhash);
String key = tailMap.firstKey(); if (!tailMap.isEmpty()) {
if (key.startsWith(domhash)) { String key = tailMap.firstKey();
return key.substring(7); if (key.startsWith(domhash)) {
} else { return key.substring(7);
return null; }
} }
} }
synchronized(structure_new) {
tailMap = structure_new.tailMap(domhash);
if (!tailMap.isEmpty()) {
String key = tailMap.firstKey();
if (key.startsWith(domhash)) {
return key.substring(7);
}
}
}
return null;
} }
private void learn(yacyURL url, StringBuffer reference /*string of b64(12digits)-hashes*/) { private void learn(yacyURL url, StringBuffer reference /*string of b64(12digits)-hashes*/) {
@ -297,15 +330,42 @@ public class plasmaWebStructure {
} }
// store the map back to the structure // store the map back to the structure
synchronized(structure) { synchronized(structure_new) {
structure.put(domhash + "," + url.getHost(), map2refstr(refs)); structure_new.put(domhash + "," + url.getHost(), map2refstr(refs));
}
}
private static final void joinStructure(TreeMap<String, String> into, TreeMap<String, String> from) {
for (Map.Entry<String, String> e: from.entrySet()) {
if (into.containsKey(e.getKey())) {
Map<String, Integer> s0 = refstr2map(into.get(e.getKey()));
Map<String, Integer> s1 = refstr2map(e.getValue());
for (Map.Entry<String, Integer> r: s1.entrySet()) {
if (s0.containsKey(r.getKey())) {
s0.put(r.getKey(), s0.get(r.getKey()).intValue() + r.getValue().intValue());
} else {
s0.put(r.getKey(), r.getValue().intValue());
}
}
into.put(e.getKey(), map2refstr(s0));
} else {
into.put(e.getKey(), e.getValue());
}
}
}
public void joinOldNew() {
synchronized(structure_new) {
joinStructure(this.structure_old, this.structure_new);
this.structure_new.clear();
} }
} }
public void saveWebStructure() { public void saveWebStructure() {
joinOldNew();
try { try {
synchronized(structure) { synchronized(structure_old) {
serverFileUtils.saveMap(this.structureFile, this.structure, "Web Structure Syntax: <b64hash(6)>','<host> to <date-yyyymmdd(8)>{<target-b64hash(6)><target-count-hex(4)>}*"); serverFileUtils.saveMap(this.structureFile, this.structure_old, "Web Structure Syntax: <b64hash(6)>','<host> to <date-yyyymmdd(8)>{<target-b64hash(6)><target-count-hex(4)>}*");
} }
} catch (IOException e) { } catch (IOException e) {
e.printStackTrace(); e.printStackTrace();
@ -316,8 +376,9 @@ public class plasmaWebStructure {
// find domain with most references // find domain with most references
String maxhost = null; String maxhost = null;
int refsize, maxref = 0; int refsize, maxref = 0;
synchronized(structure) { joinOldNew();
for (Map.Entry<String, String> entry : structure.entrySet()) { synchronized(structure_new) {
for (Map.Entry<String, String> entry : structure_old.entrySet()) {
refsize = entry.getValue().length(); refsize = entry.getValue().length();
if (refsize > maxref) { if (refsize > maxref) {
maxref = refsize; maxref = refsize;
@ -328,9 +389,9 @@ public class plasmaWebStructure {
return maxhost; return maxhost;
} }
public Iterator<structureEntry> structureEntryIterator() { public Iterator<structureEntry> structureEntryIterator(boolean latest) {
// iterates objects of type structureEntry // iterates objects of type structureEntry
return new structureIterator(); return new structureIterator(latest);
} }
public class structureIterator implements Iterator<structureEntry> { public class structureIterator implements Iterator<structureEntry> {
@ -338,8 +399,8 @@ public class plasmaWebStructure {
private Iterator<Map.Entry<String, String>> i; private Iterator<Map.Entry<String, String>> i;
private structureEntry nextentry; private structureEntry nextentry;
public structureIterator() { public structureIterator(boolean latest) {
i = structure.entrySet().iterator(); i = ((latest) ? structure_new : structure_old).entrySet().iterator();
next0(); next0();
} }

Write Preview
Loading…
Cancel
Save