boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

enhanced the new link structure graph

Browse Source
pull/1/head
Michael Peter Christen 11 years ago
parent 926d28dd3f
commit e8ddd415a8
4 changed files with 52 additions and 21 deletions
Show Stats Download Patch File Download Diff File

48
htroot/api/linkstructure.java
Unescape View File

@ -21,6 +21,7 @@
import java.net.MalformedURLException;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.concurrent.BlockingQueue;
@ -92,7 +93,9 @@ public class linkstructure {
);
SolrDocument doc;
Map<String, FailType> errorDocs = new HashMap<String, FailType>();
Map<String, HyperlinkEdge> edges = new HashMap<String, HyperlinkEdge>();
Map<String, HyperlinkEdge> inboundEdges = new HashMap<String, HyperlinkEdge>();
Map<String, HyperlinkEdge> outboundEdges = new HashMap<String, HyperlinkEdge>();
Map<String, HyperlinkEdge> errorEdges = new HashMap<String, HyperlinkEdge>();
try {
while ((doc = docs.take()) != AbstractSolrConnector.POISON_DOCUMENT) {
String u = (String) doc.getFieldValue(CollectionSchema.sku.getSolrFieldName());
@ -110,7 +113,7 @@ public class linkstructure {
try {
DigestURL linkurl = new DigestURL(link, null);
String edgehash = ids + ASCII.String(linkurl.hash());
edges.put(edgehash, new HyperlinkEdge(from, linkurl, HyperlinkEdge.Type.InboundOk));
inboundEdges.put(edgehash, new HyperlinkEdge(from, linkurl, HyperlinkEdge.Type.Inbound));
} catch (MalformedURLException e) {}
}
links = URIMetadataNode.getLinks(doc, false); // outbound
@ -119,26 +122,49 @@ public class linkstructure {
try {
DigestURL linkurl = new DigestURL(link, null);
String edgehash = ids + ASCII.String(linkurl.hash());
edges.put(edgehash, new HyperlinkEdge(from, linkurl, HyperlinkEdge.Type.Outbound));
outboundEdges.put(edgehash, new HyperlinkEdge(from, linkurl, HyperlinkEdge.Type.Outbound));
} catch (MalformedURLException e) {}
}
}
if (edges.size() > maxnodes) break;
if (inboundEdges.size() + outboundEdges.size() > maxnodes) break;
}
} catch (InterruptedException e) {
} catch (MalformedURLException e) {
}
// we use the errorDocs to mark all edges with endpoint to error documents
for (Map.Entry<String, HyperlinkEdge> edge: edges.entrySet()) {
if (errorDocs.containsKey(edge.getValue().target.toNormalform(true))) edge.getValue().type = HyperlinkEdge.Type.Dead;
Iterator<Map.Entry<String, HyperlinkEdge>> i = inboundEdges.entrySet().iterator();
Map.Entry<String, HyperlinkEdge> edge;
while (i.hasNext()) {
edge = i.next();
if (errorDocs.containsKey(edge.getValue().target.toNormalform(true))) {
i.remove();
edge.getValue().type = HyperlinkEdge.Type.Dead;
errorEdges.put(edge.getKey(), edge.getValue());
}
}
i = outboundEdges.entrySet().iterator();
while (i.hasNext()) {
edge = i.next();
if (errorDocs.containsKey(edge.getValue().target.toNormalform(true))) {
i.remove();
edge.getValue().type = HyperlinkEdge.Type.Dead;
errorEdges.put(edge.getKey(), edge.getValue());
}
}
// we put all edges together in a specific order which is used to create nodes in a svg display:
// notes that appear first are possible painted over by nodes coming later.
// less important nodes shall appear therefore first
Map<String, HyperlinkEdge> edges = new LinkedHashMap<String, HyperlinkEdge>();
edges.putAll(outboundEdges);
edges.putAll(inboundEdges);
edges.putAll(errorEdges);
// finally just write out the edge array
int c = 0;
for (Map.Entry<String, HyperlinkEdge> edge: edges.entrySet()) {
prop.putJSON("list_" + c + "_source", edge.getValue().source.getPath());
prop.putJSON("list_" + c + "_target", edge.getValue().type.equals(HyperlinkEdge.Type.Outbound) ? edge.getValue().target.toNormalform(true) : edge.getValue().target.getPath());
prop.putJSON("list_" + c + "_type", edge.getValue().type.name());
for (Map.Entry<String, HyperlinkEdge> e: edges.entrySet()) {
prop.putJSON("list_" + c + "_source", e.getValue().source.getPath());
prop.putJSON("list_" + c + "_target", e.getValue().type.equals(HyperlinkEdge.Type.Outbound) ? e.getValue().target.toNormalform(true) : e.getValue().target.getPath());
prop.putJSON("list_" + c + "_type", e.getValue().type.name());
prop.put("list_" + c + "_eol", 1);
c++;
}

4
htroot/env/hypertree.css vendored
Unescape View File

@ -5,7 +5,7 @@
#Dead {
fill: red;
}
#InboundOk {
#Inbound {
fill: green;
}
.hypertree {
@ -22,7 +22,7 @@
stroke: red;
stroke-dasharray: 0,2 1;
}
.hypertree-link.InboundOk {
.hypertree-link.Inbound {
stroke: green;
}
circle {

19
htroot/js/hypertree.js
Unescape View File

@ -3,13 +3,14 @@ function linkstructure(hostname, element, width, height, maxtime, maxnodes) {
var links = [];
$.getJSON("/api/linkstructure.json?about=" + hostname + "&maxtime=" + maxtime + "&maxnodes=" + maxnodes, function(links) {
links.forEach(function(link) {
link.source = nodes[link.source] || (nodes[link.source] = {name: link.source});
link.target = nodes[link.target] || (nodes[link.target] = {name: link.target});
link.source = nodes[link.source] || (nodes[link.source] = {name: link.source, type:"Inbound"});
link.target = nodes[link.target] || (nodes[link.target] = {name: link.target, type:link.type});
});
var force = d3.layout.force().nodes(d3.values(nodes)).links(links).size([width, height]).linkDistance(60).charge(-800).on("tick", tick).start();
force.gravity(0.7);
var svg = d3.select(element).append("svg").attr("id", "hypertree").attr("width", width).attr("height", height);
svg.append("defs").selectAll("marker")
.data(["Dead", "Outbound", "InboundOk"])
.data(["Dead", "Outbound", "Inbound"])
.enter().append("marker")
.attr("id", function(d) { return d; })
.attr("viewBox", "0 -5 10 10")
@ -26,11 +27,15 @@ function linkstructure(hostname, element, width, height, maxtime, maxnodes) {
svg.append("text").attr("x", 10).attr("y", height - 20).text("green: links to same domain").attr("style", "font-size:9px").attr("fill", "green");
svg.append("text").attr("x", 10).attr("y", height - 10).text("blue: links to other domains").attr("style", "font-size:9px").attr("fill", "lightblue");
svg.append("text").attr("x", 10).attr("y", height).text("red: dead links").attr("style", "font-size:9px").attr("fill", "red");
var path = svg.append("g").selectAll("path").data(force.links()).enter().append("path").attr("class",
function(d) {return "hypertree-link " + d.type; }).attr("marker-end", function(d) { return "url(#" + d.type + ")";});
var path = svg.append("g")
.selectAll("path").data(force.links()).enter().append("path")
.attr("class",function(d) {return "hypertree-link " + d.type; })
.attr("marker-end", function(d) { return "url(#" + d.type + ")";});
var circle = svg.append("g").selectAll("circle").data(force.nodes()).enter().append("circle").attr("r", 4).call(force.drag);
var text = svg.append("g").selectAll("text").data(force.nodes()).enter().append("text").attr("x", 8).attr("y", ".31em").text(
function(d) {return d.name; });
var text = svg.append("g")
.selectAll("text").data(force.nodes()).enter().append("text").attr("x", 8).attr("y", ".31em")
.attr("style", function(d) {return d.type == "Outbound" ? "fill:#888888;" : "fill:#000000;";})
.text(function(d) {return d.name;});
function tick() {
path.attr("d", linkArc);
circle.attr("transform", transform);

2
source/net/yacy/search/schema/HyperlinkEdge.java
Unescape View File

@ -25,7 +25,7 @@ import net.yacy.cora.document.id.DigestURL;
public class HyperlinkEdge {
public static enum Type {
InboundOk, InboundNofollow, Outbound, Dead;
Inbound, InboundNofollow, Outbound, Dead;
}
public DigestURL source, target;

Write Preview
Loading…
Cancel
Save