From d9766ca981088327f18b6125e5fea2f591e17fa2 Mon Sep 17 00:00:00 2001 From: luccioman Date: Mon, 16 Jan 2017 18:41:58 +0100 Subject: [PATCH] Fixed WatchWebStructure_p.html render to include https URLs. As described in mantis 721 (http://mantis.tokeek.de/view.php?id=721) WatchWebStructure_p.html failed to include in its structure view https and other protocols and ports than default http. --- htroot/WebStructurePicture_p.java | 36 ++++------ .../peers/graphics/WebStructureGraph.java | 68 ++++++++++++++++--- .../peers/graphics/WebStructureGraphTest.java | 48 +++++++++++++ 3 files changed, 120 insertions(+), 32 deletions(-) diff --git a/htroot/WebStructurePicture_p.java b/htroot/WebStructurePicture_p.java index b9f2a82a9..8efa8ace8 100644 --- a/htroot/WebStructurePicture_p.java +++ b/htroot/WebStructurePicture_p.java @@ -24,16 +24,12 @@ // along with this program; if not, write to the Free Software // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -import java.net.MalformedURLException; -import java.util.AbstractMap; -import java.util.ArrayList; import java.util.Date; -import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; -import java.util.List; import java.util.Map; +import java.util.Set; -import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.order.Base64Order; import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.sorting.ClusteredScoreMap; @@ -114,18 +110,11 @@ public class WebStructurePicture_p { String[] hostlist = CommonPattern.COMMA.split(hosts); for (int i = 0; i < hostlist.length; i++) { String host = hostlist[i]; - String hash = null; - try { - hash = new DigestURL("http://" + host).hosthash(); - } catch (final MalformedURLException e) { - continue; - } - Map.Entry centernode = new AbstractMap.SimpleEntry(hash, host); double angle = 2.0d * i * Math.PI / hostlist.length; if (hostlist.length == 3) angle -= Math.PI / 2; if (hostlist.length == 4) angle += Math.PI / 4; - graph.addNode(centernode.getValue(), Math.cos(angle) / 8, Math.sin(angle) / 8, 0); - place(graph, sb.webStructure, centernode, bf, nodes, timeout, hostlist.length == 1 ? 0 : 1, hostlist.length == 1 ? depth : depth + 1, cyc); + graph.addNode(host, Math.cos(angle) / 8, Math.sin(angle) / 8, 0); + place(graph, sb.webStructure, host, bf, nodes, timeout, hostlist.length == 1 ? 0 : 1, hostlist.length == 1 ? depth : depth + 1, cyc); } // apply physics to it to get a better shape @@ -151,19 +140,18 @@ public class WebStructurePicture_p { } private static final int place( - final GraphPlotter graph, final WebStructureGraph structure, Map.Entry pivotnode, + final GraphPlotter graph, final WebStructureGraph structure, String hostName, int bf, int maxnodes, final long timeout, int nextlayer, final int maxlayer, final int cyc) { - Point pivotpoint = graph.getNode(pivotnode.getValue()); + Point pivotpoint = graph.getNode(hostName); int branches = 0; if (nextlayer == maxlayer) return branches; nextlayer++; final double radius = 1.0 / (1 << nextlayer); - final WebStructureGraph.StructureEntry sr = structure.outgoingReferences(pivotnode.getKey()); - final Map next = (sr == null) ? new HashMap() : sr.references; + final Map next = structure.outgoingReferencesByHostName(hostName); ClusteredScoreMap next0 = new ClusteredScoreMap(false); for (Map.Entry entry: next.entrySet()) next0.set(entry.getKey(), entry.getValue()); // first set points to next hosts - final List> targets = new ArrayList>(); + final Set targetHostNames = new HashSet(); int maxtargetrefs = 8, maxthisrefs = 8; int targetrefs, thisrefs; double rr, re; @@ -176,7 +164,7 @@ public class WebStructurePicture_p { targetrefs = structure.referencesCount(targethash); // can be cpu/time-critical maxtargetrefs = Math.max(targetrefs, maxtargetrefs); maxthisrefs = Math.max(thisrefs, maxthisrefs); - targets.add(new AbstractMap.SimpleEntry(targethash, targethost)); + targetHostNames.add(targethost); if (graph.getNode(targethost) != null) continue; // set a new point. It is placed on a circle around the host point final double angle = ((Base64Order.enhancedCoder.cardinal((targethash + "____").getBytes()) / maxlongd) + (cyc / 360.0d)) * 2.0d * Math.PI; @@ -189,11 +177,11 @@ public class WebStructurePicture_p { } // recursively set next hosts int nextnodes; - for (Map.Entry target: targets) { - nextnodes = ((maxnodes <= 0) || (System.currentTimeMillis() >= timeout)) ? 0 : place(graph, structure, target, bf, maxnodes, timeout, nextlayer, maxlayer, cyc); + for (String targetHostName: targetHostNames) { + nextnodes = ((maxnodes <= 0) || (System.currentTimeMillis() >= timeout)) ? 0 : place(graph, structure, targetHostName, bf, maxnodes, timeout, nextlayer, maxlayer, cyc); branches += nextnodes; maxnodes -= nextnodes; - graph.setEdge(pivotnode.getValue(), target.getValue()); + graph.setEdge(hostName, targetHostName); } return branches; } diff --git a/source/net/yacy/peers/graphics/WebStructureGraph.java b/source/net/yacy/peers/graphics/WebStructureGraph.java index a82ce862a..20cd1163d 100644 --- a/source/net/yacy/peers/graphics/WebStructureGraph.java +++ b/source/net/yacy/peers/graphics/WebStructureGraph.java @@ -354,6 +354,37 @@ public class WebStructureGraph { return new StructureEntry(hosthash, hostname, date, h); } + /** + * Compute outgoing references from source hostName on any source protocol or port. + * @param srcHostName reference source host name + * @return outgoing references mapped from target host hash to count + */ + public Map outgoingReferencesByHostName(final String srcHostName) { + Set srcHostHashes = this.hostName2HostHashes(srcHostName); + final Map targetHashesToCount = new HashMap(); + for(String srcHostHash : srcHostHashes) { + final WebStructureGraph.StructureEntry sr = this.outgoingReferences(srcHostHash); + if(sr != null) { + for(java.util.Map.Entry ref : sr.references.entrySet()) { + Integer refsNb = targetHashesToCount.get(ref.getKey()); + if(refsNb != null) { + if(ref.getValue() != null) { + refsNb += ref.getValue(); + } + } else { + if(ref.getValue() != null) { + refsNb = ref.getValue(); + } else { + refsNb = Integer.valueOf(0); + } + } + targetHashesToCount.put(ref.getKey(), refsNb); + } + } + } + return targetHashesToCount; + } + public StructureEntry incomingReferences(final String hosthash) { final String hostname = hostHash2hostName(hosthash); if ( hostname == null ) { @@ -743,29 +774,50 @@ public class WebStructureGraph { } } + /** + * @return the host name having the most outgoing references or null when the structure is empty + */ public String hostWithMaxReferences() { // find host with most references - String maxhost = null; + Map hostNamesToRefsNb = new TreeMap<>(); int refsize, maxref = 0; + String hostName, maxHostName = null; + Integer refsNb; synchronized ( this.structure_old ) { for ( final Map.Entry entry : this.structure_old.entrySet() ) { refsize = entry.getValue().length; - if ( refsize > maxref ) { - maxref = refsize; - maxhost = entry.getKey().substring(7); + hostName = entry.getKey().substring(7); + refsNb = hostNamesToRefsNb.get(hostName); + if(refsNb == null) { + refsNb = refsize; + } else { + refsNb += refsize; } + if ( refsNb > maxref ) { + maxref = refsNb; + maxHostName = hostName; + } + hostNamesToRefsNb.put(hostName, refsNb); } } synchronized ( this.structure_new ) { for ( final Map.Entry entry : this.structure_new.entrySet() ) { refsize = entry.getValue().length; - if ( refsize > maxref ) { - maxref = refsize; - maxhost = entry.getKey().substring(7); + hostName = entry.getKey().substring(7); + refsNb = hostNamesToRefsNb.get(hostName); + if(refsNb == null) { + refsNb = refsize; + } else { + refsNb += refsize; + } + if ( refsNb > maxref ) { + maxref = refsNb; + maxHostName = hostName; } + hostNamesToRefsNb.put(hostName, refsNb); } } - return maxhost; + return maxHostName; } public ReversibleScoreMap hostReferenceScore() { diff --git a/test/java/net/yacy/peers/graphics/WebStructureGraphTest.java b/test/java/net/yacy/peers/graphics/WebStructureGraphTest.java index f660827ea..63fd6e029 100644 --- a/test/java/net/yacy/peers/graphics/WebStructureGraphTest.java +++ b/test/java/net/yacy/peers/graphics/WebStructureGraphTest.java @@ -23,6 +23,7 @@ package net.yacy.peers.graphics; import java.net.MalformedURLException; import java.util.HashSet; +import java.util.Map; import java.util.Set; import org.junit.Assert; @@ -120,6 +121,53 @@ public class WebStructureGraphTest { graph.close(); } } + + /** + * Out going references by host name + */ + @Test + public void outgoingReferencesByHostName() throws MalformedURLException { + WebStructureGraph graph = new WebStructureGraph(null); + try { + final DigestURL httpSource = new DigestURL("http://source.net/index.html"); + Set targets = new HashSet<>(); + final DigestURL indexTarget = new DigestURL("http://target.com/index.html"); + targets.add(indexTarget); + LearnObject lro = new LearnObject(httpSource, targets); + graph.learnrefs(lro); + + final DigestURL httpsSource = new DigestURL("https://source.net/index.html"); + targets = new HashSet<>(); + final DigestURL pathTarget = new DigestURL("http://target.com/path"); + targets.add(pathTarget); + lro = new LearnObject(httpsSource, targets); + graph.learnrefs(lro); + + final DigestURL otherPortSource = new DigestURL("https://source.net:8080/index.html"); + targets = new HashSet<>(); + final DigestURL queryTarget = new DigestURL("http://target.com/query?param=value"); + targets.add(queryTarget); + lro = new LearnObject(otherPortSource, targets); + graph.learnrefs(lro); + + /* Check that accumulated references from the host name is retrieved from structure */ + Map outRefs = graph.outgoingReferencesByHostName("source.net"); + + Assert.assertNotNull(outRefs); + Assert.assertEquals(1, outRefs.size()); + Assert.assertEquals(new DigestURL("http://target.com").hosthash(), outRefs.keySet().iterator().next()); + Assert.assertEquals(Integer.valueOf(3), outRefs.values().iterator().next()); + + /* Check that accumulated references from unknown host name is empty */ + outRefs = graph.outgoingReferencesByHostName("test.net"); + + Assert.assertNotNull(outRefs); + Assert.assertTrue(outRefs.isEmpty()); + + } finally { + graph.close(); + } + } /** * Most basic incoming references unit test