boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

Fixed WatchWebStructure_p.html render to include https URLs.

Browse Source 
As described in mantis 721 (http://mantis.tokeek.de/view.php?id=721)
WatchWebStructure_p.html failed to include in its structure view https
and other protocols and ports than default http.
pull/105/head
luccioman 8 years ago
parent ed3dd5e31a
commit d9766ca981
3 changed files with 120 additions and 32 deletions
Show Stats Download Patch File Download Diff File

36
htroot/WebStructurePicture_p.java
Unescape View File

@ -24,16 +24,12 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
import java.net.MalformedURLException;
import java.util.AbstractMap;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import net.yacy.cora.document.id.DigestURL;
import net.yacy.cora.order.Base64Order;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.cora.sorting.ClusteredScoreMap;
@ -114,18 +110,11 @@ public class WebStructurePicture_p {
String[] hostlist = CommonPattern.COMMA.split(hosts);
for (int i = 0; i < hostlist.length; i++) {
String host = hostlist[i];
String hash = null;
try {
hash = new DigestURL("http://" + host).hosthash();
} catch (final MalformedURLException e) {
continue;
}
Map.Entry<String, String> centernode = new AbstractMap.SimpleEntry<String, String>(hash, host);
double angle = 2.0d * i * Math.PI / hostlist.length;
if (hostlist.length == 3) angle -= Math.PI / 2;
if (hostlist.length == 4) angle += Math.PI / 4;
graph.addNode(centernode.getValue(), Math.cos(angle) / 8, Math.sin(angle) / 8, 0);
place(graph, sb.webStructure, centernode, bf, nodes, timeout, hostlist.length == 1 ? 0 : 1, hostlist.length == 1 ? depth : depth + 1, cyc);
graph.addNode(host, Math.cos(angle) / 8, Math.sin(angle) / 8, 0);
place(graph, sb.webStructure, host, bf, nodes, timeout, hostlist.length == 1 ? 0 : 1, hostlist.length == 1 ? depth : depth + 1, cyc);
}
// apply physics to it to get a better shape
@ -151,19 +140,18 @@ public class WebStructurePicture_p {
}
private static final int place(
final GraphPlotter graph, final WebStructureGraph structure, Map.Entry<String, String> pivotnode,
final GraphPlotter graph, final WebStructureGraph structure, String hostName,
int bf, int maxnodes, final long timeout, int nextlayer, final int maxlayer, final int cyc) {
Point pivotpoint = graph.getNode(pivotnode.getValue());
Point pivotpoint = graph.getNode(hostName);
int branches = 0;
if (nextlayer == maxlayer) return branches;
nextlayer++;
final double radius = 1.0 / (1 << nextlayer);
final WebStructureGraph.StructureEntry sr = structure.outgoingReferences(pivotnode.getKey());
final Map<String, Integer> next = (sr == null) ? new HashMap<String, Integer>() : sr.references;
final Map<String, Integer> next = structure.outgoingReferencesByHostName(hostName);
ClusteredScoreMap<String> next0 = new ClusteredScoreMap<String>(false);
for (Map.Entry<String, Integer> entry: next.entrySet()) next0.set(entry.getKey(), entry.getValue());
// first set points to next hosts
final List<Map.Entry<String, String>> targets = new ArrayList<Map.Entry<String, String>>();
final Set<String> targetHostNames = new HashSet<String>();
int maxtargetrefs = 8, maxthisrefs = 8;
int targetrefs, thisrefs;
double rr, re;
@ -176,7 +164,7 @@ public class WebStructurePicture_p {
targetrefs = structure.referencesCount(targethash); // can be cpu/time-critical
maxtargetrefs = Math.max(targetrefs, maxtargetrefs);
maxthisrefs = Math.max(thisrefs, maxthisrefs);
targets.add(new AbstractMap.SimpleEntry<String, String>(targethash, targethost));
targetHostNames.add(targethost);
if (graph.getNode(targethost) != null) continue;
// set a new point. It is placed on a circle around the host point
final double angle = ((Base64Order.enhancedCoder.cardinal((targethash + "____").getBytes()) / maxlongd) + (cyc / 360.0d)) * 2.0d * Math.PI;
@ -189,11 +177,11 @@ public class WebStructurePicture_p {
}
// recursively set next hosts
int nextnodes;
for (Map.Entry<String, String> target: targets) {
nextnodes = ((maxnodes <= 0) || (System.currentTimeMillis() >= timeout)) ? 0 : place(graph, structure, target, bf, maxnodes, timeout, nextlayer, maxlayer, cyc);
for (String targetHostName: targetHostNames) {
nextnodes = ((maxnodes <= 0) || (System.currentTimeMillis() >= timeout)) ? 0 : place(graph, structure, targetHostName, bf, maxnodes, timeout, nextlayer, maxlayer, cyc);
branches += nextnodes;
maxnodes -= nextnodes;
graph.setEdge(pivotnode.getValue(), target.getValue());
graph.setEdge(hostName, targetHostName);
}
return branches;
}

68
source/net/yacy/peers/graphics/WebStructureGraph.java
Unescape View File

@ -354,6 +354,37 @@ public class WebStructureGraph {
return new StructureEntry(hosthash, hostname, date, h);
}
/**
* Compute outgoing references from source hostName on any source protocol or port.
* @param srcHostName reference source host name
* @return outgoing references mapped from target host hash to count
*/
public Map<String, Integer> outgoingReferencesByHostName(final String srcHostName) {
Set<String> srcHostHashes = this.hostName2HostHashes(srcHostName);
final Map<String, Integer> targetHashesToCount = new HashMap<String, Integer>();
for(String srcHostHash : srcHostHashes) {
final WebStructureGraph.StructureEntry sr = this.outgoingReferences(srcHostHash);
if(sr != null) {
for(java.util.Map.Entry<String, Integer> ref : sr.references.entrySet()) {
Integer refsNb = targetHashesToCount.get(ref.getKey());
if(refsNb != null) {
if(ref.getValue() != null) {
refsNb += ref.getValue();
}
} else {
if(ref.getValue() != null) {
refsNb = ref.getValue();
} else {
refsNb = Integer.valueOf(0);
}
}
targetHashesToCount.put(ref.getKey(), refsNb);
}
}
}
return targetHashesToCount;
}
public StructureEntry incomingReferences(final String hosthash) {
final String hostname = hostHash2hostName(hosthash);
if ( hostname == null ) {
@ -743,29 +774,50 @@ public class WebStructureGraph {
}
}
/**
* @return the host name having the most outgoing references or null when the structure is empty
*/
public String hostWithMaxReferences() {
// find host with most references
String maxhost = null;
Map<String, Integer> hostNamesToRefsNb = new TreeMap<>();
int refsize, maxref = 0;
String hostName, maxHostName = null;
Integer refsNb;
synchronized ( this.structure_old ) {
for ( final Map.Entry<String, byte[]> entry : this.structure_old.entrySet() ) {
refsize = entry.getValue().length;
if ( refsize > maxref ) {
maxref = refsize;
maxhost = entry.getKey().substring(7);
hostName = entry.getKey().substring(7);
refsNb = hostNamesToRefsNb.get(hostName);
if(refsNb == null) {
refsNb = refsize;
} else {
refsNb += refsize;
}
if ( refsNb > maxref ) {
maxref = refsNb;
maxHostName = hostName;
}
hostNamesToRefsNb.put(hostName, refsNb);
}
}
synchronized ( this.structure_new ) {
for ( final Map.Entry<String, byte[]> entry : this.structure_new.entrySet() ) {
refsize = entry.getValue().length;
if ( refsize > maxref ) {
maxref = refsize;
maxhost = entry.getKey().substring(7);
hostName = entry.getKey().substring(7);
refsNb = hostNamesToRefsNb.get(hostName);
if(refsNb == null) {
refsNb = refsize;
} else {
refsNb += refsize;
}
if ( refsNb > maxref ) {
maxref = refsNb;
maxHostName = hostName;
}
hostNamesToRefsNb.put(hostName, refsNb);
}
}
return maxhost;
return maxHostName;
}
public ReversibleScoreMap<String> hostReferenceScore() {

48
test/java/net/yacy/peers/graphics/WebStructureGraphTest.java
Unescape View File

@ -23,6 +23,7 @@ package net.yacy.peers.graphics;
import java.net.MalformedURLException;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import org.junit.Assert;
@ -120,6 +121,53 @@ public class WebStructureGraphTest {
graph.close();
}
}
/**
* Out going references by host name
*/
@Test
public void outgoingReferencesByHostName() throws MalformedURLException {
WebStructureGraph graph = new WebStructureGraph(null);
try {
final DigestURL httpSource = new DigestURL("http://source.net/index.html");
Set<DigestURL> targets = new HashSet<>();
final DigestURL indexTarget = new DigestURL("http://target.com/index.html");
targets.add(indexTarget);
LearnObject lro = new LearnObject(httpSource, targets);
graph.learnrefs(lro);
final DigestURL httpsSource = new DigestURL("https://source.net/index.html");
targets = new HashSet<>();
final DigestURL pathTarget = new DigestURL("http://target.com/path");
targets.add(pathTarget);
lro = new LearnObject(httpsSource, targets);
graph.learnrefs(lro);
final DigestURL otherPortSource = new DigestURL("https://source.net:8080/index.html");
targets = new HashSet<>();
final DigestURL queryTarget = new DigestURL("http://target.com/query?param=value");
targets.add(queryTarget);
lro = new LearnObject(otherPortSource, targets);
graph.learnrefs(lro);
/* Check that accumulated references from the host name is retrieved from structure */
Map<String, Integer> outRefs = graph.outgoingReferencesByHostName("source.net");
Assert.assertNotNull(outRefs);
Assert.assertEquals(1, outRefs.size());
Assert.assertEquals(new DigestURL("http://target.com").hosthash(), outRefs.keySet().iterator().next());
Assert.assertEquals(Integer.valueOf(3), outRefs.values().iterator().next());
/* Check that accumulated references from unknown host name is empty */
outRefs = graph.outgoingReferencesByHostName("test.net");
Assert.assertNotNull(outRefs);
Assert.assertTrue(outRefs.isEmpty());
} finally {
graph.close();
}
}
/**
* Most basic incoming references unit test

Write Preview
Loading…
Cancel
Save