- added animated visualization for DHT-in and DHT-out in network graphic

- found and fixed a possible memory leak in YaCy internal RSS feed system
- some refactoring in RSS feed mechanisms to make this possible

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6950 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 15 years ago
parent bf25407fdd
commit 5d00888c95

@ -30,6 +30,7 @@ import net.yacy.cora.document.RSSFeed;
import net.yacy.cora.document.RSSReader;
import net.yacy.kelondro.data.meta.DigestURI;
import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.util.DateFormatter;
import de.anomic.http.server.RequestHeader;
import de.anomic.server.serverObjects;
@ -74,7 +75,7 @@ public class FeedReader_p {
prop.putHTML("page_items_" + i + "_title", item.getTitle());
prop.putHTML("page_items_" + i + "_link", item.getLink());
prop.putHTML("page_items_" + i + "_description", item.getDescription());
prop.putHTML("page_items_" + i + "_date", item.getPubDate());
prop.putHTML("page_items_" + i + "_date", DateFormatter.formatShortSecond(item.getPubDate()));
i++;
}
prop.put("page_items", feed.size());

@ -10,7 +10,7 @@
</style>
<script type="text/javascript">
imagestub = "NetworkPicture.png?width=768&height=576&bgcolor=FFFFFF&coronaangle=";
imagestub = "NetworkPicture.png?width=768&height=576&bgcolor=FFFFFF&ct=5000&coronaangle=";
imageanimindex = 0;
imageloadindex = 0;
imagecycles = 0;
@ -218,10 +218,56 @@ To see a list of all APIs, please visit the <a href="http://www.yacy-websuche.de
</table>
</div>
<div class="left"><p>
<div class="left">
<a href="NetworkPicture.png?width=768&height=576&bgcolor=FFFFFF"><img id="NetworkPicture" src="NetworkPicture.png?width=768&height=576&bgcolor=FFFFFF&coronaangle=0" alt="The YaCy Network" /></a>
</p></div>
<div class="left">
<table border="0">
<tr>
<td style="background-color:#668877">&nbsp;</td>
<td>dark font</td>
<td>:</td>
<td>senior/principal peers</td>
</tr>
<tr>
<td style="background-color:#B3CCB3">&nbsp;</td>
<td>lightred font</td>
<td>:</td>
<td>passive peers</td>
</tr>
<tr>
<td style="background-color:#CCB3B3">&nbsp;</td>
<td>turquoise font</td>
<td>:</td>
<td>junior peers</td>
</tr>
<tr>
<td style="background-color:#BE6F4F">&nbsp;</td>
<td>red point</td>
<td>:</td>
<td>this peer</td>
</tr>
<tr>
<td style="background-color:#AAAAAA">&nbsp;</td>
<td>waves</td>
<td>:</td>
<td>crawling activity</td>
</tr>
<tr>
<td style="background-color:#FF6666">&nbsp;</td>
<td>red lines</td>
<td>:</td>
<td>DHT-out</td>
</tr>
<tr>
<td style="background-color:#66FF66">&nbsp;</td>
<td>green lines</td>
<td>:</td>
<td>DHT-in</td>
</tr>
</table>
</div>
<div class="left">
Your Peer:
<table border="0" cellpadding="2" cellspacing="1">
<tr class="TableHeader">
@ -264,46 +310,7 @@ To see a list of all APIs, please visit the <a href="http://www.yacy-websuche.de
</tr>
</table>
</div>
<div class="left">
<table border="0">
<tr>
<td>&nbsp;</td>
<td><strong>Network legend:</strong></td>
<td></td>
<td></td>
</tr>
<tr>
<td style="background-color:#668877">&nbsp;</td>
<td>dark font</td>
<td>:</td>
<td>senior/principal peers</td>
</tr>
<tr>
<td style="background-color:#B3CCB3">&nbsp;</td>
<td>lightred font</td>
<td>:</td>
<td>passiv peers ( &lt; 5 hour passive time)</td>
</tr>
<tr>
<td style="background-color:#CCB3B3">&nbsp;</td>
<td>turquoise font</td>
<td>:</td>
<td>junior peers</td>
</tr>
<tr>
<td style="background-color:#BE6F4F">&nbsp;</td>
<td>red point</td>
<td>:</td>
<td>this peer</td>
</tr>
<tr>
<td style="background-color:#AAAAAA">&nbsp;</td>
<td>waves</td>
<td>:</td>
<td>crawling activity</td>
</tr>
</table>
</div>
::
#(comment)#

@ -47,6 +47,7 @@ public class NetworkPicture {
String bgcolor = NetworkGraph.COL_BACKGROUND;
boolean corona = true;
int coronaangle = 0;
long communicationTimeout = -1;
if (post != null) {
width = post.getInt("width", 768);
@ -56,6 +57,7 @@ public class NetworkPicture {
maxCount = post.getInt("max", 1000);
corona = post.get("corona", "true").equals("true");
coronaangle = (corona) ? post.getInt("coronaangle", 0) : -1;
communicationTimeout = post.getLong("ct", -1);
bgcolor = post.get("bgcolor", bgcolor);
}
@ -71,7 +73,7 @@ public class NetworkPicture {
if (passiveLimit > 1000000) passiveLimit = 1000000;
if (potentialLimit > 1000000) potentialLimit = 1000000;
if (maxCount > 10000) maxCount = 10000;
return NetworkGraph.getNetworkPicture(sb.peers, 10000, width, height, passiveLimit, potentialLimit, maxCount, coronaangle, env.getConfig(SwitchboardConstants.NETWORK_NAME, "unspecified"), env.getConfig("network.unit.description", "unspecified"), bgcolor);
return NetworkGraph.getNetworkPicture(sb.peers, 10000, width, height, passiveLimit, potentialLimit, maxCount, coronaangle, communicationTimeout, env.getConfig(SwitchboardConstants.NETWORK_NAME, "unspecified"), env.getConfig("network.unit.description", "unspecified"), bgcolor);
}
}

@ -26,14 +26,12 @@
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
import java.net.MalformedURLException;
import java.text.ParseException;
import java.util.Date;
import java.util.Iterator;
import net.yacy.cora.document.RSSFeed;
import net.yacy.cora.document.Hit;
import net.yacy.kelondro.data.meta.DigestURI;
import net.yacy.kelondro.util.DateFormatter;
import de.anomic.crawler.retrieval.Request;
import de.anomic.http.server.RequestHeader;
@ -68,11 +66,7 @@ public class rct_p {
url = null;
}
Date loaddate;
try {
loaddate = DateFormatter.parseShortSecond(item.getPubDate());
} catch (final ParseException e) {
loaddate = new Date();
}
loaddate = item.getPubDate();
final DigestURI referrer = null; // referrer needed!
final String urlRejectReason = sb.crawlStacker.urlInAcceptedDomain(url);
if (urlRejectReason == null) {

@ -29,7 +29,6 @@ package de.anomic.crawler;
import java.io.File;
import java.io.IOException;
import java.net.MalformedURLException;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Date;
import java.util.Iterator;
@ -41,7 +40,6 @@ import net.yacy.cora.document.RSSFeed;
import net.yacy.kelondro.data.meta.DigestURI;
import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.order.Base64Order;
import net.yacy.kelondro.util.DateFormatter;
import net.yacy.kelondro.util.FileUtils;
import net.yacy.kelondro.workflow.WorkflowJob;
@ -435,11 +433,7 @@ public class CrawlQueues {
} catch (final MalformedURLException e) {
referrer = null;
}
try {
loaddate = DateFormatter.parseShortSecond(item.getPubDate());
} catch (final ParseException e) {
loaddate = new Date();
}
loaddate = item.getPubDate();
final String urlRejectReason = sb.crawlStacker.urlInAcceptedDomain(url);
if (urlRejectReason == null) {
// stack url

@ -35,6 +35,9 @@ import java.awt.image.BufferedImage;
import java.util.Date;
import java.util.Iterator;
import net.yacy.cora.document.Hit;
import net.yacy.cora.document.RSSFeed;
import net.yacy.cora.document.RSSFeed.YaCyChannel;
import net.yacy.visualization.PrintTool;
import net.yacy.visualization.RasterPlotter;
@ -65,9 +68,11 @@ public class NetworkGraph {
private static final String COL_POTENTIAL_DOT = "002200";
private static final String COL_POTENTIAL_LINE = "224422";
private static final String COL_POTENTIAL_TEXT = "336633";
private static final String COL_WE_DOT = "FF0000";
private static final String COL_WE_LINE = "FFAAAA";
private static final String COL_WE_TEXT = "FFCCCC";
private static final String COL_MYPEER_DOT = "FF0000";
private static final String COL_MYPEER_LINE = "FFAAAA";
private static final String COL_MYPEER_TEXT = "FFCCCC";
private static final String COL_DHTOUT = "440000";
private static final String COL_DHTIN = "004400";
private static final String COL_BORDER = "000000";
private static final String COL_NORMAL_TEXT = "000000";
@ -118,7 +123,7 @@ public class NetworkGraph {
if (primarySearches == null) return null; // this was a local search and there are no threads
// get a copy of a recent network picture
final RasterPlotter eventPicture = getNetworkPicture(seedDB, 120000, 640, 480, 300, 300, 1000, coronaangle, Switchboard.getSwitchboard().getConfig(SwitchboardConstants.NETWORK_NAME, "unspecified"), Switchboard.getSwitchboard().getConfig("network.unit.description", "unspecified"), COL_BACKGROUND);
final RasterPlotter eventPicture = getNetworkPicture(seedDB, 120000, 640, 480, 300, 300, 1000, coronaangle, -1, Switchboard.getSwitchboard().getConfig(SwitchboardConstants.NETWORK_NAME, "unspecified"), Switchboard.getSwitchboard().getConfig("network.unit.description", "unspecified"), COL_BACKGROUND);
//if (eventPicture instanceof ymageMatrix) eventPicture = (ymageMatrix) eventPicture; //new ymageMatrix((ymageMatrix) eventPicture);
// TODO: fix cloning of ymageMatrix pictures
@ -161,11 +166,11 @@ public class NetworkGraph {
return eventPicture;
}
public static RasterPlotter getNetworkPicture(final yacySeedDB seedDB, final long maxAge, final int width, final int height, final int passiveLimit, final int potentialLimit, final int maxCount, final int coronaangle, final String networkName, final String networkTitle, final String bgcolor) {
return drawNetworkPicture(seedDB, width, height, passiveLimit, potentialLimit, maxCount, coronaangle, networkName, networkTitle, bgcolor);
public static RasterPlotter getNetworkPicture(final yacySeedDB seedDB, final long maxAge, final int width, final int height, final int passiveLimit, final int potentialLimit, final int maxCount, final int coronaangle, final long communicationTimeout, final String networkName, final String networkTitle, final String bgcolor) {
return drawNetworkPicture(seedDB, width, height, passiveLimit, potentialLimit, maxCount, coronaangle, communicationTimeout, networkName, networkTitle, bgcolor);
}
private static RasterPlotter drawNetworkPicture(final yacySeedDB seedDB, final int width, final int height, final int passiveLimit, final int potentialLimit, final int maxCount, final int coronaangle, final String networkName, final String networkTitle, final String bgcolor) {
private static RasterPlotter drawNetworkPicture(final yacySeedDB seedDB, final int width, final int height, final int passiveLimit, final int potentialLimit, final int maxCount, final int coronaangle, final long communicationTimeout, final String networkName, final String networkTitle, final String bgcolor) {
RasterPlotter networkPicture = new RasterPlotter(width, height, (bgcolor.equals("000000")) ? RasterPlotter.MODE_ADD : RasterPlotter.MODE_SUB, bgcolor);
if (seedDB == null) return networkPicture; // no other peers known
@ -226,12 +231,28 @@ public class NetworkGraph {
totalCount += count;
// draw my own peer
drawNetworkPicturePeer(networkPicture, width / 2, height / 2 + 20, innerradius, outerradius, seedDB.mySeed(), COL_WE_DOT, COL_WE_LINE, COL_WE_TEXT, coronaangle);
drawNetworkPicturePeer(networkPicture, width / 2, height / 2 + 20, innerradius, outerradius, seedDB.mySeed(), COL_MYPEER_DOT, COL_MYPEER_LINE, COL_MYPEER_TEXT, coronaangle);
// draw DHT activity
if (communicationTimeout >= 0) {
Date horizon = new Date(System.currentTimeMillis() - communicationTimeout);
for (Hit event: RSSFeed.channels(YaCyChannel.DHTRECEIVE)) {
assert event != null;
assert event.getPubDate() != null;
if (event.getPubDate().after(horizon)) {
//System.out.println("*** NETWORK-DHTRECEIVE: " + event.getLink());
drawNetworkPictureConnection(networkPicture, width / 2, height / 2 + 20, innerradius, seedDB.mySeed(), seedDB.get(event.getLink()), COL_DHTIN);
}
}
for (Hit event: RSSFeed.channels(YaCyChannel.DHTSEND)) {
assert event != null;
assert event.getPubDate() != null;
if (event.getPubDate().after(horizon)) {
//System.out.println("*** NETWORK-DHTSEND: " + event.getLink());
drawNetworkPictureConnection(networkPicture, width / 2, height / 2 + 20, innerradius, seedDB.mySeed(), seedDB.get(event.getLink()), COL_DHTOUT);
}
}
}
// draw description
networkPicture.setColor(COL_HEADLINE);
@ -243,7 +264,17 @@ public class NetworkGraph {
return networkPicture;
}
private static void drawNetworkPicturePeer(final RasterPlotter img, final int x, final int y, final int innerradius, final int outerradius, final yacySeed seed, final String colorDot, final String colorLine, final String colorText, final int coronaangle) {
private static void drawNetworkPictureConnection(final RasterPlotter img, final int centerX, final int centerY, final int innerradius, final yacySeed mySeed, final yacySeed otherSeed, final String colorLine) {
final int angleMy = (int) (360.0 * (((double) FlatWordPartitionScheme.std.dhtPosition(mySeed.hash.getBytes(), null)) / ((double) Long.MAX_VALUE)));
final int angleOther = (int) (360.0 * (((double) FlatWordPartitionScheme.std.dhtPosition(otherSeed.hash.getBytes(), null)) / ((double) Long.MAX_VALUE)));
// draw line
img.setColor(colorLine);
img.arcLine(centerX, centerY, innerradius, innerradius - 20, angleMy);
img.arcLine(centerX, centerY, innerradius, innerradius - 20, angleOther);
img.arcConnect(centerX, centerY, innerradius - 20, angleMy, angleOther);
}
private static void drawNetworkPicturePeer(final RasterPlotter img, final int centerX, final int centerY, final int innerradius, final int outerradius, final yacySeed seed, final String colorDot, final String colorLine, final String colorText, final int coronaangle) {
final String name = seed.getName().toUpperCase() /*+ ":" + seed.hash + ":" + (((double) ((int) (100 * (((double) yacySeed.dhtPosition(seed.hash)) / ((double) yacySeed.maxDHTDistance))))) / 100.0)*/;
if (name.length() < shortestName) shortestName = name.length();
if (name.length() > longestName) longestName = name.length();
@ -255,13 +286,13 @@ public class NetworkGraph {
if (dotsize > 18) dotsize = 18;
// draw dot
img.setColor(colorDot);
img.arcDot(x, y, innerradius, angle, dotsize);
img.arcDot(centerX, centerY, innerradius, angle, dotsize);
// draw line to text
img.setColor(colorLine);
img.arcLine(x, y, innerradius + 18, innerradius + linelength, angle);
img.arcLine(centerX, centerY, innerradius + 18, innerradius + linelength, angle);
// draw text
img.setColor(colorText);
PrintTool.arcPrint(img, x, y, innerradius + linelength, angle, name);
PrintTool.arcPrint(img, centerX, centerY, innerradius + linelength, angle, name);
// draw corona around dot for crawling activity
int ppm20 = seed.getPPM() / 20;
@ -271,13 +302,13 @@ public class NetworkGraph {
// draw a wave around crawling peers
long strength;
img.setColor("303030");
img.arcArc(x, y, innerradius, angle, dotsize + 1, dotsize + 1, 0, 360);
img.arcArc(centerX, centerY, innerradius, angle, dotsize + 1, dotsize + 1, 0, 360);
final int waveradius = innerradius / 2;
for (int r = 0; r < waveradius; r++) {
strength = (waveradius - r) * (long) (0x08 * ppm20 * (1.0 + Math.sin(ca + Math.PI * 16 * r / waveradius))) / waveradius;
//System.out.println("r = " + r + ", Strength = " + strength);
img.setColor((strength << 16) | (strength << 8) | strength);
img.arcArc(x, y, innerradius, angle, dotsize + r, dotsize + r, 0, 360);
img.arcArc(centerX, centerY, innerradius, angle, dotsize + r, dotsize + r, 0, 360);
}
}
}

@ -22,6 +22,8 @@
package net.yacy.cora.document;
import java.util.Date;
public interface Hit {
public void setAuthor(String title);
@ -40,14 +42,13 @@ public interface Hit {
public void setDescription(String description);
public void setCreator(String pubdate);
public void setPubDate(String pubdate);
public void setPubDate(Date pubdate);
public void setGuid(String guid);
public void setDocs(String guid);
public void setDocs(String docs);
public void setSubject(String[] tags);
public String getAuthor();
@ -65,10 +66,12 @@ public interface Hit {
public String getDescription();
public String getPubDate();
public Date getPubDate();
public String getGuid();
public String getDocs();
public String[] getSubject();
}

@ -20,10 +20,12 @@
package net.yacy.cora.document;
import java.util.Collections;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentLinkedQueue;
public class RSSFeed implements Iterable<Hit> {
@ -57,13 +59,11 @@ public class RSSFeed implements Iterable<Hit> {
// class variables
private RSSMessage channel;
private String imageURL;
ConcurrentLinkedQueue<String> messageQueue; // a list of GUIDs, so the items can be retrieved by a specific order
ConcurrentHashMap<String, RSSMessage> messages; // a guid:Item map
private Map<String, RSSMessage> messages; // a guid:Item map
private int maxsize;
public RSSFeed() {
messageQueue = new ConcurrentLinkedQueue<String>();
messages = new ConcurrentHashMap<String, RSSMessage>();
messages = Collections.synchronizedMap(new LinkedHashMap<String, RSSMessage>());
channel = null;
maxsize = Integer.MAX_VALUE;
}
@ -75,7 +75,7 @@ public class RSSFeed implements Iterable<Hit> {
public void setMaxsize(final int maxsize) {
this.maxsize = maxsize;
while (messageQueue.size() > this.maxsize) pollMessage();
while (messages.size() > this.maxsize) pollMessage();
}
public void setChannel(final RSSMessage channelItem) {
@ -96,9 +96,8 @@ public class RSSFeed implements Iterable<Hit> {
public void addMessage(final RSSMessage item) {
final String guid = item.getGuid();
messageQueue.add(guid);
messages.put(guid, item);
while (messageQueue.size() > this.maxsize) pollMessage();
while (messages.size() > this.maxsize) pollMessage();
}
public RSSMessage getMessage(final String guid) {
@ -120,11 +119,13 @@ public class RSSFeed implements Iterable<Hit> {
public RSSMessage pollMessage() {
// retrieve and delete item
if (messageQueue.isEmpty()) return null;
final String nextGUID = messageQueue.poll();
synchronized (messages) {
if (messages.isEmpty()) return null;
final String nextGUID = messages.keySet().iterator().next();
if (nextGUID == null) return null;
return messages.remove(nextGUID);
}
}
public class messageIterator implements Iterator<Hit>{
@ -132,7 +133,7 @@ public class RSSFeed implements Iterable<Hit> {
String lastGUID;
public messageIterator() {
GUIDiterator = messageQueue.iterator();
GUIDiterator = messages.keySet().iterator();
lastGUID = null;
}

@ -20,12 +20,15 @@
package net.yacy.cora.document;
import java.text.ParseException;
import java.util.Date;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import net.yacy.kelondro.util.DateFormatter;
public class RSSMessage implements Hit {
// statics for item generation and automatic categorization
@ -84,7 +87,7 @@ public class RSSMessage implements Hit {
setValue("title", title);
setValue("description", description);
setValue("link", link);
setValue("pubDate", new Date().toString());
setValue("pubDate", DateFormatter.formatShortSecond(new Date()));
setValue("guid", Integer.toHexString((title + description + link).hashCode()));
}
@ -137,8 +140,19 @@ public class RSSMessage implements Hit {
return Token.language.valueFrom(this.map);
}
public String getPubDate() {
return Token.pubDate.valueFrom(this.map);
public Date getPubDate() {
String dateString = Token.pubDate.valueFrom(this.map);
Date date;
try {
date = DateFormatter.parseShortSecond(dateString);
} catch (ParseException e) {
try {
date = DateFormatter.parseISO8601(dateString);
} catch (ParseException e1) {
date = DateFormatter.parseHTTPDate(dateString);
}
}
return date;
}
public String getGuid() {
@ -159,59 +173,51 @@ public class RSSMessage implements Hit {
return this.map.toString();
}
public void setAuthor(String title) {
// TODO Auto-generated method stub
public void setAuthor(String author) {
setValue("author", author);
}
public void setCategory(String title) {
// TODO Auto-generated method stub
public void setCategory(String category) {
setValue("category", category);
}
public void setCopyright(String title) {
// TODO Auto-generated method stub
public void setCopyright(String copyright) {
setValue("copyright", copyright);
}
public void setCreator(String pubdate) {
// TODO Auto-generated method stub
public void setSubject(String[] tags) {
StringBuilder sb = new StringBuilder(tags.length * 10);
for (String tag: tags) sb.append(tag).append(',');
if (sb.length() > 0) sb.setLength(sb.length() - 1);
setValue("subject", sb.toString());
}
public void setDescription(String description) {
// TODO Auto-generated method stub
setValue("description", description);
}
public void setDocs(String guid) {
// TODO Auto-generated method stub
public void setDocs(String docs) {
setValue("docs", docs);
}
public void setGuid(String guid) {
// TODO Auto-generated method stub
setValue("guid", guid);
}
public void setLanguage(String title) {
// TODO Auto-generated method stub
public void setLanguage(String language) {
setValue("language", language);
}
public void setLink(String link) {
// TODO Auto-generated method stub
setValue("link", link);
}
public void setPubDate(String pubdate) {
// TODO Auto-generated method stub
public void setPubDate(Date pubdate) {
setValue("pubDate", DateFormatter.formatISO8601(new Date()));
}
public void setReferrer(String title) {
// TODO Auto-generated method stub
public void setReferrer(String referrer) {
setValue("referrer", referrer);
}
public void setSize(long size) {
@ -225,7 +231,6 @@ public class RSSMessage implements Hit {
}
public void setTitle(String title) {
// TODO Auto-generated method stub
setValue("title", title);
}
}

@ -315,6 +315,16 @@ public class RasterPlotter {
dot(x, y, dotRadius, true);
}
public void arcConnect(final int cx, final int cy, final int arcRadius, final int angle1, final int angle2) {
double a1 = Math.PI * ((double) angle1) / 180.0;
double a2 = Math.PI * ((double) angle2) / 180.0;
final int x1 = cx + (int) (arcRadius * Math.cos(a1));
final int y1 = cy - (int) (arcRadius * Math.sin(a1));
final int x2 = cx + (int) (arcRadius * Math.cos(a2));
final int y2 = cy - (int) (arcRadius * Math.sin(a2));
line(x1, y1, x2, y2);
}
public void arcArc(final int cx, final int cy, final int arcRadius, final int angle, final int innerRadius, final int outerRadius, final int fromArc, final int toArc) {
double a = Math.PI * ((double) angle) / 180.0;
final int x = cx + (int) (arcRadius * Math.cos(a));

Loading…
Cancel
Save