From 5d00888c95abafd670bb719a31672f3742b439b1 Mon Sep 17 00:00:00 2001
From: orbiter <orbiter@6c8d7289-2bf4-0310-a012-ef5d649a1542>
Date: Sun, 27 Jun 2010 10:45:20 +0000
Subject: [PATCH] - added animated visualization for DHT-in and DHT-out in
 network graphic - found and fixed a possible memory leak in YaCy internal RSS
 feed system - some refactoring in RSS feed mechanisms to make this possible

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6950 6c8d7289-2bf4-0310-a012-ef5d649a1542
---
 htroot/FeedReader_p.java                      |  3 +-
 htroot/Network.html                           | 91 ++++++++++---------
 htroot/NetworkPicture.java                    |  4 +-
 htroot/rct_p.java                             |  8 +-
 source/de/anomic/crawler/CrawlQueues.java     |  8 +-
 .../de/anomic/yacy/graphics/NetworkGraph.java | 65 +++++++++----
 source/net/yacy/cora/document/Hit.java        | 13 ++-
 source/net/yacy/cora/document/RSSFeed.java    | 27 +++---
 source/net/yacy/cora/document/RSSMessage.java | 81 +++++++++--------
 .../net/yacy/visualization/RasterPlotter.java | 10 ++
 10 files changed, 179 insertions(+), 131 deletions(-)

diff --git a/htroot/FeedReader_p.java b/htroot/FeedReader_p.java
index d2059e187..b1c26dbf5 100644
--- a/htroot/FeedReader_p.java
+++ b/htroot/FeedReader_p.java
@@ -30,6 +30,7 @@ import net.yacy.cora.document.RSSFeed;
 import net.yacy.cora.document.RSSReader;
 import net.yacy.kelondro.data.meta.DigestURI;
 import net.yacy.kelondro.logging.Log;
+import net.yacy.kelondro.util.DateFormatter;
 
 import de.anomic.http.server.RequestHeader;
 import de.anomic.server.serverObjects;
@@ -74,7 +75,7 @@ public class FeedReader_p {
                     prop.putHTML("page_items_" + i + "_title", item.getTitle());
                     prop.putHTML("page_items_" + i + "_link", item.getLink());
                     prop.putHTML("page_items_" + i + "_description", item.getDescription());
-                    prop.putHTML("page_items_" + i + "_date", item.getPubDate());
+                    prop.putHTML("page_items_" + i + "_date", DateFormatter.formatShortSecond(item.getPubDate()));
                     i++;
                 }
                 prop.put("page_items", feed.size());
diff --git a/htroot/Network.html b/htroot/Network.html
index 67cb396bf..a73ebc00d 100644
--- a/htroot/Network.html
+++ b/htroot/Network.html
@@ -10,7 +10,7 @@
     </style>
     
   <script type="text/javascript">
-    imagestub = "NetworkPicture.png?width=768&height=576&bgcolor=FFFFFF&coronaangle=";
+    imagestub = "NetworkPicture.png?width=768&height=576&bgcolor=FFFFFF&ct=5000&coronaangle=";
     imageanimindex = 0;
     imageloadindex = 0;
     imagecycles = 0;
@@ -218,10 +218,56 @@ To see a list of all APIs, please visit the <a href="http://www.yacy-websuche.de
     </table>
     </div>
     
-    <div class="left"><p>
+    <div class="left">
     <a href="NetworkPicture.png?width=768&height=576&bgcolor=FFFFFF"><img id="NetworkPicture" src="NetworkPicture.png?width=768&height=576&bgcolor=FFFFFF&coronaangle=0" alt="The YaCy Network" /></a>
     </p></div>
 	<div class="left">
+    <table border="0">
+            <tr>
+                    <td style="background-color:#668877">&nbsp;</td>
+                    <td>dark font</td>
+                    <td>:</td>
+                    <td>senior/principal peers</td>
+            </tr>
+            <tr>
+                    <td style="background-color:#B3CCB3">&nbsp;</td>
+                    <td>lightred font</td>
+                    <td>:</td>
+                    <td>passive peers</td>
+            </tr>
+            <tr>
+                    <td style="background-color:#CCB3B3">&nbsp;</td>
+                    <td>turquoise font</td>
+                    <td>:</td>
+                    <td>junior peers</td>
+            </tr>
+            <tr>
+                    <td style="background-color:#BE6F4F">&nbsp;</td>
+                    <td>red point</td>
+                    <td>:</td>
+                    <td>this peer</td>
+            </tr>
+            <tr>
+                    <td style="background-color:#AAAAAA">&nbsp;</td>
+                    <td>waves</td>
+                    <td>:</td>
+                    <td>crawling activity</td>
+            </tr>
+            <tr>
+                    <td style="background-color:#FF6666">&nbsp;</td>
+                    <td>red lines</td>
+                    <td>:</td>
+                    <td>DHT-out</td>
+            </tr>
+            <tr>
+                    <td style="background-color:#66FF66">&nbsp;</td>
+                    <td>green lines</td>
+                    <td>:</td>
+                    <td>DHT-in</td>
+            </tr>
+    </table>
+    </div>
+    <div class="left">
     Your Peer:
     <table border="0" cellpadding="2" cellspacing="1">
       <tr class="TableHeader">
@@ -264,46 +310,7 @@ To see a list of all APIs, please visit the <a href="http://www.yacy-websuche.de
       </tr>
     </table>
     </div>
-    <div class="left">
-    <table border="0">
-            <tr>
-                <td>&nbsp;</td>
-                    <td><strong>Network legend:</strong></td>
-                    <td></td>
-                    <td></td>
-            </tr>
-            <tr>
-                    <td style="background-color:#668877">&nbsp;</td>
-                    <td>dark font</td>
-                    <td>:</td>
-                    <td>senior/principal peers</td>
-            </tr>
-            <tr>
-                    <td style="background-color:#B3CCB3">&nbsp;</td>
-                    <td>lightred font</td>
-                    <td>:</td>
-                    <td>passiv peers ( &lt; 5 hour passive time)</td>
-            </tr>
-            <tr>
-                    <td style="background-color:#CCB3B3">&nbsp;</td>
-                    <td>turquoise font</td>
-                    <td>:</td>
-                    <td>junior peers</td>
-            </tr>
-            <tr>
-                    <td style="background-color:#BE6F4F">&nbsp;</td>
-                    <td>red point</td>
-                    <td>:</td>
-                    <td>this peer</td>
-            </tr>
-            <tr>
-                    <td style="background-color:#AAAAAA">&nbsp;</td>
-                    <td>waves</td>
-                    <td>:</td>
-                    <td>crawling activity</td>
-            </tr>
-    </table>
-    </div>
+    
     
     ::
     #(comment)#
diff --git a/htroot/NetworkPicture.java b/htroot/NetworkPicture.java
index 16256c9ff..6fd72b7b8 100644
--- a/htroot/NetworkPicture.java
+++ b/htroot/NetworkPicture.java
@@ -47,6 +47,7 @@ public class NetworkPicture {
         String bgcolor = NetworkGraph.COL_BACKGROUND;
         boolean corona = true;
         int coronaangle = 0;
+        long communicationTimeout = -1;
         
         if (post != null) {
             width = post.getInt("width", 768);
@@ -56,6 +57,7 @@ public class NetworkPicture {
             maxCount = post.getInt("max", 1000);
             corona = post.get("corona", "true").equals("true");
             coronaangle = (corona) ? post.getInt("coronaangle", 0) : -1;
+            communicationTimeout = post.getLong("ct", -1);
             bgcolor = post.get("bgcolor", bgcolor);
         }
         
@@ -71,7 +73,7 @@ public class NetworkPicture {
         if (passiveLimit > 1000000) passiveLimit = 1000000;
         if (potentialLimit > 1000000) potentialLimit = 1000000;
         if (maxCount > 10000) maxCount = 10000;
-        return NetworkGraph.getNetworkPicture(sb.peers, 10000, width, height, passiveLimit, potentialLimit, maxCount, coronaangle, env.getConfig(SwitchboardConstants.NETWORK_NAME, "unspecified"), env.getConfig("network.unit.description", "unspecified"), bgcolor);
+        return NetworkGraph.getNetworkPicture(sb.peers, 10000, width, height, passiveLimit, potentialLimit, maxCount, coronaangle, communicationTimeout, env.getConfig(SwitchboardConstants.NETWORK_NAME, "unspecified"), env.getConfig("network.unit.description", "unspecified"), bgcolor);
     }
     
 }
diff --git a/htroot/rct_p.java b/htroot/rct_p.java
index 716685881..52b916d0b 100644
--- a/htroot/rct_p.java
+++ b/htroot/rct_p.java
@@ -26,14 +26,12 @@
 // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
 import java.net.MalformedURLException;
-import java.text.ParseException;
 import java.util.Date;
 import java.util.Iterator;
 
 import net.yacy.cora.document.RSSFeed;
 import net.yacy.cora.document.Hit;
 import net.yacy.kelondro.data.meta.DigestURI;
-import net.yacy.kelondro.util.DateFormatter;
 
 import de.anomic.crawler.retrieval.Request;
 import de.anomic.http.server.RequestHeader;
@@ -68,11 +66,7 @@ public class rct_p {
                             url = null;
                         }
                         Date loaddate;
-                        try {
-                            loaddate = DateFormatter.parseShortSecond(item.getPubDate());
-                        } catch (final ParseException e) {
-                            loaddate = new Date();
-                        }
+                        loaddate = item.getPubDate();
                         final DigestURI referrer = null; // referrer needed!
                         final String urlRejectReason = sb.crawlStacker.urlInAcceptedDomain(url);
                         if (urlRejectReason == null) {
diff --git a/source/de/anomic/crawler/CrawlQueues.java b/source/de/anomic/crawler/CrawlQueues.java
index 172454036..efb633c87 100644
--- a/source/de/anomic/crawler/CrawlQueues.java
+++ b/source/de/anomic/crawler/CrawlQueues.java
@@ -29,7 +29,6 @@ package de.anomic.crawler;
 import java.io.File;
 import java.io.IOException;
 import java.net.MalformedURLException;
-import java.text.ParseException;
 import java.util.ArrayList;
 import java.util.Date;
 import java.util.Iterator;
@@ -41,7 +40,6 @@ import net.yacy.cora.document.RSSFeed;
 import net.yacy.kelondro.data.meta.DigestURI;
 import net.yacy.kelondro.logging.Log;
 import net.yacy.kelondro.order.Base64Order;
-import net.yacy.kelondro.util.DateFormatter;
 import net.yacy.kelondro.util.FileUtils;
 import net.yacy.kelondro.workflow.WorkflowJob;
 
@@ -435,11 +433,7 @@ public class CrawlQueues {
             } catch (final MalformedURLException e) {
                 referrer = null;
             }
-            try {
-                loaddate = DateFormatter.parseShortSecond(item.getPubDate());
-            } catch (final ParseException e) {
-                loaddate = new Date();
-            }
+            loaddate = item.getPubDate();
             final String urlRejectReason = sb.crawlStacker.urlInAcceptedDomain(url);
             if (urlRejectReason == null) {
                 // stack url
diff --git a/source/de/anomic/yacy/graphics/NetworkGraph.java b/source/de/anomic/yacy/graphics/NetworkGraph.java
index aad0eef14..405cae361 100644
--- a/source/de/anomic/yacy/graphics/NetworkGraph.java
+++ b/source/de/anomic/yacy/graphics/NetworkGraph.java
@@ -35,6 +35,9 @@ import java.awt.image.BufferedImage;
 import java.util.Date;
 import java.util.Iterator;
 
+import net.yacy.cora.document.Hit;
+import net.yacy.cora.document.RSSFeed;
+import net.yacy.cora.document.RSSFeed.YaCyChannel;
 import net.yacy.visualization.PrintTool;
 import net.yacy.visualization.RasterPlotter;
 
@@ -65,9 +68,11 @@ public class NetworkGraph {
     private static final String COL_POTENTIAL_DOT  = "002200";
     private static final String COL_POTENTIAL_LINE = "224422";
     private static final String COL_POTENTIAL_TEXT = "336633";
-    private static final String COL_WE_DOT         = "FF0000";
-    private static final String COL_WE_LINE        = "FFAAAA";
-    private static final String COL_WE_TEXT        = "FFCCCC";
+    private static final String COL_MYPEER_DOT     = "FF0000";
+    private static final String COL_MYPEER_LINE    = "FFAAAA";
+    private static final String COL_MYPEER_TEXT    = "FFCCCC";
+    private static final String COL_DHTOUT         = "440000";
+    private static final String COL_DHTIN          = "004400";
     
     private static final String COL_BORDER         = "000000";
     private static final String COL_NORMAL_TEXT    = "000000";
@@ -118,7 +123,7 @@ public class NetworkGraph {
         if (primarySearches == null) return null; // this was a local search and there are no threads
 
         // get a copy of a recent network picture
-        final RasterPlotter eventPicture = getNetworkPicture(seedDB, 120000, 640, 480, 300, 300, 1000, coronaangle, Switchboard.getSwitchboard().getConfig(SwitchboardConstants.NETWORK_NAME, "unspecified"), Switchboard.getSwitchboard().getConfig("network.unit.description", "unspecified"), COL_BACKGROUND);
+        final RasterPlotter eventPicture = getNetworkPicture(seedDB, 120000, 640, 480, 300, 300, 1000, coronaangle, -1, Switchboard.getSwitchboard().getConfig(SwitchboardConstants.NETWORK_NAME, "unspecified"), Switchboard.getSwitchboard().getConfig("network.unit.description", "unspecified"), COL_BACKGROUND);
         //if (eventPicture instanceof ymageMatrix) eventPicture = (ymageMatrix) eventPicture; //new ymageMatrix((ymageMatrix) eventPicture);
         // TODO: fix cloning of ymageMatrix pictures
         
@@ -161,11 +166,11 @@ public class NetworkGraph {
         return eventPicture;
     }
 
-    public static RasterPlotter getNetworkPicture(final yacySeedDB seedDB, final long maxAge, final int width, final int height, final int passiveLimit, final int potentialLimit, final int maxCount, final int coronaangle, final String networkName, final String networkTitle, final String bgcolor) {
-        return drawNetworkPicture(seedDB, width, height, passiveLimit, potentialLimit, maxCount, coronaangle, networkName, networkTitle, bgcolor);
+    public static RasterPlotter getNetworkPicture(final yacySeedDB seedDB, final long maxAge, final int width, final int height, final int passiveLimit, final int potentialLimit, final int maxCount, final int coronaangle, final long communicationTimeout, final String networkName, final String networkTitle, final String bgcolor) {
+        return drawNetworkPicture(seedDB, width, height, passiveLimit, potentialLimit, maxCount, coronaangle, communicationTimeout, networkName, networkTitle, bgcolor);
     }
 
-    private static RasterPlotter drawNetworkPicture(final yacySeedDB seedDB, final int width, final int height, final int passiveLimit, final int potentialLimit, final int maxCount, final int coronaangle, final String networkName, final String networkTitle, final String bgcolor) {
+    private static RasterPlotter drawNetworkPicture(final yacySeedDB seedDB, final int width, final int height, final int passiveLimit, final int potentialLimit, final int maxCount, final int coronaangle, final long communicationTimeout, final String networkName, final String networkTitle, final String bgcolor) {
 
         RasterPlotter networkPicture = new RasterPlotter(width, height, (bgcolor.equals("000000")) ? RasterPlotter.MODE_ADD : RasterPlotter.MODE_SUB, bgcolor);
         if (seedDB == null) return networkPicture; // no other peers known
@@ -226,12 +231,28 @@ public class NetworkGraph {
         totalCount += count;
 
         // draw my own peer
-        drawNetworkPicturePeer(networkPicture, width / 2, height / 2 + 20, innerradius, outerradius, seedDB.mySeed(), COL_WE_DOT, COL_WE_LINE, COL_WE_TEXT, coronaangle);
+        drawNetworkPicturePeer(networkPicture, width / 2, height / 2 + 20, innerradius, outerradius, seedDB.mySeed(), COL_MYPEER_DOT, COL_MYPEER_LINE, COL_MYPEER_TEXT, coronaangle);
 
         // draw DHT activity
-        
-        
-        
+        if (communicationTimeout >= 0) {
+            Date horizon = new Date(System.currentTimeMillis() - communicationTimeout);
+            for (Hit event: RSSFeed.channels(YaCyChannel.DHTRECEIVE)) {
+                assert event != null;
+                assert event.getPubDate() != null;
+                if (event.getPubDate().after(horizon)) {
+                    //System.out.println("*** NETWORK-DHTRECEIVE: " + event.getLink());
+                    drawNetworkPictureConnection(networkPicture, width / 2, height / 2 + 20, innerradius, seedDB.mySeed(), seedDB.get(event.getLink()), COL_DHTIN);
+                }
+            }
+            for (Hit event: RSSFeed.channels(YaCyChannel.DHTSEND)) {
+                assert event != null;
+                assert event.getPubDate() != null;
+                if (event.getPubDate().after(horizon)) {
+                    //System.out.println("*** NETWORK-DHTSEND: " + event.getLink());
+                    drawNetworkPictureConnection(networkPicture, width / 2, height / 2 + 20, innerradius, seedDB.mySeed(), seedDB.get(event.getLink()), COL_DHTOUT);
+                }
+            }
+        }        
         
         // draw description
         networkPicture.setColor(COL_HEADLINE);
@@ -243,7 +264,17 @@ public class NetworkGraph {
         return networkPicture;
     }
 
-    private static void drawNetworkPicturePeer(final RasterPlotter img, final int x, final int y, final int innerradius, final int outerradius, final yacySeed seed, final String colorDot, final String colorLine, final String colorText, final int coronaangle) {
+    private static void drawNetworkPictureConnection(final RasterPlotter img, final int centerX, final int centerY, final int innerradius, final yacySeed mySeed, final yacySeed otherSeed, final String colorLine) {
+        final int angleMy = (int) (360.0 * (((double) FlatWordPartitionScheme.std.dhtPosition(mySeed.hash.getBytes(), null)) / ((double) Long.MAX_VALUE)));
+        final int angleOther = (int) (360.0 * (((double) FlatWordPartitionScheme.std.dhtPosition(otherSeed.hash.getBytes(), null)) / ((double) Long.MAX_VALUE)));
+        // draw line
+        img.setColor(colorLine);
+        img.arcLine(centerX, centerY, innerradius, innerradius - 20, angleMy);
+        img.arcLine(centerX, centerY, innerradius, innerradius - 20, angleOther);
+        img.arcConnect(centerX, centerY, innerradius - 20, angleMy, angleOther);
+    }
+    
+    private static void drawNetworkPicturePeer(final RasterPlotter img, final int centerX, final int centerY, final int innerradius, final int outerradius, final yacySeed seed, final String colorDot, final String colorLine, final String colorText, final int coronaangle) {
         final String name = seed.getName().toUpperCase() /*+ ":" + seed.hash + ":" + (((double) ((int) (100 * (((double) yacySeed.dhtPosition(seed.hash)) / ((double) yacySeed.maxDHTDistance))))) / 100.0)*/;
         if (name.length() < shortestName) shortestName = name.length();
         if (name.length() > longestName) longestName = name.length();
@@ -255,13 +286,13 @@ public class NetworkGraph {
         if (dotsize > 18) dotsize = 18;
         // draw dot
         img.setColor(colorDot);
-        img.arcDot(x, y, innerradius, angle, dotsize);
+        img.arcDot(centerX, centerY, innerradius, angle, dotsize);
         // draw line to text
         img.setColor(colorLine);
-        img.arcLine(x, y, innerradius + 18, innerradius + linelength, angle);
+        img.arcLine(centerX, centerY, innerradius + 18, innerradius + linelength, angle);
         // draw text
         img.setColor(colorText);
-        PrintTool.arcPrint(img, x, y, innerradius + linelength, angle, name);
+        PrintTool.arcPrint(img, centerX, centerY, innerradius + linelength, angle, name);
 
         // draw corona around dot for crawling activity
         int ppm20 = seed.getPPM() / 20;
@@ -271,13 +302,13 @@ public class NetworkGraph {
             // draw a wave around crawling peers
             long strength;
             img.setColor("303030");
-            img.arcArc(x, y, innerradius, angle, dotsize + 1, dotsize + 1, 0, 360);
+            img.arcArc(centerX, centerY, innerradius, angle, dotsize + 1, dotsize + 1, 0, 360);
             final int waveradius = innerradius / 2;
             for (int r = 0; r < waveradius; r++) {
                 strength = (waveradius - r) * (long) (0x08 * ppm20 * (1.0 + Math.sin(ca + Math.PI * 16 * r / waveradius))) / waveradius;
                 //System.out.println("r = " + r + ", Strength = " + strength);
                 img.setColor((strength << 16) | (strength << 8) | strength);
-                img.arcArc(x, y, innerradius, angle, dotsize + r, dotsize + r, 0, 360);
+                img.arcArc(centerX, centerY, innerradius, angle, dotsize + r, dotsize + r, 0, 360);
             }
         }
     }
diff --git a/source/net/yacy/cora/document/Hit.java b/source/net/yacy/cora/document/Hit.java
index d6c7df073..c0a7a537f 100644
--- a/source/net/yacy/cora/document/Hit.java
+++ b/source/net/yacy/cora/document/Hit.java
@@ -22,6 +22,8 @@
 
 package net.yacy.cora.document;
 
+import java.util.Date;
+
 public interface Hit {
 
     public void setAuthor(String title);
@@ -40,14 +42,13 @@ public interface Hit {
     
     public void setDescription(String description);
     
-    public void setCreator(String pubdate);
-    
-    public void setPubDate(String pubdate);
+    public void setPubDate(Date pubdate);
     
     public void setGuid(String guid);
     
-    public void setDocs(String guid);
+    public void setDocs(String docs);
 
+    public void setSubject(String[] tags);
     
     public String getAuthor();
     
@@ -65,10 +66,12 @@ public interface Hit {
     
     public String getDescription();
     
-    public String getPubDate();
+    public Date   getPubDate();
     
     public String getGuid();
     
     public String getDocs();
 
+    public String[] getSubject();
+
 }
diff --git a/source/net/yacy/cora/document/RSSFeed.java b/source/net/yacy/cora/document/RSSFeed.java
index 89b123a37..128454506 100644
--- a/source/net/yacy/cora/document/RSSFeed.java
+++ b/source/net/yacy/cora/document/RSSFeed.java
@@ -20,10 +20,12 @@
 
 package net.yacy.cora.document;
 
+import java.util.Collections;
 import java.util.HashSet;
 import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentLinkedQueue;
 
 public class RSSFeed implements Iterable<Hit> {
 
@@ -57,13 +59,11 @@ public class RSSFeed implements Iterable<Hit> {
     // class variables
     private RSSMessage channel;
     private String imageURL;
-    ConcurrentLinkedQueue<String> messageQueue; // a list of GUIDs, so the items can be retrieved by a specific order
-    ConcurrentHashMap<String, RSSMessage> messages; // a guid:Item map
+    private Map<String, RSSMessage> messages; // a guid:Item map
     private int maxsize;
     
     public RSSFeed() {
-        messageQueue = new ConcurrentLinkedQueue<String>();
-        messages = new ConcurrentHashMap<String, RSSMessage>();
+        messages = Collections.synchronizedMap(new LinkedHashMap<String, RSSMessage>());
         channel = null;
         maxsize = Integer.MAX_VALUE;
     }
@@ -75,7 +75,7 @@ public class RSSFeed implements Iterable<Hit> {
 
     public void setMaxsize(final int maxsize) {
         this.maxsize = maxsize;
-        while (messageQueue.size() > this.maxsize) pollMessage();
+        while (messages.size() > this.maxsize) pollMessage();
     }
     
     public void setChannel(final RSSMessage channelItem) {
@@ -96,9 +96,8 @@ public class RSSFeed implements Iterable<Hit> {
     
     public void addMessage(final RSSMessage item) {
         final String guid = item.getGuid();
-        messageQueue.add(guid);
         messages.put(guid, item);
-        while (messageQueue.size() > this.maxsize) pollMessage();
+        while (messages.size() > this.maxsize) pollMessage();
     }
     
     public RSSMessage getMessage(final String guid) {
@@ -120,10 +119,12 @@ public class RSSFeed implements Iterable<Hit> {
     
     public RSSMessage pollMessage() {
         // retrieve and delete item
-        if (messageQueue.isEmpty()) return null;
-        final String nextGUID = messageQueue.poll();
-        if (nextGUID == null) return null;
-        return messages.remove(nextGUID);
+        synchronized (messages) {
+            if (messages.isEmpty()) return null;
+            final String nextGUID = messages.keySet().iterator().next();
+            if (nextGUID == null) return null;
+            return messages.remove(nextGUID);
+        }
     }
 
     public class messageIterator implements Iterator<Hit>{
@@ -132,7 +133,7 @@ public class RSSFeed implements Iterable<Hit> {
         String lastGUID;
         
         public messageIterator() {
-            GUIDiterator = messageQueue.iterator();
+            GUIDiterator = messages.keySet().iterator();
             lastGUID = null;
         }
 
diff --git a/source/net/yacy/cora/document/RSSMessage.java b/source/net/yacy/cora/document/RSSMessage.java
index 34be1bd78..ec6fe2000 100644
--- a/source/net/yacy/cora/document/RSSMessage.java
+++ b/source/net/yacy/cora/document/RSSMessage.java
@@ -20,12 +20,15 @@
 
 package net.yacy.cora.document;
 
+import java.text.ParseException;
 import java.util.Date;
 import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 
+import net.yacy.kelondro.util.DateFormatter;
+
 public class RSSMessage implements Hit {
 
     // statics for item generation and automatic categorization
@@ -84,7 +87,7 @@ public class RSSMessage implements Hit {
         setValue("title", title);
         setValue("description", description);
         setValue("link", link);
-        setValue("pubDate", new Date().toString());
+        setValue("pubDate", DateFormatter.formatShortSecond(new Date()));
         setValue("guid", Integer.toHexString((title + description + link).hashCode()));
     }
     
@@ -137,8 +140,19 @@ public class RSSMessage implements Hit {
         return Token.language.valueFrom(this.map);
     }
     
-    public String getPubDate() {
-        return Token.pubDate.valueFrom(this.map);
+    public Date getPubDate() {
+        String dateString = Token.pubDate.valueFrom(this.map);
+        Date date;
+        try {
+            date = DateFormatter.parseShortSecond(dateString);
+        } catch (ParseException e) {
+            try {
+                date = DateFormatter.parseISO8601(dateString);
+            } catch (ParseException e1) {
+                date = DateFormatter.parseHTTPDate(dateString);
+            }
+        }
+        return date;
     }
     
     public String getGuid() {
@@ -159,59 +173,51 @@ public class RSSMessage implements Hit {
         return this.map.toString();
     }
     
-    public void setAuthor(String title) {
-        // TODO Auto-generated method stub
-        
+    public void setAuthor(String author) {
+        setValue("author", author);
     }
 
-    public void setCategory(String title) {
-        // TODO Auto-generated method stub
-        
+    public void setCategory(String category) {
+        setValue("category", category);
     }
 
-    public void setCopyright(String title) {
-        // TODO Auto-generated method stub
-        
+    public void setCopyright(String copyright) {
+        setValue("copyright", copyright);
     }
-
-    public void setCreator(String pubdate) {
-        // TODO Auto-generated method stub
-        
+    
+    public void setSubject(String[] tags) {
+        StringBuilder sb = new StringBuilder(tags.length * 10);
+        for (String tag: tags) sb.append(tag).append(',');
+        if (sb.length() > 0) sb.setLength(sb.length() - 1);
+        setValue("subject", sb.toString());
     }
-
+    
     public void setDescription(String description) {
-        // TODO Auto-generated method stub
-        
+        setValue("description", description);
     }
 
-    public void setDocs(String guid) {
-        // TODO Auto-generated method stub
-        
+    public void setDocs(String docs) {
+        setValue("docs", docs);
     }
 
     public void setGuid(String guid) {
-        // TODO Auto-generated method stub
-        
+        setValue("guid", guid);
     }
 
-    public void setLanguage(String title) {
-        // TODO Auto-generated method stub
-        
+    public void setLanguage(String language) {
+        setValue("language", language);
     }
 
     public void setLink(String link) {
-        // TODO Auto-generated method stub
-        
+        setValue("link", link);
     }
 
-    public void setPubDate(String pubdate) {
-        // TODO Auto-generated method stub
-        
+    public void setPubDate(Date pubdate) {
+        setValue("pubDate", DateFormatter.formatISO8601(new Date()));
     }
-
-    public void setReferrer(String title) {
-        // TODO Auto-generated method stub
-        
+    
+    public void setReferrer(String referrer) {
+        setValue("referrer", referrer);
     }
 
     public void setSize(long size) {
@@ -225,7 +231,6 @@ public class RSSMessage implements Hit {
     }
 
     public void setTitle(String title) {
-        // TODO Auto-generated method stub
-        
+        setValue("title", title);
     }
 }
diff --git a/source/net/yacy/visualization/RasterPlotter.java b/source/net/yacy/visualization/RasterPlotter.java
index 1f514fbde..e07ffe731 100644
--- a/source/net/yacy/visualization/RasterPlotter.java
+++ b/source/net/yacy/visualization/RasterPlotter.java
@@ -315,6 +315,16 @@ public class RasterPlotter {
         dot(x, y, dotRadius, true);
     }
     
+    public void arcConnect(final int cx, final int cy, final int arcRadius, final int angle1, final int angle2) {
+        double a1 = Math.PI * ((double) angle1) / 180.0;
+        double a2 = Math.PI * ((double) angle2) / 180.0;
+        final int x1 = cx + (int) (arcRadius * Math.cos(a1));
+        final int y1 = cy - (int) (arcRadius * Math.sin(a1));
+        final int x2 = cx + (int) (arcRadius * Math.cos(a2));
+        final int y2 = cy - (int) (arcRadius * Math.sin(a2));
+        line(x1, y1, x2, y2);
+    }
+    
     public void arcArc(final int cx, final int cy, final int arcRadius, final int angle, final int innerRadius, final int outerRadius, final int fromArc, final int toArc) {
         double a = Math.PI * ((double) angle) / 180.0;
         final int x = cx + (int) (arcRadius * Math.cos(a));