From 208b5297f1ef6c0e3c7df5275b1412d3a979f95f Mon Sep 17 00:00:00 2001
From: orbiter <orbiter@6c8d7289-2bf4-0310-a012-ef5d649a1542>
Date: Thu, 5 Jul 2007 22:56:37 +0000
Subject: [PATCH] enhanced handling of news records: result is a speedup of
 Surftips, Supporter, and Network page

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@3954 6c8d7289-2bf4-0310-a012-ef5d649a1542
---
 htroot/CrawlStartSimple_p.java                | 76 +++++++++----------
 htroot/Network.java                           | 33 ++++----
 htroot/News.java                              | 19 +++--
 htroot/Supporter.html                         |  2 +-
 htroot/Supporter.java                         | 25 +++---
 htroot/Surftips.java                          | 43 ++++++-----
 source/de/anomic/kelondro/kelondroStack.java  | 51 ++++---------
 .../de/anomic/plasma/plasmaCrawlBalancer.java |  2 +-
 .../de/anomic/plasma/plasmaSwitchboard.java   |  6 ++
 source/de/anomic/yacy/yacyNewsPool.java       |  9 +++
 source/de/anomic/yacy/yacyNewsQueue.java      | 39 +++++++++-
 yacy.init                                     |  5 ++
 12 files changed, 170 insertions(+), 140 deletions(-)

diff --git a/htroot/CrawlStartSimple_p.java b/htroot/CrawlStartSimple_p.java
index ecdf849ad..41fec93b5 100644
--- a/htroot/CrawlStartSimple_p.java
+++ b/htroot/CrawlStartSimple_p.java
@@ -24,8 +24,8 @@
 // along with this program; if not, write to the Free Software
 // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
-import java.io.IOException;
 import java.util.Enumeration;
+import java.util.Iterator;
 
 import de.anomic.http.httpHeader;
 import de.anomic.plasma.plasmaURL;
@@ -101,56 +101,50 @@ public class CrawlStartSimple_p {
         boolean dark = true;   
         
         // create other peer crawl table using YaCyNews
-        int availableNews = yacyCore.newsPool.size(yacyNewsPool.INCOMING_DB);
+        Iterator recordIterator = yacyCore.newsPool.recordIterator(yacyNewsPool.INCOMING_DB, true);
         int showedCrawl = 0;
         yacyNewsRecord record;
         yacySeed peer;
         String peername;
-        try {
-            for (int c = 0; c < availableNews; c++) {
-                record = yacyCore.newsPool.get(yacyNewsPool.INCOMING_DB, c);
-                if (record == null) continue;
-                if (record.category().equals(yacyNewsPool.CATEGORY_CRAWL_START)) {
-                    peer = yacyCore.seedDB.get(record.originator());
-                    if (peer == null) peername = record.originator(); else peername = peer.getName();
-                    prop.put("otherCrawlStartInProgress_" + showedCrawl + "_dark", ((dark) ? 1 : 0));
-                    prop.put("otherCrawlStartInProgress_" + showedCrawl + "_cre", record.created());
-                    prop.put("otherCrawlStartInProgress_" + showedCrawl + "_peername", peername);
-                    prop.put("otherCrawlStartInProgress_" + showedCrawl + "_startURL", record.attributes().get("startURL").toString());
-                    prop.put("otherCrawlStartInProgress_" + showedCrawl + "_intention", record.attributes().get("intention").toString());
-                    prop.put("otherCrawlStartInProgress_" + showedCrawl + "_generalDepth", record.attributes().get("generalDepth"));
-                    prop.put("otherCrawlStartInProgress_" + showedCrawl + "_crawlingQ", (record.attributes().get("crawlingQ").equals("true")) ? 1 : 0);
-                    showedCrawl++;
-                    if (showedCrawl > 20) break;
-                }
-                
+        while (recordIterator.hasNext()) {
+            record = (yacyNewsRecord) recordIterator.next();
+            if (record == null) continue;
+            if (record.category().equals(yacyNewsPool.CATEGORY_CRAWL_START)) {
+                peer = yacyCore.seedDB.get(record.originator());
+                if (peer == null) peername = record.originator(); else peername = peer.getName();
+                prop.put("otherCrawlStartInProgress_" + showedCrawl + "_dark", ((dark) ? 1 : 0));
+                prop.put("otherCrawlStartInProgress_" + showedCrawl + "_cre", record.created());
+                prop.put("otherCrawlStartInProgress_" + showedCrawl + "_peername", peername);
+                prop.put("otherCrawlStartInProgress_" + showedCrawl + "_startURL", record.attributes().get("startURL").toString());
+                prop.put("otherCrawlStartInProgress_" + showedCrawl + "_intention", record.attributes().get("intention").toString());
+                prop.put("otherCrawlStartInProgress_" + showedCrawl + "_generalDepth", record.attributes().get("generalDepth"));
+                prop.put("otherCrawlStartInProgress_" + showedCrawl + "_crawlingQ", (record.attributes().get("crawlingQ").equals("true")) ? 1 : 0);
+                showedCrawl++;
+                if (showedCrawl > 20) break;
             }
-        } catch (IOException e) {}
+        }
         prop.put("otherCrawlStartInProgress", showedCrawl);
         
         // finished remote crawls
-        availableNews = yacyCore.newsPool.size(yacyNewsPool.PROCESSED_DB);
+        recordIterator = yacyCore.newsPool.recordIterator(yacyNewsPool.PROCESSED_DB, true);
         showedCrawl = 0;
-        try {
-            for (int c = 0; c < availableNews; c++) {
-                record = yacyCore.newsPool.get(yacyNewsPool.PROCESSED_DB, c);
-                if (record == null) continue;
-                if (record.category().equals(yacyNewsPool.CATEGORY_CRAWL_START)) {
-                    peer = yacyCore.seedDB.get(record.originator());
-                    if (peer == null) peername = record.originator(); else peername = peer.getName();
-                    prop.put("otherCrawlStartFinished_" + showedCrawl + "_dark", ((dark) ? 1 : 0));
-                    prop.put("otherCrawlStartFinished_" + showedCrawl + "_cre", record.created());
-                    prop.put("otherCrawlStartFinished_" + showedCrawl + "_peername", peername);
-                    prop.put("otherCrawlStartFinished_" + showedCrawl + "_startURL", record.attributes().get("startURL").toString());
-                    prop.put("otherCrawlStartFinished_" + showedCrawl + "_intention", record.attributes().get("intention").toString());
-                    prop.put("otherCrawlStartFinished_" + showedCrawl + "_generalDepth", record.attributes().get("generalDepth"));
-                    prop.put("otherCrawlStartFinished_" + showedCrawl + "_crawlingQ", (record.attributes().get("crawlingQ").equals("true")) ? 1 : 0);
-                    showedCrawl++;
-                    if (showedCrawl > 20) break;
-                }
-                
+        while (recordIterator.hasNext()) {
+            record = (yacyNewsRecord) recordIterator.next();
+            if (record == null) continue;
+            if (record.category().equals(yacyNewsPool.CATEGORY_CRAWL_START)) {
+                peer = yacyCore.seedDB.get(record.originator());
+                if (peer == null) peername = record.originator(); else peername = peer.getName();
+                prop.put("otherCrawlStartFinished_" + showedCrawl + "_dark", ((dark) ? 1 : 0));
+                prop.put("otherCrawlStartFinished_" + showedCrawl + "_cre", record.created());
+                prop.put("otherCrawlStartFinished_" + showedCrawl + "_peername", peername);
+                prop.put("otherCrawlStartFinished_" + showedCrawl + "_startURL", record.attributes().get("startURL").toString());
+                prop.put("otherCrawlStartFinished_" + showedCrawl + "_intention", record.attributes().get("intention").toString());
+                prop.put("otherCrawlStartFinished_" + showedCrawl + "_generalDepth", record.attributes().get("generalDepth"));
+                prop.put("otherCrawlStartFinished_" + showedCrawl + "_crawlingQ", (record.attributes().get("crawlingQ").equals("true")) ? 1 : 0);
+                showedCrawl++;
+                if (showedCrawl > 20) break;
             }
-        } catch (IOException e) {}
+        }
         prop.put("otherCrawlStartFinished", showedCrawl);
 
         
diff --git a/htroot/Network.java b/htroot/Network.java
index 895f2a2f4..094866d13 100644
--- a/htroot/Network.java
+++ b/htroot/Network.java
@@ -46,10 +46,10 @@
 // javac -classpath .:../classes Network.java
 // if the shell's current path is HTROOT
 
-import java.io.IOException;
 import java.util.Enumeration;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.Iterator;
 import java.util.Map;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
@@ -269,25 +269,22 @@ public class Network {
                     final HashMap updatedWiki = new HashMap();
                     final HashMap updatedBlog = new HashMap();
                     final HashMap isCrawling = new HashMap();
-                    int availableNews = yacyCore.newsPool.size(yacyNewsPool.INCOMING_DB);
-                    if (availableNews > 300) { availableNews = 300; }
                     yacyNewsRecord record;
-                    try {
-                        for (int c = availableNews - 1; c >= 0; c--) {
-                            record = yacyCore.newsPool.get(yacyNewsPool.INCOMING_DB, c);
-                            if (record == null) {
-                                break;
-                            } else if (record.category().equals(yacyNewsPool.CATEGORY_PROFILE_UPDATE)) {
-                                updatedProfile.add(record.originator());
-                            } else if (record.category().equals(yacyNewsPool.CATEGORY_WIKI_UPDATE)) {
-                                updatedWiki.put(record.originator(), record.attributes());
-                            } else if (record.category().equals(yacyNewsPool.CATEGORY_BLOG_ADD)) {
-                                updatedBlog.put(record.originator(), record.attributes());
-                            } else if (record.category().equals(yacyNewsPool.CATEGORY_CRAWL_START)) {
-                                isCrawling.put(record.originator(), record.attributes().get("startURL"));
-                            }
+                    Iterator recordIterator = yacyCore.newsPool.recordIterator(yacyNewsPool.INCOMING_DB, true);
+                    while (recordIterator.hasNext()) {
+                        record = (yacyNewsRecord) recordIterator.next();
+                        if (record == null) {
+                            continue;
+                        } else if (record.category().equals(yacyNewsPool.CATEGORY_PROFILE_UPDATE)) {
+                            updatedProfile.add(record.originator());
+                        } else if (record.category().equals(yacyNewsPool.CATEGORY_WIKI_UPDATE)) {
+                            updatedWiki.put(record.originator(), record.attributes());
+                        } else if (record.category().equals(yacyNewsPool.CATEGORY_BLOG_ADD)) {
+                            updatedBlog.put(record.originator(), record.attributes());
+                        } else if (record.category().equals(yacyNewsPool.CATEGORY_CRAWL_START)) {
+                            isCrawling.put(record.originator(), record.attributes().get("startURL"));
                         }
-                    } catch (IOException e) {}
+                    }
 
                     boolean dark = true;
                     yacySeed seed;
diff --git a/htroot/News.java b/htroot/News.java
index dd118fdae..8f3ef766d 100644
--- a/htroot/News.java
+++ b/htroot/News.java
@@ -99,7 +99,7 @@ public class News {
                         yacyCore.newsPool.clear(tableID);
                     } else {
                         while (yacyCore.newsPool.size(tableID) > 0) {
-                            record = yacyCore.newsPool.get(tableID, 0);
+                            record = (yacyNewsRecord) yacyCore.newsPool.recordIterator(tableID, true).next();
                             yacyCore.newsPool.moveOff(tableID, record.id());
                         }
                     }
@@ -127,14 +127,15 @@ public class News {
             if (yacyCore.seedDB == null) {
                 
             } else {
-                int maxCount = yacyCore.newsPool.size(tableID);
-                if (maxCount > 300) maxCount = 300;
-                
+                int maxCount = Math.min(1000, yacyCore.newsPool.size(tableID));
+                Iterator recordIterator = yacyCore.newsPool.recordIterator(tableID, false);
                 yacyNewsRecord record;
                 yacySeed seed;
-                for (int i = 0; i < maxCount; i++) try {
-                    record = yacyCore.newsPool.get(tableID, i);
+                int i = 0;
+                while ((recordIterator.hasNext()) && (i < maxCount)) {
+                    record = (yacyNewsRecord) recordIterator.next();
                     if (record == null) continue;
+                    
                     seed = yacyCore.seedDB.getConnected(record.originator());
                     if (seed == null) seed = yacyCore.seedDB.getDisconnected(record.originator());
                     String category = record.category();
@@ -199,8 +200,10 @@ public class News {
                     prop.put("table_list_" + i + "_link", link);
                     prop.put("table_list_" + i + "_title", title);
                     prop.put("table_list_" + i + "_description", description);
-                } catch (IOException e) {e.printStackTrace();}
-                prop.put("table_list", maxCount);
+                    
+                    i++;
+                }
+                prop.put("table_list", i);
             }
         }
         
diff --git a/htroot/Supporter.html b/htroot/Supporter.html
index bf4005763..32383408d 100644
--- a/htroot/Supporter.html
+++ b/htroot/Supporter.html
@@ -48,7 +48,7 @@
      </div>
     <!-- link end -->
     #{/results}#
-    <p class="afterSupporter">
+    <p class="aftersurftips">
     ...provided by YaCy peers with an URL in their profile. This shows only URLs from peers that are currently online.<br />
     </p>
     <script type="text/javascript">
diff --git a/htroot/Supporter.java b/htroot/Supporter.java
index bcef7e6c7..920d77c1c 100644
--- a/htroot/Supporter.java
+++ b/htroot/Supporter.java
@@ -176,9 +176,10 @@ public class Supporter {
     private static void accumulateVotes(HashMap negativeHashes, HashMap positiveHashes, int dbtype) {
         int maxCount = Math.min(1000, yacyCore.newsPool.size(dbtype));
         yacyNewsRecord record;
-        
-        for (int j = 0; j < maxCount; j++) try {
-            record = yacyCore.newsPool.get(dbtype, j);
+        Iterator recordIterator = yacyCore.newsPool.recordIterator(dbtype, true);
+        int j = 0;
+        while ((recordIterator.hasNext()) && (j++ < maxCount)) {
+            record = (yacyNewsRecord) recordIterator.next();
             if (record == null) continue;
             
             if (record.category().equals(yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD)) {
@@ -196,7 +197,7 @@ public class Supporter {
                     else positiveHashes.put(urlhash, new Integer(i.intValue() + factor));
                 }
             }
-        } catch (IOException e) {e.printStackTrace();}
+        }
     }
     
     private static void accumulateSupporter(
@@ -204,18 +205,20 @@ public class Supporter {
             HashMap negativeHashes, HashMap positiveHashes, int dbtype) {
         int maxCount = Math.min(1000, yacyCore.newsPool.size(dbtype));
         yacyNewsRecord record;
+        Iterator recordIterator = yacyCore.newsPool.recordIterator(dbtype, true);
+        int j = 0;
         String url = "", urlhash;
         kelondroRow.Entry entry;
         int score = 0;
         Integer vote;
         yacySeed seed;
-        for (int j = 0; j < maxCount; j++) try {
-            record = yacyCore.newsPool.get(dbtype, j);
+        while ((recordIterator.hasNext()) && (j++ < maxCount)) {
+            record = (yacyNewsRecord) recordIterator.next();
             if (record == null) continue;
             
             entry = null;
             if ((record.category().equals(yacyNewsPool.CATEGORY_PROFILE_UPDATE)) &&
-                ((seed = yacyCore.seedDB.getConnected(record.originator())) != null)) {
+                ((seed = yacyCore.seedDB.getConnected(record.originator())) != null)) try {
                 url = record.attribute("homepage", "");
                 if (url.length() < 12) continue;
                 entry = rowdef.newEntry(new byte[][]{
@@ -225,10 +228,10 @@ public class Supporter {
                                 record.id().getBytes()
                         });
                 score = 1 + timeFactor(record.created());
-            }
+            } catch (IOException e) {}
 
             if ((record.category().equals(yacyNewsPool.CATEGORY_PROFILE_BROADCAST)) &&
-                ((seed = yacyCore.seedDB.getConnected(record.originator())) != null)) {
+                ((seed = yacyCore.seedDB.getConnected(record.originator())) != null)) try {
                 url = record.attribute("homepage", "");
                 if (url.length() < 12) continue;
                 entry = rowdef.newEntry(new byte[][]{
@@ -238,7 +241,7 @@ public class Supporter {
                                 record.id().getBytes()
                         });
                 score = 1 + timeFactor(record.created());
-            }
+            } catch (IOException e) {}
 
             // add/subtract votes and write record
             if (entry != null) {
@@ -264,7 +267,7 @@ public class Supporter {
                 }
             }
             
-        } catch (IOException e) {e.printStackTrace();}
+        }
         
     }
     
diff --git a/htroot/Surftips.java b/htroot/Surftips.java
index e22afe097..236740bac 100644
--- a/htroot/Surftips.java
+++ b/htroot/Surftips.java
@@ -141,7 +141,7 @@ public class Surftips {
                 if ((url == null) || (title == null) || (description == null)) continue;
                 refid = row.getColString(3, null);
                 voted = false;
-                try {
+                try {  // performance problem if published news is too full
                     voted = (yacyCore.newsPool.getSpecific(yacyNewsPool.OUTGOING_DB, yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, "refid", refid) != null) || 
                     		(yacyCore.newsPool.getSpecific(yacyNewsPool.PUBLISHED_DB, yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, "refid", refid) != null);
                 } catch (IOException e) {
@@ -184,9 +184,10 @@ public class Surftips {
     private static void accumulateVotes(HashMap negativeHashes, HashMap positiveHashes, int dbtype) {
         int maxCount = Math.min(1000, yacyCore.newsPool.size(dbtype));
         yacyNewsRecord record;
-        
-        for (int j = 0; j < maxCount; j++) try {
-            record = yacyCore.newsPool.get(dbtype, j);
+        Iterator recordIterator = yacyCore.newsPool.recordIterator(dbtype, true);
+        int j = 0;
+        while ((recordIterator.hasNext()) && (j++ < maxCount)) {
+            record = (yacyNewsRecord) recordIterator.next();
             if (record == null) continue;
             
             if (record.category().equals(yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD)) {
@@ -204,7 +205,7 @@ public class Surftips {
                     else positiveHashes.put(urlhash, new Integer(i.intValue() + factor));
                 }
             }
-        } catch (IOException e) {e.printStackTrace();}
+        }
     }
     
     private static void accumulateSurftips(
@@ -212,16 +213,18 @@ public class Surftips {
             HashMap negativeHashes, HashMap positiveHashes, int dbtype) {
         int maxCount = Math.min(1000, yacyCore.newsPool.size(dbtype));
         yacyNewsRecord record;
+        Iterator recordIterator = yacyCore.newsPool.recordIterator(dbtype, true);
+        int j = 0;
         String url = "", urlhash;
         kelondroRow.Entry entry;
         int score = 0;
         Integer vote;
-        for (int j = 0; j < maxCount; j++) try {
-            record = yacyCore.newsPool.get(dbtype, j);
+        while ((recordIterator.hasNext()) && (j++ < maxCount)) {
+            record = (yacyNewsRecord) recordIterator.next();
             if (record == null) continue;
             
             entry = null;
-            if (record.category().equals(yacyNewsPool.CATEGORY_CRAWL_START)) {
+            if (record.category().equals(yacyNewsPool.CATEGORY_CRAWL_START)) try {
                 String intention = record.attribute("intention", "");
                 url = record.attribute("startURL", "");
                 if (url.length() < 12) continue;
@@ -232,9 +235,9 @@ public class Surftips {
                                 record.id().getBytes()
                         });
                 score = 2 + Math.min(10, intention.length() / 4) + timeFactor(record.created());
-            }
+            } catch (IOException e) {}
             
-            if (record.category().equals(yacyNewsPool.CATEGORY_BOOKMARK_ADD)) {
+            if (record.category().equals(yacyNewsPool.CATEGORY_BOOKMARK_ADD)) try {
                 url = record.attribute("url", "");
                 if (url.length() < 12) continue;
                 entry = rowdef.newEntry(new byte[][]{
@@ -244,9 +247,9 @@ public class Surftips {
                                 record.id().getBytes()
                         });
                 score = 8 + timeFactor(record.created());
-            }
+            } catch (IOException e) {}
             
-            if (record.category().equals(yacyNewsPool.CATEGORY_SURFTIPP_ADD)) {
+            if (record.category().equals(yacyNewsPool.CATEGORY_SURFTIPP_ADD)) try {
                 url = record.attribute("url", "");
                 if (url.length() < 12) continue;
                 entry = rowdef.newEntry(new byte[][]{
@@ -256,9 +259,9 @@ public class Surftips {
                                 record.id().getBytes()
                         });
                 score = 5 + timeFactor(record.created());
-            }
+            } catch (IOException e) {}
             
-            if (record.category().equals(yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD)) {
+            if (record.category().equals(yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD)) try {
                 if (!(record.attribute("vote", "negative").equals("positive"))) continue;
                 url = record.attribute("url", "");
                 if (url.length() < 12) continue;
@@ -269,9 +272,9 @@ public class Surftips {
                                 record.attribute("refid", "").getBytes()
                         });
                 score = 5 + timeFactor(record.created());
-            }
+            } catch (IOException e) {}
             
-            if (record.category().equals(yacyNewsPool.CATEGORY_WIKI_UPDATE)) {
+            if (record.category().equals(yacyNewsPool.CATEGORY_WIKI_UPDATE)) try {
                 yacySeed seed = yacyCore.seedDB.getConnected(record.originator());
                 if (seed == null) seed = yacyCore.seedDB.getDisconnected(record.originator());
                 if (seed != null) {
@@ -284,9 +287,9 @@ public class Surftips {
                         });
                     score = 4 + timeFactor(record.created());
                 }
-            }
+            } catch (IOException e) {}
             
-            if (record.category().equals(yacyNewsPool.CATEGORY_BLOG_ADD)) {
+            if (record.category().equals(yacyNewsPool.CATEGORY_BLOG_ADD)) try {
                 yacySeed seed = yacyCore.seedDB.getConnected(record.originator());
                 if (seed == null) seed = yacyCore.seedDB.getDisconnected(record.originator());
                 if (seed != null) {
@@ -299,7 +302,7 @@ public class Surftips {
                         });
                     score = 4 + timeFactor(record.created());
                 }
-            }
+            } catch (IOException e) {}
 
             // add/subtract votes and write record
             if (entry != null) {
@@ -325,7 +328,7 @@ public class Surftips {
                 }
             }
             
-        } catch (IOException e) {e.printStackTrace();}
+        }
         
     }
     
diff --git a/source/de/anomic/kelondro/kelondroStack.java b/source/de/anomic/kelondro/kelondroStack.java
index ef826412c..76ca8bf0f 100644
--- a/source/de/anomic/kelondro/kelondroStack.java
+++ b/source/de/anomic/kelondro/kelondroStack.java
@@ -107,12 +107,20 @@ public final class kelondroStack extends kelondroRecords {
         return open(f, row);
     }
 
-    public class Counter implements Iterator {
+    public Iterator iterator(boolean up) {
+        // iterates the elements in an ordered way.
+        // returns kelondroRecords.Node - type Objects
+        return new nodeIterator(up);
+    }
+
+    public class nodeIterator implements Iterator {
         Handle nextHandle = null;
         Handle lastHandle = null;
+        boolean up;
 
-        public Counter() {
-            nextHandle = getHandle(root);
+        public nodeIterator(boolean up) {
+            this.up = up;
+            nextHandle = getHandle((up) ? root : toor);
         }
 
         public boolean hasNext() {
@@ -122,7 +130,7 @@ public final class kelondroStack extends kelondroRecords {
         public Object next() {
         	lastHandle = nextHandle;
             try {
-                nextHandle = getNode(nextHandle, null, 0, false).getOHHandle(right);
+                nextHandle = getNode(nextHandle, null, 0, false).getOHHandle((up) ? right : left);
                 return getNode(lastHandle, null, 0, true);
             } catch (IOException e) {
                 throw new kelondroException(filename, "IO error at Counter:next()");
@@ -288,39 +296,6 @@ public final class kelondroStack extends kelondroRecords {
         if (h == null) return null; else return getNode(h, true);
     }
 
-    public Iterator iterator() {
-        // iterates the elements in an ordered way.
-        // returns Node - type Objects
-        return new Counter();
-    }
-
-    public Iterator keyIterator() {
-        // iterates byte[] - objects
-        return new keyIterator(iterator());
-    }
-    
-    public class keyIterator implements Iterator {
-
-        Iterator ni;
-        
-        public keyIterator(Iterator i) {
-            ni = i;
-        }
-        
-        public boolean hasNext() {
-            return ni.hasNext();
-        }
-        
-        public Object next() {
-            return ((kelondroRecords.Node) ni.next()).getKey();
-        }
-        
-        public void remove() {
-            ni.remove();
-        }
-        
-    }
-    
     public int imp(File file, String separator) throws IOException {
         // imports a value-separated file, returns number of records that have been read
         RandomAccessFile f = null;
@@ -368,7 +343,7 @@ public final class kelondroStack extends kelondroRecords {
         super.print(false);
         Node n;
         try {
-            Iterator it = iterator();
+            Iterator it = iterator(true);
             kelondroRow.Entry r;
             while (it.hasNext()) {
                 n = (Node) it.next();
diff --git a/source/de/anomic/plasma/plasmaCrawlBalancer.java b/source/de/anomic/plasma/plasmaCrawlBalancer.java
index bb3d3e069..27dade343 100644
--- a/source/de/anomic/plasma/plasmaCrawlBalancer.java
+++ b/source/de/anomic/plasma/plasmaCrawlBalancer.java
@@ -166,7 +166,7 @@ public class plasmaCrawlBalancer {
        
        // iterate through the file stack
        // in general this is a bad idea. But this can only be avoided by avoidance of this method
-       i = urlFileStack.iterator();
+       i = urlFileStack.iterator(true);
        while (i.hasNext()) {
            h = new String(((kelondroRecords.Node) i.next()).getKey());
            if (h.equals(urlhash)) {
diff --git a/source/de/anomic/plasma/plasmaSwitchboard.java b/source/de/anomic/plasma/plasmaSwitchboard.java
index 39896c2fe..f7c345fb0 100644
--- a/source/de/anomic/plasma/plasmaSwitchboard.java
+++ b/source/de/anomic/plasma/plasmaSwitchboard.java
@@ -1974,6 +1974,12 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
                 log.logFine("Cleaning Incoming News, " + yacyCore.newsPool.size(yacyNewsPool.INCOMING_DB) + " entries on stack");
                 if (yacyCore.newsPool.automaticProcess() > 0) hasDoneSomething = true;
             } catch (IOException e) {}
+            if (getConfigBool("cleanup.deletionProcessedNews", true)) {
+                yacyCore.newsPool.clear(yacyNewsPool.PROCESSED_DB);
+            }
+            if (getConfigBool("cleanup.deletionPublishedNews", true)) {
+                yacyCore.newsPool.clear(yacyNewsPool.PUBLISHED_DB);
+            }
             
             // clean up seed-dbs
             if(getConfigBool("routing.deleteOldSeeds.permission",true)) {
diff --git a/source/de/anomic/yacy/yacyNewsPool.java b/source/de/anomic/yacy/yacyNewsPool.java
index bbbcc6f76..1c2655e0e 100644
--- a/source/de/anomic/yacy/yacyNewsPool.java
+++ b/source/de/anomic/yacy/yacyNewsPool.java
@@ -47,6 +47,7 @@ package de.anomic.yacy;
 import java.io.File;
 import java.io.IOException;
 import java.util.HashSet;
+import java.util.Iterator;
 import java.util.Map;
 
 import de.anomic.net.URL;
@@ -286,6 +287,12 @@ public class yacyNewsPool {
         return newsDB.size();
     }
     
+    public Iterator recordIterator(int dbKey, boolean up) {
+        // returns an iterator of yacyNewsRecord-type objects
+        yacyNewsQueue queue = switchQueue(dbKey);
+        return queue.records(up);
+    }
+    
     public void publishMyNews(yacyNewsRecord record) {
         // this shall be called if our peer generated a new news record and wants to publish it
         try {
@@ -395,6 +402,7 @@ public class yacyNewsPool {
         return false;
     }
     
+    /*
     public yacyNewsRecord get(int dbKey, int element) throws IOException {
         yacyNewsQueue queue = switchQueue(dbKey);
         yacyNewsRecord record = null;
@@ -410,6 +418,7 @@ public class yacyNewsPool {
         }
         return record;
     }
+    */
     
     public synchronized yacyNewsRecord getSpecific(int dbKey, String category, String key, String value) throws IOException {
         yacyNewsQueue queue = switchQueue(dbKey);
diff --git a/source/de/anomic/yacy/yacyNewsQueue.java b/source/de/anomic/yacy/yacyNewsQueue.java
index a46116bd3..16d1b7da0 100644
--- a/source/de/anomic/yacy/yacyNewsQueue.java
+++ b/source/de/anomic/yacy/yacyNewsQueue.java
@@ -47,9 +47,11 @@ package de.anomic.yacy;
 import java.io.File;
 import java.io.IOException;
 import java.util.Date;
+import java.util.Iterator;
 
 import de.anomic.kelondro.kelondroColumn;
 import de.anomic.kelondro.kelondroNaturalOrder;
+import de.anomic.kelondro.kelondroRecords;
 import de.anomic.kelondro.kelondroRow;
 import de.anomic.kelondro.kelondroStack;
 
@@ -62,8 +64,8 @@ public class yacyNewsQueue {
     public static final kelondroRow rowdef = new kelondroRow(new kelondroColumn[]{
             new kelondroColumn("newsid", kelondroColumn.celltype_string, kelondroColumn.encoder_bytes, yacyNewsRecord.idLength, "id = created + originator"),
             new kelondroColumn("last touched", kelondroColumn.celltype_string, kelondroColumn.encoder_bytes, yacyCore.universalDateShortPattern.length(), "")
-    },
-    kelondroNaturalOrder.naturalOrder, 0
+        },
+        kelondroNaturalOrder.naturalOrder, 0
     );
 
     public yacyNewsQueue(File path, yacyNewsDB newsDB) {
@@ -160,5 +162,38 @@ public class yacyNewsQueue {
                 yacyCore.universalDateShortString(new Date()).getBytes()});
         return b;
     }
+    
+    public Iterator records(boolean up) {
+        return new newsIterator(up);
+    }
+    
+    public class newsIterator implements Iterator {
+        // iterates yacyNewsRecord-type objects
+        
+        Iterator stackNodeIterator;
+        
+        public newsIterator(boolean up) {
+            stackNodeIterator = queueStack.iterator(up);
+        }
+        
+        public boolean hasNext() {
+            return stackNodeIterator.hasNext();
+        }
+
+        public Object next() {
+            kelondroRecords.Node n = (kelondroRecords.Node) stackNodeIterator.next();
+            try {
+                kelondroRow.Entry entry = queueStack.row().newEntry(n.getValueRow());
+                return b2r(entry);
+            } catch (IOException e) {
+                return null;
+            }
+        }
+
+        public void remove() {
+            stackNodeIterator.remove();
+        }
+        
+    }
 
 }
\ No newline at end of file
diff --git a/yacy.init b/yacy.init
index d4453f55b..9e293609d 100644
--- a/yacy.init
+++ b/yacy.init
@@ -532,6 +532,11 @@ filterOutStopwordsFromTopwords=true
 90_cleanup_busysleep=300000
 90_cleanup_memprereq=0
 
+# cleanup-process:
+# properties for tasks that are performed during cleanup
+cleanup.deletionProcessedNews = true
+cleanup.deletionPublishedNews = true
+
 # multiprocessor-settings
 # you may want to run time-consuming processes on several processors
 # the most time-consuming process is the indexing-Process