From 60117a2073f74751b632409eb41fdb41d1677d2f Mon Sep 17 00:00:00 2001
From: orbiter <orbiter@6c8d7289-2bf4-0310-a012-ef5d649a1542>
Date: Fri, 29 Jul 2005 14:10:19 +0000
Subject: [PATCH] integrated YaCyNews basis

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@455 6c8d7289-2bf4-0310-a012-ef5d649a1542
---
 doc/Download.html                         |   6 +-
 doc/Links.html                            |   3 +
 htroot/Help.html                          |   3 +-
 htroot/News.html                          |  67 ++++++++
 htroot/News.java                          |  96 ++++++++++++
 htroot/env/templates/header.template      |   3 +-
 source/de/anomic/yacy/yacyCore.java       |  15 ++
 source/de/anomic/yacy/yacyNewsAction.java |  70 +++++++++
 source/de/anomic/yacy/yacyNewsDB.java     | 177 ++++++++++++++++++++++
 source/de/anomic/yacy/yacyNewsPool.java   | 123 +++++++++++++++
 source/de/anomic/yacy/yacyNewsQueue.java  | 152 +++++++++++++++++++
 source/de/anomic/yacy/yacyNewsRecord.java | 149 ++++++++++++++++++
 12 files changed, 859 insertions(+), 5 deletions(-)
 create mode 100644 htroot/News.html
 create mode 100644 htroot/News.java
 create mode 100644 source/de/anomic/yacy/yacyNewsAction.java
 create mode 100644 source/de/anomic/yacy/yacyNewsDB.java
 create mode 100644 source/de/anomic/yacy/yacyNewsPool.java
 create mode 100644 source/de/anomic/yacy/yacyNewsQueue.java
 create mode 100644 source/de/anomic/yacy/yacyNewsRecord.java

diff --git a/doc/Download.html b/doc/Download.html
index 29fac61f3..8e6607b04 100644
--- a/doc/Download.html
+++ b/doc/Download.html
@@ -43,12 +43,12 @@ Download Mirrors and YaCy Flavours:
   <li>Generic release of YaCy (all platforms with J2SE 1.4.2: Linux, Mac OS X, Windows, Solaris):</li>
   <ul>
     <li><tt>from yacy.net&nbsp;&nbsp;&nbsp;: <a href="http://www.yacy.net/yacy/release/yacy_v0.391_20050726_434.tar.gz"><tt>yacy_v0.391_20050726_434.tar.gz</tt></a></tt></li>
-    <li><tt>from BerliOS.de&nbsp;: <a href="http://download.berlios.de/yacy/yacy_v0.39_20050722_425.tar.gz"><tt>yacy_v0.39_20050722_425.tar.gz</tt></a></tt></li>
+    <li><tt>from BerliOS.de&nbsp;: <a href="http://download.berlios.de/yacy/yacy_v0.391_20050726_434.tar.gz"><tt>yacy_v0.391_20050726_434.tar.gz</tt></a></tt></li>
   </ul>
   <li>Windows-flavour release of YaCy (same code as generic release, but with convenient Windows-Installer):</li>
   <ul>
-    <li><tt>from yacy.net&nbsp;&nbsp;&nbsp;: <a href="http://www.yacy.net/yacy/release/yacy_v0.39_20050722_425.exe"><tt>yacy_v0.39_20050722_425.exe</tt></a></tt></li>
-    <li><tt>from BerliOS.de&nbsp;: <a href="http://download.berlios.de/yacy/yacy_v0.39_20050722_425.exe"><tt>yacy_v0.39_20050722_425.exe</tt></a></tt></li>
+    <li><tt>from yacy.net&nbsp;&nbsp;&nbsp;: <a href="http://www.yacy.net/yacy/release/yacy_v0.391_20050726_434.exe"><tt>yacy_v0.391_20050726_434.exe</tt></a></tt></li>
+    <li><tt>from BerliOS.de&nbsp;: <a href="http://download.berlios.de/yacy/yacy_v0.391_20050726_434.exe"><tt>yacy_v0.391_20050726_434.exe</tt></a></tt></li>
   </ul>
 </ul>
 </p>
diff --git a/doc/Links.html b/doc/Links.html
index c9335f42b..5ac5ec7ae 100644
--- a/doc/Links.html
+++ b/doc/Links.html
@@ -34,6 +34,9 @@ globalheader();
 
 <p>Publications about YaCy
 <ul>
+<li>Article in the german computer magazine <a href="http://www.heise.de/ct/"><b>c't</b></a>,
+  issue <a href="http://www.heise.de/ct/05/16/006/">c't 16/2005</a>,
+  page 52 (german): "<a href="http://www.heise.de/ct/05/16/052/"><b>Wider die Monokultur</b></a>"</li>
 <li>Article in <a href="http://www.ccc.de/updates/2005/ds86">Datenschleuder #086</a>,
  magazine of the german 'chaos computer association' (<b>Chaos Computer Club</b>, <b>CCC</b>), german:
  <a href="http://www.yacy.net/yacy/material/YaCy-Datenschleuder086.pdf"><b>"YaCy -- Peer-to-Peer Web-Suchmaschine"</b></a></li>
diff --git a/htroot/Help.html b/htroot/Help.html
index 461f6dfcd..2875acc61 100644
--- a/htroot/Help.html
+++ b/htroot/Help.html
@@ -63,7 +63,8 @@ You may want to use accesskeys to navigate through the YaCy webinterface:<p>
     <li> All OS and Amaya: Strg + Accesskey<p>
 	
 s --> Search Page<br>
-n --> Network<br>
+n --> News<br>
+w --> Network<br>
 t --> Status<br>
 
 #[footer]#
diff --git a/htroot/News.html b/htroot/News.html
new file mode 100644
index 000000000..5e0d166e0
--- /dev/null
+++ b/htroot/News.html
@@ -0,0 +1,67 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN" "http://www.w3.org/TR/REC-html40/loose.dtd">
+<html>
+<head>
+<title>YaCy: News</title>
+#[metas]#
+</head>
+<body marginheight="0" marginwidth="0" leftmargin="0" topmargin="0">
+#[header]#
+<table width="100%" border="0" cellpadding="0" cellspacing="0" class="SubMenu">
+<tr height="10"><td colspan="9" class="MenuHeader">&nbsp;Network Menu</td></tr>
+<tr height="2"><td colspan="9"></td></tr>
+<tr class="TableHeader">
+<td width="20%" class="MenuSubItem">&nbsp;&nbsp;<a href="/News.html" class="MenuItemLink">News&nbsp;Overview</a>&nbsp;</td>
+<td class="MenuSubSpacer"></td>
+<td width="20%" class="MenuSubItem">&nbsp;&nbsp;<a href="/News.html?page=1" class="MenuItemLink">Incoming&nbsp;News</a>&nbsp;</td>
+<td class="MenuSubSpacer"></td>
+<td width="20%" class="MenuSubItem">&nbsp;&nbsp;<a href="/News.html?page=2" class="MenuItemLink">Processed&nbsp;News</a>&nbsp;</td>
+<td class="MenuSubSpacer"></td>
+<td width="20%" class="MenuSubItem">&nbsp;&nbsp;<a href="/News.html?page=3" class="MenuItemLink">Outgoing&nbsp;News</a>&nbsp;</td>
+<td class="MenuSubSpacer"></td>
+<td width="20%" class="MenuSubItem">&nbsp;&nbsp;<a href="/News.html?page=4" class="MenuItemLink">Published&nbsp;News</a>&nbsp;</td>
+</tr>
+</table>
+<br>
+#(page)#
+<h2>News Overview</h2>
+::
+<h2>Incoming&nbsp;News</h2>
+::
+<h2>Processed&nbsp;News</h2>
+::
+<h2>Outgoing&nbsp;News</h2>
+::
+<h2>Published&nbsp;News</h2>
+#(/page)#
+<p>
+#(table)#
+<p>
+This is the news system (currently under testing).
+</p>
+::
+<p>
+<table border="0" cellpadding="2" cellspacing="1">
+<tr class="TableHeader" valign="bottom">
+<td class="small">Originator</td>
+<td class="small">Created</td>
+<td class="small">Category</td>
+<td class="small">Received</td>
+<td class="small">Distributed</td>
+<td class="small">Attributes</td>
+</tr>
+#{list}#
+<tr class="TableCell#(dark)#Light::Dark::Summary#(/dark)#">
+<td class="small" align="right">#[ori]#</td>
+<td class="small" align="right">#[cre]#</td>
+<td class="small" align="right">#[cat]#</td>
+<td class="small" align="right">#[rec]#</td>
+<td class="small" align="right">#[dis]#</td>
+<td class="small" align="right">#[att]#</td>
+</tr>
+#{/list}#
+</table>
+</p>
+#(/table)#
+#[footer]#
+</body>
+</html>
diff --git a/htroot/News.java b/htroot/News.java
new file mode 100644
index 000000000..cc16a8708
--- /dev/null
+++ b/htroot/News.java
@@ -0,0 +1,96 @@
+// News.java
+// -----------------------
+// part of YaCy
+// (C) by Michael Peter Christen; mc@anomic.de
+// first published on http://www.anomic.de
+// Frankfurt, Germany, 2005
+// last major change: 29.07.2005
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program; if not, write to the Free Software
+// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+//
+// Using this software in any meaning (reading, learning, copying, compiling,
+// running) means that you agree that the Author(s) is (are) not responsible
+// for cost, loss of data or any harm that may be caused directly or indirectly
+// by usage of this softare or this documentation. The usage of this software
+// is on your own risk. The installation and usage (starting/running) of this
+// software may allow other people or application to access your computer and
+// any attached devices and is highly dependent on the configuration of the
+// software which must be done by the user of the software; the author(s) is
+// (are) also not responsible for proper configuration and usage of the
+// software, even if provoked by documentation provided together with
+// the software.
+//
+// Any changes to this file according to the GPL as documented in the file
+// gpl.txt aside this file in the shipment you received can be done to the
+// lines that follows this copyright notice here, but changes must not be
+// done inside the copyright notive above. A re-distribution must contain
+// the intact and unchanged copyright notice.
+// Contributions and changes to the program code must be marked as such.
+
+// You must compile this file with
+// javac -classpath .:../classes Network.java
+// if the shell's current path is HTROOT
+
+import java.util.Enumeration;
+import java.util.HashMap;
+import java.io.IOException;
+
+import de.anomic.http.httpHeader;
+import de.anomic.server.serverObjects;
+import de.anomic.server.serverSwitch;
+import de.anomic.server.serverDate;
+import de.anomic.yacy.yacyCore;
+import de.anomic.yacy.yacyNewsRecord;
+
+public class News {
+    
+    public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch sb) {
+        // return variable that accumulates replacements
+        serverObjects prop = new serverObjects();
+        boolean overview = (post == null) || (((String) post.get("page", "0")).equals("0"));
+        
+        if (overview) {
+            // show overview
+            prop.put("table", 0);
+            prop.put("page", 0);
+        } else {
+            // generate table
+            int tableID = Integer.parseInt((String) post.get("page", "1")) - 1;
+            prop.put("table", 1);
+            prop.put("page", tableID + 1);
+            
+            if (yacyCore.seedDB == null) {
+                
+            } else {
+                int maxCount = yacyCore.newsPool.size(tableID);
+                if (maxCount > 500) maxCount = 500;
+                
+                yacyNewsRecord record;
+                for (int i = 0; i < maxCount; i++) try {
+                    record = yacyCore.newsPool.get(tableID, i);
+                    prop.put("table_list_" + i + "_ori", record.originator());
+                    prop.put("table_list_" + i + "_cre", yacyCore.universalDateShortString(record.created()));
+                    prop.put("table_list_" + i + "_cat", record.category());
+                    prop.put("table_list_" + i + "_rec", record.received());
+                    prop.put("table_list_" + i + "_dis", record.attributes().toString());
+                } catch (IOException e) {e.printStackTrace();}
+                prop.put("table_list", maxCount);
+            }
+        }
+        // return rewrite properties
+        return prop;
+    }
+
+}
diff --git a/htroot/env/templates/header.template b/htroot/env/templates/header.template
index 479f396ef..14515d4c5 100644
--- a/htroot/env/templates/header.template
+++ b/htroot/env/templates/header.template
@@ -59,7 +59,8 @@
   
   <tr><td class="MenuHeader">&nbsp;Peer&nbsp;Control</td></tr>
   <tr><td class="MenuItem">&nbsp;<a href="/Status.html" accesskey="t" class="MenuItemLink">Status</a></td></tr>
-  <tr><td class="MenuItem">&nbsp;<a href="/Network.html" accesskey="n" class="MenuItemLink">Network</a></td></tr>
+  <tr><td class="MenuItem">&nbsp;<a href="/News.html" accesskey="n" class="MenuItemLink">News</a></td></tr>
+  <tr><td class="MenuItem">&nbsp;<a href="/Network.html" accesskey="w" class="MenuItemLink">Network</a></td></tr>
   <tr><td class="MenuItem">&nbsp;<img border="0" src="/env/grafics/lock.gif" align="top">&nbsp;<a href="/ViewLog_p.html" class="MenuItemLink">Log</a></td></tr>
   <tr><td class="MenuItem">&nbsp;<img border="0" src="/env/grafics/lock.gif" align="top">&nbsp;<a href="/Settings_p.html" class="MenuItemLink">Settings</a></td></tr>
   <tr><td class="MenuItem">&nbsp;<img border="0" src="/env/grafics/lock.gif" align="top">&nbsp;<a href="/Performance_p.html" class="MenuItemLink">Performance</a></td></tr>
diff --git a/source/de/anomic/yacy/yacyCore.java b/source/de/anomic/yacy/yacyCore.java
index 98e1fd671..79c5e22d5 100644
--- a/source/de/anomic/yacy/yacyCore.java
+++ b/source/de/anomic/yacy/yacyCore.java
@@ -83,6 +83,7 @@ public class yacyCore {
     public static ThreadGroup publishThreadGroup = new ThreadGroup("publishThreadGroup");
     public static long startupTime = System.currentTimeMillis();
     public static yacySeedDB seedDB = null;
+    public static yacyNewsPool newsPool = null;
     public static final Hashtable seedUploadMethods = new Hashtable();
     public static yacyPeerActions peerActions = null;
     public static yacyDHTAction dhtAgent = null;
@@ -164,13 +165,18 @@ public class yacyCore {
                 new File(yacyDBPath, "seed.pot.db"),
                 mem);
         
+        // create or init news database
+        newsPool = new yacyNewsPool(yacyDBPath, 1024);
+        
         loadSeedUploadMethods();
         
+        // deploy peer actions
         peerActions = new yacyPeerActions(seedDB, switchboard,
                 new File(sb.getRootPath(), sb.getConfig("superseedFile", "superseed.txt")),
                 switchboard.getConfig("superseedLocation", "http://www.yacy.net/yacy/superseed.txt"));
         dhtAgent = new yacyDHTAction(seedDB);
         peerActions.deploy(dhtAgent);
+        peerActions.deploy(new yacyNewsAction(newsPool));
         
         // create or init index sharing
         //shareManager = new yacyShare(switchboard);
@@ -277,6 +283,7 @@ public class yacyCore {
         // before publishing, update some seed data
         peerActions.updateMySeed();
         
+        
         // publish own seed to other peer, this can every peer, but makes only sense for senior peers
         int oldSize = seedDB.sizeConnected();
         if (oldSize == 0) {
@@ -380,6 +387,14 @@ public class yacyCore {
             }
             if (seeds == null) return 0;
             
+            // include a YaCyNews record to my seed
+            try {
+                yacyNewsRecord record = newsPool.dequeueMyNews();
+                if (record != null) seedDB.mySeed.put("news", de.anomic.tools.crypt.simpleEncode(record.toString()));
+            } catch (IOException e) {
+                e.printStackTrace();
+            }
+            
             // holding a reference to all started threads
             int contactedSeedCount = 0;
             List syncList = Collections.synchronizedList(new LinkedList()); // memory for threads
diff --git a/source/de/anomic/yacy/yacyNewsAction.java b/source/de/anomic/yacy/yacyNewsAction.java
new file mode 100644
index 000000000..4b1817a91
--- /dev/null
+++ b/source/de/anomic/yacy/yacyNewsAction.java
@@ -0,0 +1,70 @@
+// yacyNewsAction.java
+// -----------------------
+// part of YaCy
+// (C) by Michael Peter Christen; mc@anomic.de
+// first published on http://www.anomic.de
+// Frankfurt, Germany, 2005
+// last major change: 29.07.2005
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program; if not, write to the Free Software
+// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+//
+// Using this software in any meaning (reading, learning, copying, compiling,
+// running) means that you agree that the Author(s) is (are) not responsible
+// for cost, loss of data or any harm that may be caused directly or indirectly
+// by usage of this softare or this documentation. The usage of this software
+// is on your own risk. The installation and usage (starting/running) of this
+// software may allow other people or application to access your computer and
+// any attached devices and is highly dependent on the configuration of the
+// software which must be done by the user of the software; the author(s) is
+// (are) also not responsible for proper configuration and usage of the
+// software, even if provoked by documentation provided together with
+// the software.
+//
+// Any changes to this file according to the GPL as documented in the file
+// gpl.txt aside this file in the shipment you received can be done to the
+// lines that follows this copyright notice here, but changes must not be
+// done inside the copyright notice above. A re-distribution must contain
+// the intact and unchanged copyright notice.
+// Contributions and changes to the program code must be marked as such.
+
+package de.anomic.yacy;
+
+import java.io.IOException;
+
+public class yacyNewsAction implements yacyPeerAction {
+    
+    yacyNewsPool pool;
+    
+    public yacyNewsAction(yacyNewsPool pool) {
+        this.pool = pool;
+    }
+    
+    public void processPeerArrival(yacySeed peer, boolean direct) {
+        String recordString = peer.get("news", null);
+        if (recordString == null) return;
+        yacyNewsRecord record = new yacyNewsRecord(de.anomic.tools.crypt.simpleDecode(recordString, ""));
+        try {
+            this.pool.enqueueGlobalNews(record);
+        } catch (IOException e) {e.printStackTrace();}
+    }
+    
+    public void processPeerDeparture(yacySeed peer) {
+    }
+    
+    public void processPeerPing(yacySeed peer) {
+        processPeerArrival(peer, true);
+    }
+    
+}
diff --git a/source/de/anomic/yacy/yacyNewsDB.java b/source/de/anomic/yacy/yacyNewsDB.java
new file mode 100644
index 000000000..780a5b118
--- /dev/null
+++ b/source/de/anomic/yacy/yacyNewsDB.java
@@ -0,0 +1,177 @@
+// yacyNewsQueue.java
+// -----------------------
+// part of YaCy
+// (C) by Michael Peter Christen; mc@anomic.de
+// first published on http://www.anomic.de
+// Frankfurt, Germany, 2005
+// last major change: 13.07.2005
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program; if not, write to the Free Software
+// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+//
+// Using this software in any meaning (reading, learning, copying, compiling,
+// running) means that you agree that the Author(s) is (are) not responsible
+// for cost, loss of data or any harm that may be caused directly or indirectly
+// by usage of this softare or this documentation. The usage of this software
+// is on your own risk. The installation and usage (starting/running) of this
+// software may allow other people or application to access your computer and
+// any attached devices and is highly dependent on the configuration of the
+// software which must be done by the user of the software; the author(s) is
+// (are) also not responsible for proper configuration and usage of the
+// software, even if provoked by documentation provided together with
+// the software.
+//
+// Any changes to this file according to the GPL as documented in the file
+// gpl.txt aside this file in the shipment you received can be done to the
+// lines that follows this copyright notice here, but changes must not be
+// done inside the copyright notice above. A re-distribution must contain
+// the intact and unchanged copyright notice.
+// Contributions and changes to the program code must be marked as such.
+
+package de.anomic.yacy;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Iterator;
+
+import de.anomic.yacy.yacyCore;
+import de.anomic.kelondro.kelondroTree;
+import de.anomic.kelondro.kelondroException;
+import de.anomic.kelondro.kelondroRecords;
+import de.anomic.server.serverCodings;
+
+public class yacyNewsDB {
+    
+    private File path;
+    private int bufferkb;
+    private kelondroTree news;
+    
+    public static final int attributesMaxLength = yacyNewsRecord.maxNewsRecordLength
+                 - yacyNewsRecord.idLength()
+                 - yacyNewsRecord.categoryStringLength
+                 - yacyCore.universalDatePattern.length()
+                 - 2;
+    
+    public yacyNewsDB(File path, int bufferkb) throws IOException {
+        this.path = path;
+        this.bufferkb = bufferkb;
+        
+        if (path.exists())
+            news = new kelondroTree(path, bufferkb);
+        else
+            news = createDB(path,bufferkb);        
+    }
+    
+    private static kelondroTree createDB(File path, int bufferkb) throws IOException {
+        return new kelondroTree(path, bufferkb, new int[] {
+                yacyNewsRecord.idLength(), // id = created + originator
+                yacyNewsRecord.categoryStringLength,    // category
+                yacyCore.universalDatePattern.length(), // received
+                2,
+                attributesMaxLength
+            });
+    }
+
+    private void resetDB() throws IOException {
+        try {close();} catch (Exception e) {}
+        if (path.exists()) path.delete();
+        news = createDB(path, bufferkb);
+    }
+    
+    public void close() {
+        if (news != null) try {news.close();} catch (IOException e) {}
+        news = null;
+    }
+
+    public void finalize() {
+        close();
+    }
+    
+    public int size() {
+        return news.size();
+    }
+
+    public synchronized yacyNewsRecord put(yacyNewsRecord record) throws IOException {
+        try {
+            return b2r(news.put(r2b(record)));
+        } catch (kelondroException e) {
+            resetDB();
+            return b2r(news.put(r2b(record)));
+        }
+    }
+    
+    public synchronized Iterator news() throws IOException {
+        // the iteration iterates yacyNewsRecord - type objects
+        return new recordIterator();
+    }
+    
+    public class recordIterator implements Iterator {
+        
+        Iterator nodeIterator;
+        
+        public recordIterator() throws IOException {
+            nodeIterator = news.rows(true, false);
+        }
+        
+        public boolean hasNext() {
+            return nodeIterator.hasNext();
+        }
+        
+        public Object next() {
+            try {
+                return b2r((byte[][]) nodeIterator.next());
+            } catch (IOException e) {
+                return null;
+            }
+        }
+        
+        public void remove() {
+        }
+        
+    }
+    
+    public synchronized yacyNewsRecord get(String id) throws IOException {
+        try {
+            return b2r(news.get(id.getBytes()));
+        } catch (kelondroException e) {
+            resetDB();
+            return null;
+        }
+    }
+
+    private static yacyNewsRecord b2r(byte[][] b) throws IOException {
+        if (b == null) return null;
+        return new yacyNewsRecord(
+            new String(b[0]),
+            new String(b[1]),
+            yacyCore.parseUniversalDate(new String(b[2])),
+            (int) serverCodings.enhancedCoder.decodeBase64Long(new String(b[3])),
+            serverCodings.string2map(new String(b[4]))
+        );
+    }
+
+    private static byte[][] r2b(yacyNewsRecord r) throws IOException {
+        if (r == null) return null;
+        String attributes = r.attributes().toString();
+        if (attributes.length() > attributesMaxLength) throw new IllegalArgumentException("attribute length=" + attributes.length() + " exceeds maximum size=" + attributesMaxLength);
+        byte[][] b = new byte[5][];
+        b[0] = r.id().getBytes();
+        b[1] = r.category().getBytes();
+        b[2] = yacyCore.universalDateShortString(r.created()).getBytes();
+        b[3] = serverCodings.enhancedCoder.encodeBase64Long(r.distributed(), 2).getBytes();
+        b[4] = attributes.getBytes();
+        return b;
+    }
+    
+}
diff --git a/source/de/anomic/yacy/yacyNewsPool.java b/source/de/anomic/yacy/yacyNewsPool.java
new file mode 100644
index 000000000..ab16a66b3
--- /dev/null
+++ b/source/de/anomic/yacy/yacyNewsPool.java
@@ -0,0 +1,123 @@
+// yacyNewsActions.java
+// -----------------------
+// part of YaCy
+// (C) by Michael Peter Christen; mc@anomic.de
+// first published on http://www.anomic.de
+// Frankfurt, Germany, 2005
+// last major change: 13.07.2005
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program; if not, write to the Free Software
+// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+//
+// Using this software in any meaning (reading, learning, copying, compiling,
+// running) means that you agree that the Author(s) is (are) not responsible
+// for cost, loss of data or any harm that may be caused directly or indirectly
+// by usage of this softare or this documentation. The usage of this software
+// is on your own risk. The installation and usage (starting/running) of this
+// software may allow other people or application to access your computer and
+// any attached devices and is highly dependent on the configuration of the
+// software which must be done by the user of the software; the author(s) is
+// (are) also not responsible for proper configuration and usage of the
+// software, even if provoked by documentation provided together with
+// the software.
+//
+// Any changes to this file according to the GPL as documented in the file
+// gpl.txt aside this file in the shipment you received can be done to the
+// lines that follows this copyright notice here, but changes must not be
+// done inside the copyright notice above. A re-distribution must contain
+// the intact and unchanged copyright notice.
+// Contributions and changes to the program code must be marked as such.
+
+package de.anomic.yacy;
+
+import java.io.File;
+import java.io.IOException;
+
+public class yacyNewsPool {
+    
+    public static final int INCOMING_DB  = 0;
+    public static final int PROCESSED_DB = 1;
+    public static final int OUTGOING_DB  = 2;
+    public static final int PUBLISHED_DB = 3;
+    
+    private yacyNewsDB newsDB;
+    private yacyNewsQueue outgoingNews, publishedNews, incomingNews, processedNews;
+    private int maxDistribution;
+    
+    public yacyNewsPool(File yacyDBPath, int bufferkb) throws IOException {
+        newsDB = new yacyNewsDB(new File(yacyDBPath, "news0.db"), bufferkb);
+        outgoingNews  = new yacyNewsQueue(new File(yacyDBPath, "newsOut0.stack"), newsDB);
+        publishedNews = new yacyNewsQueue(new File(yacyDBPath, "newsPublished0.stack"), newsDB);
+        incomingNews  = new yacyNewsQueue(new File(yacyDBPath, "newsIn0.stack"), newsDB);
+        processedNews = new yacyNewsQueue(new File(yacyDBPath, "newsProcessed0.stack"), newsDB);
+        maxDistribution = 30;
+    }
+    
+    public void enqueueMyNews(yacyNewsRecord record) throws IOException {
+        if (newsDB.get(record.id()) == null) outgoingNews.push(record);
+    }
+    
+    public yacyNewsRecord dequeueMyNews() throws IOException {
+        // generate a record for next peer-ping
+        if (outgoingNews.size() == 0) return null;
+        yacyNewsRecord record = outgoingNews.topInc();
+        if (record.distributed() >= maxDistribution) {
+            // move record to its final position. This is only for history
+            publishedNews.push(outgoingNews.pop(0));
+        }
+        return record;
+    }
+    
+    public void enqueueGlobalNews(yacyNewsRecord record) throws IOException {
+        if (newsDB.get(record.id()) == null) incomingNews.push(record);
+    }
+    
+    public yacyNewsRecord getGlobalNews(int job) throws IOException {
+        return incomingNews.top(job);
+    }
+    
+    public synchronized boolean removeGlobalNews(String id) throws IOException {
+        yacyNewsRecord record;
+        for (int i = 0; i < incomingNews.size(); i++) {
+            record = incomingNews.top(i);
+            if (record.id().equals(id)) {
+                incomingNews.pop(i);
+                processedNews.push(record);
+                return true;
+            }
+        }
+        return false;
+    }
+    
+    public int size(int dbKey) {
+        switch (dbKey) {
+            case OUTGOING_DB:   return outgoingNews.size();
+            case PUBLISHED_DB:  return publishedNews.size();
+            case INCOMING_DB:	return incomingNews.size();
+            case PROCESSED_DB:  return processedNews.size();
+            default: return -1;
+        }
+    }
+    
+    public yacyNewsRecord get(int dbKey, int element) throws IOException {
+        switch (dbKey) {
+            case OUTGOING_DB:   return outgoingNews.top(element);
+            case PUBLISHED_DB:  return publishedNews.top(element);
+            case INCOMING_DB:	return incomingNews.top(element);
+            case PROCESSED_DB:  return processedNews.top(element);
+            default: return null;
+        }
+    }
+    
+}
diff --git a/source/de/anomic/yacy/yacyNewsQueue.java b/source/de/anomic/yacy/yacyNewsQueue.java
new file mode 100644
index 000000000..4c39a3440
--- /dev/null
+++ b/source/de/anomic/yacy/yacyNewsQueue.java
@@ -0,0 +1,152 @@
+// yacyNewsQueue.java
+// -----------------------
+// part of YaCy
+// (C) by Michael Peter Christen; mc@anomic.de
+// first published on http://www.anomic.de
+// Frankfurt, Germany, 2005
+// last major change: 13.07.2005
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program; if not, write to the Free Software
+// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+//
+// Using this software in any meaning (reading, learning, copying, compiling,
+// running) means that you agree that the Author(s) is (are) not responsible
+// for cost, loss of data or any harm that may be caused directly or indirectly
+// by usage of this softare or this documentation. The usage of this software
+// is on your own risk. The installation and usage (starting/running) of this
+// software may allow other people or application to access your computer and
+// any attached devices and is highly dependent on the configuration of the
+// software which must be done by the user of the software; the author(s) is
+// (are) also not responsible for proper configuration and usage of the
+// software, even if provoked by documentation provided together with
+// the software.
+//
+// Any changes to this file according to the GPL as documented in the file
+// gpl.txt aside this file in the shipment you received can be done to the
+// lines that follows this copyright notice here, but changes must not be
+// done inside the copyright notice above. A re-distribution must contain
+// the intact and unchanged copyright notice.
+// Contributions and changes to the program code must be marked as such.
+
+package de.anomic.yacy;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.Date;
+
+import de.anomic.htmlFilter.htmlFilterContentScraper;
+import de.anomic.http.httpHeader;
+import de.anomic.kelondro.kelondroStack;
+import de.anomic.server.serverCodings;
+import de.anomic.yacy.yacySeedDB;
+
+public class yacyNewsQueue {
+    
+    private File path;
+    private kelondroStack queueStack;
+    private yacyNewsDB newsDB;
+    
+    public yacyNewsQueue(File path, yacyNewsDB newsDB) throws IOException {
+        this.path = path;
+        this.newsDB = newsDB;
+        
+        if (path.exists())
+            queueStack = new kelondroStack(path, 0);
+        else
+            queueStack = createStack(path);
+    }
+    
+    private static kelondroStack createStack(File path) throws IOException {
+        return new kelondroStack(path, 0, new int[] {
+                yacyNewsRecord.idLength(), // id = created + originator
+                yacyCore.universalDatePattern.length() // last touched
+            });
+    }
+    
+    private void resetDB() throws IOException {
+        try {close();} catch (Exception e) {}
+        if (path.exists()) path.delete();
+        queueStack = createStack(path);
+    }
+    
+    public void close() {
+        if (queueStack != null) try {queueStack.close();} catch (IOException e) {}
+        queueStack = null;
+    }
+
+    public void finalize() {
+        close();
+    }
+    
+    public int size() {
+        return queueStack.size();
+    }
+    
+    public synchronized void push(yacyNewsRecord entry) throws IOException {
+        queueStack.push(r2b(entry, true));
+    }
+    
+    public synchronized yacyNewsRecord pop(int dist) throws IOException {
+        if (queueStack.size() == 0) return null;
+        return b2r(queueStack.pop(dist));
+    }
+
+    public synchronized yacyNewsRecord top(int dist) throws IOException {
+        if (queueStack.size() == 0) return null;
+        return b2r(queueStack.top(dist));
+    }
+    
+    public synchronized yacyNewsRecord topInc() throws IOException {
+        if (queueStack.size() == 0) return null;
+        yacyNewsRecord entry = pop(0);
+        entry.incDistribution();
+        push(entry);
+        return entry;
+    }
+    
+    /*
+    public synchronized void incDistributedCounter(yacyNewsRecord entry) throws IOException {
+        // this works only if the entry element lies ontop of the stack
+        yacyNewsRecord topEntry = top();
+        if (!(topEntry.id().equals(entry.id()))) throw new IllegalArgumentException("entry is not ontop of the stack");
+        pop();
+        entry.incDistribution();
+        push(entry);
+    }
+    */
+    
+    private yacyNewsRecord b2r(byte[][] b) throws IOException {
+        if (b == null) return null;
+        String id = new String(b[0]);
+        //Date touched = yacyCore.parseUniversalDate(new String(b[1]));
+        return newsDB.get(id);
+    }
+
+    private byte[][] r2b(yacyNewsRecord r, boolean updateDB) throws IOException {
+        if (r == null) return null;
+        if (updateDB) {
+            newsDB.put(r);
+        } else {
+            yacyNewsRecord r1 = newsDB.get(r.id());
+            if (r1 == null) newsDB.put(r);
+        }
+        byte[][] b = new byte[2][];
+        b[0] = r.id().getBytes();
+        b[1] = yacyCore.universalDateShortString(new Date()).getBytes();
+        return b;
+    }
+    
+}
diff --git a/source/de/anomic/yacy/yacyNewsRecord.java b/source/de/anomic/yacy/yacyNewsRecord.java
new file mode 100644
index 000000000..e5b8b30bb
--- /dev/null
+++ b/source/de/anomic/yacy/yacyNewsRecord.java
@@ -0,0 +1,149 @@
+// yacyNewsRecord.java
+// -----------------------
+// part of YaCy
+// (C) by Michael Peter Christen; mc@anomic.de
+// first published on http://www.anomic.de
+// Frankfurt, Germany, 2005
+// last major change: 13.07.2005
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program; if not, write to the Free Software
+// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+//
+// Using this software in any meaning (reading, learning, copying, compiling,
+// running) means that you agree that the Author(s) is (are) not responsible
+// for cost, loss of data or any harm that may be caused directly or indirectly
+// by usage of this softare or this documentation. The usage of this software
+// is on your own risk. The installation and usage (starting/running) of this
+// software may allow other people or application to access your computer and
+// any attached devices and is highly dependent on the configuration of the
+// software which must be done by the user of the software; the author(s) is
+// (are) also not responsible for proper configuration and usage of the
+// software, even if provoked by documentation provided together with
+// the software.
+//
+// Any changes to this file according to the GPL as documented in the file
+// gpl.txt aside this file in the shipment you received can be done to the
+// lines that follows this copyright notice here, but changes must not be
+// done inside the copyright notice above. A re-distribution must contain
+// the intact and unchanged copyright notice.
+// Contributions and changes to the program code must be marked as such.
+
+package de.anomic.yacy;
+
+import de.anomic.server.serverCodings;
+
+import java.util.Date;
+import java.util.Map;
+
+public class yacyNewsRecord {
+    
+    public static final int maxNewsRecordLength  = 512;
+    public static final int categoryStringLength = 8;
+    
+    private String originator;  // hash of originating peer
+    private Date   created;     // Date when news was created by originator
+    private Date   received;    // Date when news was received here at this peer
+    private String category;    // keyword that adresses possible actions
+    private int    distributed; // counter that counts number of distributions of this news record
+    private Map    attributes;  // elemets of the news for a special category
+    
+    public yacyNewsRecord(String encodedNewsString) {
+        String decodedString = serverCodings.enhancedCoder.decodeBase64String(encodedNewsString);
+        this.attributes = serverCodings.string2map(decodedString);
+        this.received = (attributes.containsKey("rec")) ? yacyCore.parseUniversalDate((String) attributes.get("rec")) : new Date();
+        this.created = (attributes.containsKey("cre")) ? yacyCore.parseUniversalDate((String) attributes.get("cre")) : new Date();
+        this.category = (attributes.containsKey("cat")) ? (String) attributes.get("cat") : null;
+        this.distributed = (attributes.containsKey("dis")) ? Integer.parseInt((String) attributes.get("dis")) : 0;
+        this.originator = (attributes.containsKey("ori")) ? (String) attributes.get("ori") : null;
+        removeStandards();
+    }
+
+    public yacyNewsRecord(String category, Map attributes) {
+        if (category.length() > categoryStringLength) throw new IllegalArgumentException("category length exceeds maximum");
+        this.attributes = attributes;
+        this.received = null;
+        this.created = new Date();
+        this.category = category;
+        this.distributed = 0;
+        this.originator = yacyCore.seedDB.mySeed.hash;
+        removeStandards();
+    }
+    
+    protected yacyNewsRecord(String id, String category, Date received, int distributed, Map attributes) {
+        this.attributes = attributes;
+        this.received = received;
+        this.created = yacyCore.parseUniversalDate(id.substring(0, yacyCore.universalDatePattern.length()));
+        this.category = category;
+        this.distributed = distributed;
+        this.originator = id.substring(yacyCore.universalDatePattern.length());
+        removeStandards();
+    }
+
+    private void removeStandards() {
+        attributes.remove("ori");
+        attributes.remove("cat");
+        attributes.remove("cre");
+        attributes.remove("rec");
+        attributes.remove("dis");
+    }
+    
+    public String toString() {
+        // this creates the string that shall be distributed
+        // attention: this has no additional encoding
+        if (originator != null) attributes.put("ori", originator);
+        if (category != null) attributes.put("cat", category);
+        attributes.put("cre", yacyCore.universalDateShortString(created));
+        attributes.put("rec", yacyCore.universalDateShortString(received));
+        attributes.put("dis", "" + distributed);
+        String theString = attributes.toString();
+        removeStandards();
+        return theString;
+    }
+    
+    public String id() {
+        return yacyCore.universalDateShortString(created) + originator;
+    }
+    
+    public static int idLength() {
+        return yacyCore.universalDatePattern.length() + yacySeedDB.commonHashLength;
+    }
+    
+    public String originator() {
+        return originator;
+    }
+    
+    public Date created() {
+        return created;
+    }
+    
+    public Date received() {
+        return received;
+    }
+    
+    public String category() {
+        return category;
+    }
+
+    public int distributed() {
+        return distributed;
+    }
+    
+    public void incDistribution() {
+        distributed++;
+    }
+    
+    public Map attributes() {
+        return attributes;
+    }
+}