several bugfixes for YaCyNews

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@465 6c8d7289-2bf4-0310-a012-ef5d649a1542

htroot/Network.html

@@ -68,7 +68,7 @@
 <td class="small"><a href="http://www.#[fullname]#.yacy" class="small">#[shortname]#</a></td>
 #(complete)#
 ::
-<td class="small"><a href="http://#[ip]#:#[port]#/NetworkActive.html?ip=" class="small">http://#[ip]#:#[port]#</a></td>
+<td class="small"><a href="http://#[ip]#:#[port]#/Network.html?page=1&ip=" class="small">http://#[ip]#:#[port]#</a></td>
 <td class=small>#[hash]#</td>
 #(/complete)#
 <td class="small">#(type)#junior::senior::<a href="#[url]#" class="small">principal</a>#(/type)#</td>

htroot/News.java

@@ -66,13 +66,12 @@ public class News {
 
         // execute commands
         if (post != null) {
-            if (switchboard.adminAuthenticated(header) < 2) {
-                // not authenticated, force log-in
-                prop.put("AUTHENTICATE", "admin log-in");
-                return prop;
-            }
             
             if ((post.containsKey("deletespecific")) && (tableID >= 0)) {
+                if (switchboard.adminAuthenticated(header) < 2) {
+                    prop.put("AUTHENTICATE", "admin log-in");
+                    return prop; // this button needs authentication, force log-in
+                }
                 Enumeration e = post.keys();
                 String check;
                 String id;
@@ -88,6 +87,10 @@ public class News {
             }
             
             if ((post.containsKey("deleteall")) && (tableID >= 0)) {
+                if (switchboard.adminAuthenticated(header) < 2) {
+                    prop.put("AUTHENTICATE", "admin log-in");
+                    return prop; // this button needs authentication, force log-in
+                }
                 yacyNewsRecord record;
                 try {
                     while (yacyCore.newsPool.size(tableID) > 0) {

source/de/anomic/kelondro/kelondroDyn.java

@@ -298,7 +298,7 @@ public class kelondroDyn extends kelondroTree {
     }
 
     public synchronized boolean existsDyn(String key) throws IOException {
-	return (getValueCached(dynKey(key, 0)) != null);
+	return (key != null) && (getValueCached(dynKey(key, 0)) != null);
     }
 
     public synchronized kelondroRA getRA(String filekey) throws IOException {

source/de/anomic/kelondro/kelondroMap.java

@@ -196,6 +196,7 @@ public class kelondroMap {
     
     public synchronized void remove(String key) throws IOException {
         // update elementCount
+        if (key == null) return;
         if ((sortfields != null) || (accfields != null)) {
             Map map = get(key);
             if (map != null) {
@@ -219,6 +220,7 @@ public class kelondroMap {
     }
     
     private void deleteSortCluster(String key) {
+        if (key == null) return;
         kelondroMScoreCluster cluster;
         for (int i = 0; i < sortfields.length; i++) {
             cluster = (kelondroMScoreCluster) sortClusterMap.get(sortfields[i]);
@@ -228,6 +230,7 @@ public class kelondroMap {
     }
         
     public synchronized Map get(String key) throws IOException {
+        if (key == null) return null;
         return get(key, true);
     }
     

source/de/anomic/yacy/yacyCore.java

@@ -107,8 +107,8 @@ public class yacyCore {
     private plasmaSwitchboard switchboard;
     
     private static TimeZone GMTTimeZone = TimeZone.getTimeZone("America/Los_Angeles");
-    public static String universalDatePattern = "yyyyMMddHHmmss";
-    public static SimpleDateFormat shortFormatter = new SimpleDateFormat(universalDatePattern);
+    public static String universalDateShortPattern = "yyyyMMddHHmmss";
+    public static SimpleDateFormat shortFormatter = new SimpleDateFormat(universalDateShortPattern);
     
     public static long universalTime() {
 	return universalDate().getTime();

source/de/anomic/yacy/yacyNewsAction.java

@@ -55,8 +55,9 @@ public class yacyNewsAction implements yacyPeerAction {
         String recordString = peer.get("news", null);
         //System.out.println("### triggered news arrival from peer " + peer.getName() + ", news " + ((recordString == null) ? "empty" : "attached"));
         if ((recordString == null) || (recordString.length() == 0)) return;
-        yacyNewsRecord record = new yacyNewsRecord(de.anomic.tools.crypt.simpleDecode(recordString, ""));
-        //System.out.println("### news arrival from peer " + peer.getName() + ", news=" + record.toString());
+        String decodedString = de.anomic.tools.crypt.simpleDecode(recordString, "");
+        yacyNewsRecord record = new yacyNewsRecord(decodedString);
+        System.out.println("### news arrival from peer " + peer.getName() + ", decoded=" + decodedString + ", record=" + recordString + ", news=" + record.toString());
         try {
             this.pool.enqueueIncomingNews(record);
         } catch (IOException e) {e.printStackTrace();}

source/de/anomic/yacy/yacyNewsDB.java

@@ -60,7 +60,7 @@ public class yacyNewsDB {
     public static final int attributesMaxLength = yacyNewsRecord.maxNewsRecordLength
                  - yacyNewsRecord.idLength()
                  - yacyNewsRecord.categoryStringLength
-                 - yacyCore.universalDatePattern.length()
+                 - yacyCore.universalDateShortPattern.length()
                  - 2;
     
     public yacyNewsDB(File path, int bufferkb) throws IOException {
@@ -77,7 +77,7 @@ public class yacyNewsDB {
         return new kelondroTree(path, bufferkb, new int[] {
                 yacyNewsRecord.idLength(), // id = created + originator
                 yacyNewsRecord.categoryStringLength,    // category
-                yacyCore.universalDatePattern.length(), // received
+                yacyCore.universalDateShortPattern.length(), // received
                 2,
                 attributesMaxLength
             });

source/de/anomic/yacy/yacyNewsPool.java

@@ -43,6 +43,7 @@ package de.anomic.yacy;
 
 import java.io.File;
 import java.io.IOException;
+import java.util.HashSet;
 
 public class yacyNewsPool {
     
@@ -77,7 +78,12 @@ public class yacyNewsPool {
         "wiki_upd", // a wiki page was updated
         "wiki_del"  // a wiki page das deleted
         // urlvotes
-    }; 
+    };
+    public static HashSet categories;
+    static {
+        categories = new HashSet();
+        for (int i = 0; i < category.length; i++) categories.add(category[i]);
+    }
     
     private yacyNewsDB newsDB;
     private yacyNewsQueue outgoingNews, publishedNews, incomingNews, processedNews;
@@ -85,11 +91,11 @@ public class yacyNewsPool {
     
     
     public yacyNewsPool(File yacyDBPath, int bufferkb) throws IOException {
-        newsDB = new yacyNewsDB(new File(yacyDBPath, "news0.db"), bufferkb);
-        outgoingNews  = new yacyNewsQueue(new File(yacyDBPath, "newsOut0.stack"), newsDB);
-        publishedNews = new yacyNewsQueue(new File(yacyDBPath, "newsPublished0.stack"), newsDB);
-        incomingNews  = new yacyNewsQueue(new File(yacyDBPath, "newsIn0.stack"), newsDB);
-        processedNews = new yacyNewsQueue(new File(yacyDBPath, "newsProcessed0.stack"), newsDB);
+        newsDB = new yacyNewsDB(new File(yacyDBPath, "news1.db"), bufferkb);
+        outgoingNews  = new yacyNewsQueue(new File(yacyDBPath, "newsOut1.stack"), newsDB);
+        publishedNews = new yacyNewsQueue(new File(yacyDBPath, "newsPublished1.stack"), newsDB);
+        incomingNews  = new yacyNewsQueue(new File(yacyDBPath, "newsIn1.stack"), newsDB);
+        processedNews = new yacyNewsQueue(new File(yacyDBPath, "newsProcessed1.stack"), newsDB);
         maxDistribution = 30;
     }
     
@@ -111,7 +117,15 @@ public class yacyNewsPool {
     
     public void enqueueIncomingNews(yacyNewsRecord record) throws IOException {
         // called if a news is attached to a seed
+
+        // check consistency
+        if (record.id() == null) return;
+        if (record.id().length() != yacyNewsRecord.idLength()) return;
+        if (record.category() == null) return;
+        if (!(categories.contains(record.category()))) return;
         if (record.created().getTime() == 0) return;
+        
+        // double-check with old news
         if (newsDB.get(record.id()) != null) return;
         incomingNews.push(record);
     }

source/de/anomic/yacy/yacyNewsQueue.java

@@ -63,16 +63,16 @@ public class yacyNewsQueue {
         this.path = path;
         this.newsDB = newsDB;
         
-        if (path.exists())
+        if (path.exists()) {
             queueStack = new kelondroStack(path, 0);
-        else
+        } else
             queueStack = createStack(path);
     }
     
     private static kelondroStack createStack(File path) throws IOException {
         return new kelondroStack(path, 0, new int[] {
                 yacyNewsRecord.idLength(), // id = created + originator
-                yacyCore.universalDatePattern.length() // last touched
+                yacyCore.universalDateShortPattern.length() // last touched
             });
     }
     

source/de/anomic/yacy/yacyNewsRecord.java

@@ -61,7 +61,6 @@ public class yacyNewsRecord {
     public yacyNewsRecord(String newsString) {
         this.attributes = serverCodings.string2map(newsString);
         this.received = (attributes.containsKey("rec")) ? yacyCore.parseUniversalDate((String) attributes.get("rec")) : new Date();
-        //this.received = new Date();
         this.created = (attributes.containsKey("cre")) ? yacyCore.parseUniversalDate((String) attributes.get("cre")) : new Date();
         this.category = (attributes.containsKey("cat")) ? (String) attributes.get("cat") : null;
         this.distributed = (attributes.containsKey("dis")) ? Integer.parseInt((String) attributes.get("dis")) : 0;
@@ -83,10 +82,10 @@ public class yacyNewsRecord {
     protected yacyNewsRecord(String id, String category, Date received, int distributed, Map attributes) {
         this.attributes = attributes;
         this.received = received;
-        this.created = yacyCore.parseUniversalDate(id.substring(0, yacyCore.universalDatePattern.length()));
+        this.created = yacyCore.parseUniversalDate(id.substring(0, yacyCore.universalDateShortPattern.length()));
         this.category = category;
         this.distributed = distributed;
-        this.originator = id.substring(yacyCore.universalDatePattern.length());
+        this.originator = id.substring(yacyCore.universalDateShortPattern.length());
         removeStandards();
     }
 
@@ -116,7 +115,7 @@ public class yacyNewsRecord {
     }
     
     public static int idLength() {
-        return yacyCore.universalDatePattern.length() + yacySeedDB.commonHashLength;
+        return yacyCore.universalDateShortPattern.length() + yacySeedDB.commonHashLength;
     }
     
     public String originator() {