diff --git a/htroot/Network.html b/htroot/Network.html index 84b0bc393..43d3746b6 100644 --- a/htroot/Network.html +++ b/htroot/Network.html @@ -68,7 +68,7 @@ #[shortname]# #(complete)# :: -http://#[ip]#:#[port]# +http://#[ip]#:#[port]# #[hash]# #(/complete)# #(type)#junior::senior::principal#(/type)# diff --git a/htroot/News.java b/htroot/News.java index 263a7bfc2..f0c2bee6d 100644 --- a/htroot/News.java +++ b/htroot/News.java @@ -66,13 +66,12 @@ public class News { // execute commands if (post != null) { - if (switchboard.adminAuthenticated(header) < 2) { - // not authenticated, force log-in - prop.put("AUTHENTICATE", "admin log-in"); - return prop; - } if ((post.containsKey("deletespecific")) && (tableID >= 0)) { + if (switchboard.adminAuthenticated(header) < 2) { + prop.put("AUTHENTICATE", "admin log-in"); + return prop; // this button needs authentication, force log-in + } Enumeration e = post.keys(); String check; String id; @@ -88,6 +87,10 @@ public class News { } if ((post.containsKey("deleteall")) && (tableID >= 0)) { + if (switchboard.adminAuthenticated(header) < 2) { + prop.put("AUTHENTICATE", "admin log-in"); + return prop; // this button needs authentication, force log-in + } yacyNewsRecord record; try { while (yacyCore.newsPool.size(tableID) > 0) { diff --git a/source/de/anomic/kelondro/kelondroDyn.java b/source/de/anomic/kelondro/kelondroDyn.java index bf89be456..74a1e2ecc 100644 --- a/source/de/anomic/kelondro/kelondroDyn.java +++ b/source/de/anomic/kelondro/kelondroDyn.java @@ -298,7 +298,7 @@ public class kelondroDyn extends kelondroTree { } public synchronized boolean existsDyn(String key) throws IOException { - return (getValueCached(dynKey(key, 0)) != null); + return (key != null) && (getValueCached(dynKey(key, 0)) != null); } public synchronized kelondroRA getRA(String filekey) throws IOException { diff --git a/source/de/anomic/kelondro/kelondroMap.java b/source/de/anomic/kelondro/kelondroMap.java index 361ac255c..7704f4033 100644 --- a/source/de/anomic/kelondro/kelondroMap.java +++ b/source/de/anomic/kelondro/kelondroMap.java @@ -196,6 +196,7 @@ public class kelondroMap { public synchronized void remove(String key) throws IOException { // update elementCount + if (key == null) return; if ((sortfields != null) || (accfields != null)) { Map map = get(key); if (map != null) { @@ -219,6 +220,7 @@ public class kelondroMap { } private void deleteSortCluster(String key) { + if (key == null) return; kelondroMScoreCluster cluster; for (int i = 0; i < sortfields.length; i++) { cluster = (kelondroMScoreCluster) sortClusterMap.get(sortfields[i]); @@ -228,6 +230,7 @@ public class kelondroMap { } public synchronized Map get(String key) throws IOException { + if (key == null) return null; return get(key, true); } diff --git a/source/de/anomic/yacy/yacyCore.java b/source/de/anomic/yacy/yacyCore.java index 889baa488..0f059b1ae 100644 --- a/source/de/anomic/yacy/yacyCore.java +++ b/source/de/anomic/yacy/yacyCore.java @@ -107,8 +107,8 @@ public class yacyCore { private plasmaSwitchboard switchboard; private static TimeZone GMTTimeZone = TimeZone.getTimeZone("America/Los_Angeles"); - public static String universalDatePattern = "yyyyMMddHHmmss"; - public static SimpleDateFormat shortFormatter = new SimpleDateFormat(universalDatePattern); + public static String universalDateShortPattern = "yyyyMMddHHmmss"; + public static SimpleDateFormat shortFormatter = new SimpleDateFormat(universalDateShortPattern); public static long universalTime() { return universalDate().getTime(); diff --git a/source/de/anomic/yacy/yacyNewsAction.java b/source/de/anomic/yacy/yacyNewsAction.java index c0ce12442..28dad728c 100644 --- a/source/de/anomic/yacy/yacyNewsAction.java +++ b/source/de/anomic/yacy/yacyNewsAction.java @@ -55,8 +55,9 @@ public class yacyNewsAction implements yacyPeerAction { String recordString = peer.get("news", null); //System.out.println("### triggered news arrival from peer " + peer.getName() + ", news " + ((recordString == null) ? "empty" : "attached")); if ((recordString == null) || (recordString.length() == 0)) return; - yacyNewsRecord record = new yacyNewsRecord(de.anomic.tools.crypt.simpleDecode(recordString, "")); - //System.out.println("### news arrival from peer " + peer.getName() + ", news=" + record.toString()); + String decodedString = de.anomic.tools.crypt.simpleDecode(recordString, ""); + yacyNewsRecord record = new yacyNewsRecord(decodedString); + System.out.println("### news arrival from peer " + peer.getName() + ", decoded=" + decodedString + ", record=" + recordString + ", news=" + record.toString()); try { this.pool.enqueueIncomingNews(record); } catch (IOException e) {e.printStackTrace();} diff --git a/source/de/anomic/yacy/yacyNewsDB.java b/source/de/anomic/yacy/yacyNewsDB.java index cd310edd0..d8cb17d85 100644 --- a/source/de/anomic/yacy/yacyNewsDB.java +++ b/source/de/anomic/yacy/yacyNewsDB.java @@ -60,7 +60,7 @@ public class yacyNewsDB { public static final int attributesMaxLength = yacyNewsRecord.maxNewsRecordLength - yacyNewsRecord.idLength() - yacyNewsRecord.categoryStringLength - - yacyCore.universalDatePattern.length() + - yacyCore.universalDateShortPattern.length() - 2; public yacyNewsDB(File path, int bufferkb) throws IOException { @@ -77,7 +77,7 @@ public class yacyNewsDB { return new kelondroTree(path, bufferkb, new int[] { yacyNewsRecord.idLength(), // id = created + originator yacyNewsRecord.categoryStringLength, // category - yacyCore.universalDatePattern.length(), // received + yacyCore.universalDateShortPattern.length(), // received 2, attributesMaxLength }); diff --git a/source/de/anomic/yacy/yacyNewsPool.java b/source/de/anomic/yacy/yacyNewsPool.java index bab8e3dfc..d4ed5501a 100644 --- a/source/de/anomic/yacy/yacyNewsPool.java +++ b/source/de/anomic/yacy/yacyNewsPool.java @@ -43,6 +43,7 @@ package de.anomic.yacy; import java.io.File; import java.io.IOException; +import java.util.HashSet; public class yacyNewsPool { @@ -77,7 +78,12 @@ public class yacyNewsPool { "wiki_upd", // a wiki page was updated "wiki_del" // a wiki page das deleted // urlvotes - }; + }; + public static HashSet categories; + static { + categories = new HashSet(); + for (int i = 0; i < category.length; i++) categories.add(category[i]); + } private yacyNewsDB newsDB; private yacyNewsQueue outgoingNews, publishedNews, incomingNews, processedNews; @@ -85,11 +91,11 @@ public class yacyNewsPool { public yacyNewsPool(File yacyDBPath, int bufferkb) throws IOException { - newsDB = new yacyNewsDB(new File(yacyDBPath, "news0.db"), bufferkb); - outgoingNews = new yacyNewsQueue(new File(yacyDBPath, "newsOut0.stack"), newsDB); - publishedNews = new yacyNewsQueue(new File(yacyDBPath, "newsPublished0.stack"), newsDB); - incomingNews = new yacyNewsQueue(new File(yacyDBPath, "newsIn0.stack"), newsDB); - processedNews = new yacyNewsQueue(new File(yacyDBPath, "newsProcessed0.stack"), newsDB); + newsDB = new yacyNewsDB(new File(yacyDBPath, "news1.db"), bufferkb); + outgoingNews = new yacyNewsQueue(new File(yacyDBPath, "newsOut1.stack"), newsDB); + publishedNews = new yacyNewsQueue(new File(yacyDBPath, "newsPublished1.stack"), newsDB); + incomingNews = new yacyNewsQueue(new File(yacyDBPath, "newsIn1.stack"), newsDB); + processedNews = new yacyNewsQueue(new File(yacyDBPath, "newsProcessed1.stack"), newsDB); maxDistribution = 30; } @@ -111,7 +117,15 @@ public class yacyNewsPool { public void enqueueIncomingNews(yacyNewsRecord record) throws IOException { // called if a news is attached to a seed + + // check consistency + if (record.id() == null) return; + if (record.id().length() != yacyNewsRecord.idLength()) return; + if (record.category() == null) return; + if (!(categories.contains(record.category()))) return; if (record.created().getTime() == 0) return; + + // double-check with old news if (newsDB.get(record.id()) != null) return; incomingNews.push(record); } diff --git a/source/de/anomic/yacy/yacyNewsQueue.java b/source/de/anomic/yacy/yacyNewsQueue.java index 29ad0100e..7c40ba626 100644 --- a/source/de/anomic/yacy/yacyNewsQueue.java +++ b/source/de/anomic/yacy/yacyNewsQueue.java @@ -63,16 +63,16 @@ public class yacyNewsQueue { this.path = path; this.newsDB = newsDB; - if (path.exists()) + if (path.exists()) { queueStack = new kelondroStack(path, 0); - else + } else queueStack = createStack(path); } private static kelondroStack createStack(File path) throws IOException { return new kelondroStack(path, 0, new int[] { yacyNewsRecord.idLength(), // id = created + originator - yacyCore.universalDatePattern.length() // last touched + yacyCore.universalDateShortPattern.length() // last touched }); } diff --git a/source/de/anomic/yacy/yacyNewsRecord.java b/source/de/anomic/yacy/yacyNewsRecord.java index 38d9380b1..fcc8840b5 100644 --- a/source/de/anomic/yacy/yacyNewsRecord.java +++ b/source/de/anomic/yacy/yacyNewsRecord.java @@ -61,7 +61,6 @@ public class yacyNewsRecord { public yacyNewsRecord(String newsString) { this.attributes = serverCodings.string2map(newsString); this.received = (attributes.containsKey("rec")) ? yacyCore.parseUniversalDate((String) attributes.get("rec")) : new Date(); - //this.received = new Date(); this.created = (attributes.containsKey("cre")) ? yacyCore.parseUniversalDate((String) attributes.get("cre")) : new Date(); this.category = (attributes.containsKey("cat")) ? (String) attributes.get("cat") : null; this.distributed = (attributes.containsKey("dis")) ? Integer.parseInt((String) attributes.get("dis")) : 0; @@ -83,10 +82,10 @@ public class yacyNewsRecord { protected yacyNewsRecord(String id, String category, Date received, int distributed, Map attributes) { this.attributes = attributes; this.received = received; - this.created = yacyCore.parseUniversalDate(id.substring(0, yacyCore.universalDatePattern.length())); + this.created = yacyCore.parseUniversalDate(id.substring(0, yacyCore.universalDateShortPattern.length())); this.category = category; this.distributed = distributed; - this.originator = id.substring(yacyCore.universalDatePattern.length()); + this.originator = id.substring(yacyCore.universalDateShortPattern.length()); removeStandards(); } @@ -116,7 +115,7 @@ public class yacyNewsRecord { } public static int idLength() { - return yacyCore.universalDatePattern.length() + yacySeedDB.commonHashLength; + return yacyCore.universalDateShortPattern.length() + yacySeedDB.commonHashLength; } public String originator() {