From b2ed4cfaf8aa473d581628454f20ecdb952e93aa Mon Sep 17 00:00:00 2001 From: orbiter Date: Tue, 28 Dec 2010 01:57:05 +0000 Subject: [PATCH] more small bugfixes and light refactoring git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7401 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- htroot/CrawlStartScanner_p.java | 36 +++---------------- htroot/yacysearch.java | 2 +- source/de/anomic/yacy/yacyNewsQueue.java | 5 +-- source/net/yacy/cora/protocol/Scanner.java | 29 +++++++++++++++ source/net/yacy/kelondro/blob/HeapReader.java | 1 + 5 files changed, 39 insertions(+), 34 deletions(-) diff --git a/htroot/CrawlStartScanner_p.java b/htroot/CrawlStartScanner_p.java index 0f60bb592..88448a11c 100644 --- a/htroot/CrawlStartScanner_p.java +++ b/htroot/CrawlStartScanner_p.java @@ -19,7 +19,6 @@ */ -import java.io.IOException; import java.net.InetAddress; import java.net.MalformedURLException; import java.util.ConcurrentModificationException; @@ -33,7 +32,6 @@ import net.yacy.cora.protocol.Domains; import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.protocol.Scanner; import net.yacy.cora.protocol.Scanner.Access; -import net.yacy.kelondro.blob.Tables; import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.order.Base64Order; @@ -166,7 +164,7 @@ public class CrawlStartScanner_p { // execute the scan results if (Scanner.scancacheSize() > 0) { // make a comment cache - Map apiCommentCache = commentCache(sb); + Map apiCommentCache = Scanner.commentCache(sb); String urlString; DigestURI u; @@ -179,7 +177,7 @@ public class CrawlStartScanner_p { try { u = new DigestURI(host.getKey().url()); urlString = u.toNormalform(true, false); - if (host.getValue() == Access.granted && inIndex(apiCommentCache, urlString) == null) { + if (host.getValue() == Access.granted && Scanner.inIndex(apiCommentCache, urlString) == null) { String path = "/Crawler_p.html?createBookmark=off&xsstopw=off&crawlingDomMaxPages=10000&intention=&range=domain&indexMedia=on&recrawl=nodoubles&xdstopw=off&storeHTCache=on&sitemapURL=&repeat_time=7&crawlingQ=on&cachePolicy=iffresh&indexText=on&crawlingMode=url&mustnotmatch=&crawlingDomFilterDepth=1&crawlingDomFilterCheck=off&crawlingstart=Start%20New%20Crawl&xpstopw=off&repeat_unit=seldays&crawlingDepth=99"; path += "&crawlingURL=" + urlString; WorkTables.execAPICall("localhost", (int) sb.getConfigLong("port", 8080), sb.getConfig("adminAccountBase64MD5", ""), path, u.hash()); @@ -198,7 +196,7 @@ public class CrawlStartScanner_p { // write scan table if (Scanner.scancacheSize() > 0) { // make a comment cache - Map apiCommentCache = commentCache(sb); + Map apiCommentCache = Scanner.commentCache(sb); // show scancache table prop.put("servertable", 1); @@ -223,8 +221,8 @@ public class CrawlStartScanner_p { prop.put("servertable_list_" + i + "_accessEmpty", host.getValue() == Access.empty ? 1 : 0); prop.put("servertable_list_" + i + "_accessGranted", host.getValue() == Access.granted ? 1 : 0); prop.put("servertable_list_" + i + "_accessDenied", host.getValue() == Access.denied ? 1 : 0); - prop.put("servertable_list_" + i + "_process", inIndex(apiCommentCache, urlString) == null ? 0 : 1); - prop.put("servertable_list_" + i + "_preselected", host.getValue() == Access.granted && inIndex(apiCommentCache, urlString) == null ? 1 : 0); + prop.put("servertable_list_" + i + "_process", Scanner.inIndex(apiCommentCache, urlString) == null ? 0 : 1); + prop.put("servertable_list_" + i + "_preselected", host.getValue() == Access.granted && Scanner.inIndex(apiCommentCache, urlString) == null ? 1 : 0); i++; } catch (MalformedURLException e) { Log.logException(e); @@ -241,28 +239,4 @@ public class CrawlStartScanner_p { return prop; } - - private static byte[] inIndex(Map commentCache, String url) { - for (Map.Entry comment: commentCache.entrySet()) { - if (comment.getValue().contains(url)) return comment.getKey(); - } - return null; - } - - private static Map commentCache(Switchboard sb) { - Map comments = new TreeMap(Base64Order.enhancedCoder); - Iterator i; - try { - i = sb.tables.iterator(WorkTables.TABLE_API_NAME); - Tables.Row row; - while (i.hasNext()) { - row = i.next(); - comments.put(row.getPK(), new String(row.get(WorkTables.TABLE_API_COL_COMMENT))); - } - } catch (IOException e) { - Log.logException(e); - } - return comments; - } - } diff --git a/htroot/yacysearch.java b/htroot/yacysearch.java index 33283f155..7375b7b1c 100644 --- a/htroot/yacysearch.java +++ b/htroot/yacysearch.java @@ -90,7 +90,7 @@ public class yacysearch { // get query String originalquerystring = (post == null) ? "" : post.get("query", post.get("search", "")).trim(); - String querystring = originalquerystring.replace('+', ' ').replace('*', ' '); + String querystring = originalquerystring.replace('+', ' ').replace('*', ' ').trim(); CrawlProfile.CacheStrategy snippetFetchStrategy = (post == null) ? null : CrawlProfile.CacheStrategy.parse(post.get("verify", "cacheonly")); final servletProperties prop = new servletProperties(); prop.put("topmenu", sb.getConfigBool("publicTopmenu", true) ? 1 : 0); diff --git a/source/de/anomic/yacy/yacyNewsQueue.java b/source/de/anomic/yacy/yacyNewsQueue.java index 33f815a72..6e59bdb13 100644 --- a/source/de/anomic/yacy/yacyNewsQueue.java +++ b/source/de/anomic/yacy/yacyNewsQueue.java @@ -198,10 +198,11 @@ public class yacyNewsQueue { } public boolean hasNext() { - return stackNodeIterator.hasNext(); + return stackNodeIterator != null && stackNodeIterator.hasNext(); } public yacyNewsDB.Record next() { + if (stackNodeIterator == null) return null; final Row.Entry row = stackNodeIterator.next(); try { return b2r(row); @@ -211,7 +212,7 @@ public class yacyNewsQueue { } public void remove() { - stackNodeIterator.remove(); + if (stackNodeIterator != null) stackNodeIterator.remove(); } } diff --git a/source/net/yacy/cora/protocol/Scanner.java b/source/net/yacy/cora/protocol/Scanner.java index a34cb17a7..5c17e0da0 100644 --- a/source/net/yacy/cora/protocol/Scanner.java +++ b/source/net/yacy/cora/protocol/Scanner.java @@ -32,15 +32,21 @@ import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.TreeMap; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.LinkedBlockingQueue; +import de.anomic.data.WorkTables; +import de.anomic.search.Switchboard; + import net.yacy.cora.document.MultiProtocolURI; import net.yacy.cora.protocol.ftp.FTPClient; import net.yacy.cora.protocol.http.HTTPClient; +import net.yacy.kelondro.blob.Tables; import net.yacy.kelondro.logging.Log; +import net.yacy.kelondro.order.Base64Order; /** * a protocol scanner @@ -323,6 +329,29 @@ public class Scanner extends Thread { return this.services; } + public static byte[] inIndex(Map commentCache, String url) { + for (Map.Entry comment: commentCache.entrySet()) { + if (comment.getValue().contains(url)) return comment.getKey(); + } + return null; + } + + public static Map commentCache(Switchboard sb) { + Map comments = new TreeMap(Base64Order.enhancedCoder); + Iterator i; + try { + i = sb.tables.iterator(WorkTables.TABLE_API_NAME); + Tables.Row row; + while (i.hasNext()) { + row = i.next(); + comments.put(row.getPK(), new String(row.get(WorkTables.TABLE_API_COL_COMMENT))); + } + } catch (IOException e) { + Log.logException(e); + } + return comments; + } + public static void main(String[] args) { //try {System.out.println("192.168.1.91: " + ping(new MultiProtocolURI("smb://192.168.1.91/"), 1000));} catch (MalformedURLException e) {} Scanner scanner = new Scanner(100, 10); diff --git a/source/net/yacy/kelondro/blob/HeapReader.java b/source/net/yacy/kelondro/blob/HeapReader.java index 534578757..d95404dfa 100644 --- a/source/net/yacy/kelondro/blob/HeapReader.java +++ b/source/net/yacy/kelondro/blob/HeapReader.java @@ -302,6 +302,7 @@ public class HeapReader { * @return the number of BLOBs in the heap */ public int size() { + if (this.index == null) return 0; synchronized (index) { return (this.index == null) ? 0 : this.index.size(); }