boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

more small bugfixes and light refactoring

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7401 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 14 years ago
parent 1b6702146f
commit b2ed4cfaf8
5 changed files with 39 additions and 34 deletions
Show Stats Download Patch File Download Diff File

36
htroot/CrawlStartScanner_p.java
Unescape View File

@ -19,7 +19,6 @@
*/
import java.io.IOException;
import java.net.InetAddress;
import java.net.MalformedURLException;
import java.util.ConcurrentModificationException;
@ -33,7 +32,6 @@ import net.yacy.cora.protocol.Domains;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.cora.protocol.Scanner;
import net.yacy.cora.protocol.Scanner.Access;
import net.yacy.kelondro.blob.Tables;
import net.yacy.kelondro.data.meta.DigestURI;
import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.order.Base64Order;
@ -166,7 +164,7 @@ public class CrawlStartScanner_p {
// execute the scan results
if (Scanner.scancacheSize() > 0) {
// make a comment cache
Map<byte[], String> apiCommentCache = commentCache(sb);
Map<byte[], String> apiCommentCache = Scanner.commentCache(sb);
String urlString;
DigestURI u;
@ -179,7 +177,7 @@ public class CrawlStartScanner_p {
try {
u = new DigestURI(host.getKey().url());
urlString = u.toNormalform(true, false);
if (host.getValue() == Access.granted && inIndex(apiCommentCache, urlString) == null) {
if (host.getValue() == Access.granted && Scanner.inIndex(apiCommentCache, urlString) == null) {
String path = "/Crawler_p.html?createBookmark=off&xsstopw=off&crawlingDomMaxPages=10000&intention=&range=domain&indexMedia=on&recrawl=nodoubles&xdstopw=off&storeHTCache=on&sitemapURL=&repeat_time=7&crawlingQ=on&cachePolicy=iffresh&indexText=on&crawlingMode=url&mustnotmatch=&crawlingDomFilterDepth=1&crawlingDomFilterCheck=off&crawlingstart=Start%20New%20Crawl&xpstopw=off&repeat_unit=seldays&crawlingDepth=99";
path += "&crawlingURL=" + urlString;
WorkTables.execAPICall("localhost", (int) sb.getConfigLong("port", 8080), sb.getConfig("adminAccountBase64MD5", ""), path, u.hash());
@ -198,7 +196,7 @@ public class CrawlStartScanner_p {
// write scan table
if (Scanner.scancacheSize() > 0) {
// make a comment cache
Map<byte[], String> apiCommentCache = commentCache(sb);
Map<byte[], String> apiCommentCache = Scanner.commentCache(sb);
// show scancache table
prop.put("servertable", 1);
@ -223,8 +221,8 @@ public class CrawlStartScanner_p {
prop.put("servertable_list_" + i + "_accessEmpty", host.getValue() == Access.empty ? 1 : 0);
prop.put("servertable_list_" + i + "_accessGranted", host.getValue() == Access.granted ? 1 : 0);
prop.put("servertable_list_" + i + "_accessDenied", host.getValue() == Access.denied ? 1 : 0);
prop.put("servertable_list_" + i + "_process", inIndex(apiCommentCache, urlString) == null ? 0 : 1);
prop.put("servertable_list_" + i + "_preselected", host.getValue() == Access.granted && inIndex(apiCommentCache, urlString) == null ? 1 : 0);
prop.put("servertable_list_" + i + "_process", Scanner.inIndex(apiCommentCache, urlString) == null ? 0 : 1);
prop.put("servertable_list_" + i + "_preselected", host.getValue() == Access.granted && Scanner.inIndex(apiCommentCache, urlString) == null ? 1 : 0);
i++;
} catch (MalformedURLException e) {
Log.logException(e);
@ -241,28 +239,4 @@ public class CrawlStartScanner_p {
return prop;
}
private static byte[] inIndex(Map<byte[], String> commentCache, String url) {
for (Map.Entry<byte[], String> comment: commentCache.entrySet()) {
if (comment.getValue().contains(url)) return comment.getKey();
}
return null;
}
private static Map<byte[], String> commentCache(Switchboard sb) {
Map<byte[], String> comments = new TreeMap<byte[], String>(Base64Order.enhancedCoder);
Iterator<Tables.Row> i;
try {
i = sb.tables.iterator(WorkTables.TABLE_API_NAME);
Tables.Row row;
while (i.hasNext()) {
row = i.next();
comments.put(row.getPK(), new String(row.get(WorkTables.TABLE_API_COL_COMMENT)));
}
} catch (IOException e) {
Log.logException(e);
}
return comments;
}
}

2
htroot/yacysearch.java
Unescape View File

@ -90,7 +90,7 @@ public class yacysearch {
// get query
String originalquerystring = (post == null) ? "" : post.get("query", post.get("search", "")).trim();
String querystring = originalquerystring.replace('+', ' ').replace('*', ' ');
String querystring = originalquerystring.replace('+', ' ').replace('*', ' ').trim();
CrawlProfile.CacheStrategy snippetFetchStrategy = (post == null) ? null : CrawlProfile.CacheStrategy.parse(post.get("verify", "cacheonly"));
final servletProperties prop = new servletProperties();
prop.put("topmenu", sb.getConfigBool("publicTopmenu", true) ? 1 : 0);

5
source/de/anomic/yacy/yacyNewsQueue.java
Unescape View File

@ -198,10 +198,11 @@ public class yacyNewsQueue {
}
public boolean hasNext() {
return stackNodeIterator.hasNext();
return stackNodeIterator != null && stackNodeIterator.hasNext();
}
public yacyNewsDB.Record next() {
if (stackNodeIterator == null) return null;
final Row.Entry row = stackNodeIterator.next();
try {
return b2r(row);
@ -211,7 +212,7 @@ public class yacyNewsQueue {
}
public void remove() {
stackNodeIterator.remove();
if (stackNodeIterator != null) stackNodeIterator.remove();
}
}

29
source/net/yacy/cora/protocol/Scanner.java
Unescape View File

@ -32,15 +32,21 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.LinkedBlockingQueue;
import de.anomic.data.WorkTables;
import de.anomic.search.Switchboard;
import net.yacy.cora.document.MultiProtocolURI;
import net.yacy.cora.protocol.ftp.FTPClient;
import net.yacy.cora.protocol.http.HTTPClient;
import net.yacy.kelondro.blob.Tables;
import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.order.Base64Order;
/**
* a protocol scanner
@ -323,6 +329,29 @@ public class Scanner extends Thread {
return this.services;
}
public static byte[] inIndex(Map<byte[], String> commentCache, String url) {
for (Map.Entry<byte[], String> comment: commentCache.entrySet()) {
if (comment.getValue().contains(url)) return comment.getKey();
}
return null;
}
public static Map<byte[], String> commentCache(Switchboard sb) {
Map<byte[], String> comments = new TreeMap<byte[], String>(Base64Order.enhancedCoder);
Iterator<Tables.Row> i;
try {
i = sb.tables.iterator(WorkTables.TABLE_API_NAME);
Tables.Row row;
while (i.hasNext()) {
row = i.next();
comments.put(row.getPK(), new String(row.get(WorkTables.TABLE_API_COL_COMMENT)));
}
} catch (IOException e) {
Log.logException(e);
}
return comments;
}
public static void main(String[] args) {
//try {System.out.println("192.168.1.91: " + ping(new MultiProtocolURI("smb://192.168.1.91/"), 1000));} catch (MalformedURLException e) {}
Scanner scanner = new Scanner(100, 10);

1
source/net/yacy/kelondro/blob/HeapReader.java
Unescape View File

@ -302,6 +302,7 @@ public class HeapReader {
* @return the number of BLOBs in the heap
*/
public int size() {
if (this.index == null) return 0;
synchronized (index) {
return (this.index == null) ? 0 : this.index.size();
}

Write Preview
Loading…
Cancel
Save