boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

- added streaming-support to CrawlURLFetchStack_p servlet

Browse Source 
- bug for NPE in list.java
- use more constants

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@3373 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
karlchenofhell 18 years ago
parent 65af9d3215
commit c016fcb10f
7 changed files with 220 additions and 110 deletions
Show Stats Download Patch File Download Diff File

82
htroot/CrawlURLFetchStack_p.java
Unescape View File

@ -39,8 +39,13 @@
// the intact and unchanged copyright notice. // the intact and unchanged copyright notice.
// Contributions and changes to the program code must be marked as such. // Contributions and changes to the program code must be marked as such.
import java.io.BufferedReader;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.PrintWriter;
import java.io.Writer; import java.io.Writer;
import java.net.MalformedURLException; import java.net.MalformedURLException;
import java.util.HashMap; import java.util.HashMap;
@ -73,10 +78,68 @@ public class CrawlURLFetchStack_p {
return stack; return stack;
} }
public static final String STREAM_CMD_ADDURLS_ = "ADD URLS: ";
public static final String STREAM_CMD_END = "END";
public static final String STREAM_RESP_OK_ADDURLS_ = "FAILED URLS: ";
public static final String STREAM_RESP_OK = "OK";
public static final String STREAM_RESP_FAILED = "FAILED";
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) { public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) {
final serverObjects prop = new serverObjects(); final serverObjects prop = new serverObjects();
plasmaSwitchboard sb = (plasmaSwitchboard)env; plasmaSwitchboard sb = (plasmaSwitchboard)env;
if (((String)header.get(httpHeader.CONNECTION_PROP_PATH)).endsWith(".stream")) {
/* =================================================================
* .stream request
* ================================================================= */
InputStream in = (InputStream)header.get(httpHeader.CONNECTION_PROP_INPUTSTREAM);
OutputStream out = (OutputStream)header.get(httpHeader.CONNECTION_PROP_OUTPUTSTREAM);
BufferedReader inrb = new BufferedReader(new InputStreamReader(in));
PrintWriter outw = new PrintWriter(out);
String line;
int addurls = 0, cururl = 0;
boolean[] status = new boolean[0];
URLFetcherStack stack = getURLFetcherStack(env);
try {
while ((line = inrb.readLine()) != null) {
// commands
if (line.startsWith(STREAM_CMD_ADDURLS_)) {
try {
addurls = Integer.parseInt(line.substring(STREAM_CMD_ADDURLS_.length()));
status = new boolean[addurls];
cururl = 0;
outw.println(STREAM_RESP_OK);
} catch (NumberFormatException e) {
outw.println(STREAM_RESP_FAILED);
}
} else if (line.equals(STREAM_CMD_END)) {
break;
} else {
if (cururl < addurls) // add url
status[cururl++] = addURL(line, stack);
if (cururl > 0 && cururl == addurls ) {
// done with parsing the passed URL count, now some status output: i.e. 'FAILED URLS: 5 of 8'
outw.print(STREAM_RESP_OK_ADDURLS_);
StringBuffer stat = new StringBuffer();
for (int i=0; i<status.length; i++)
if (!status[i]) stat.append(i).append(", ");
outw.print(stat.substring(0, stat.length() - 2));
outw.print(" of ");
outw.println(status.length);
cururl = 0;
addurls = 0;
}
}
}
} catch (IOException e) { e.printStackTrace(); }
outw.flush();
return null;
} else {
/* =================================================================
* 'normal' request
* ================================================================= */
if (post != null) { if (post != null) {
if (post.containsKey("addurls")) { if (post.containsKey("addurls")) {
prop.put("addedUrls", 1); prop.put("addedUrls", 1);
@ -146,6 +209,7 @@ public class CrawlURLFetchStack_p {
} }
} }
} }
}
putFetched(prop); putFetched(prop);
prop.put("urlCount", getURLFetcherStack(env).size()); prop.put("urlCount", getURLFetcherStack(env).size());
@ -156,7 +220,6 @@ public class CrawlURLFetchStack_p {
prop.put("remurls", sb.noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_LIMIT)); prop.put("remurls", sb.noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_LIMIT));
prop.put("locurlsVal", Math.min(sb.noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_CORE), 500)); prop.put("locurlsVal", Math.min(sb.noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_CORE), 500));
prop.put("remurlsVal", Math.min(sb.noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_LIMIT), 500)); prop.put("remurlsVal", Math.min(sb.noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_LIMIT), 500));
return prop; return prop;
} }
@ -174,14 +237,19 @@ public class CrawlURLFetchStack_p {
private static int addURLs(String[] urls, URLFetcherStack stack) { private static int addURLs(String[] urls, URLFetcherStack stack) {
int count = -1; int count = -1;
for (int i=0; i<urls.length; i++) try { for (int i=0; i<urls.length; i++)
if (urls[i].length() == 0) continue; if (addURL(urls[i], stack)) count++;
stack.push(new URL(urls[i]));
count++;
} catch (MalformedURLException e) { /* ignore this */ }
return count; return count;
} }
private static boolean addURL(String url, URLFetcherStack stack) {
try {
if (url == null || url.length() == 0) return false;
stack.push(new URL(url));
return true;
} catch (MalformedURLException e) { return false; }
}
private static int shiftFromNotice(plasmaCrawlNURL nurl, int fromStackType, URLFetcherStack stack, int count) { private static int shiftFromNotice(plasmaCrawlNURL nurl, int fromStackType, URLFetcherStack stack, int count) {
plasmaCrawlNURL.Entry entry; plasmaCrawlNURL.Entry entry;
int failed = 0; int failed = 0;
@ -196,7 +264,7 @@ public class CrawlURLFetchStack_p {
int count = 0; int count = 0;
String url; String url;
for (int i=0; i<amount; i++) { for (int i=0; i<amount; i++) {
url = post.get("url" + count++, null); url = post.get("url" + i, null);
if (url == null || url.length() == 0) continue; if (url == null || url.length() == 0) continue;
try { try {
stack.push(new URL(url)); stack.push(new URL(url));

5
htroot/CrawlURLFetch_p.html
Unescape View File

@ -87,6 +87,11 @@
<li><span class="error">#[reason]#</span>: <a href="#[url]#">#[url]#</a></li>#{/error}# <li><span class="error">#[reason]#</span>: <a href="#[url]#">#[url]#</a></li>#{/error}#
</ul> </ul>
</dd> </dd>
<dt><label for="newDelay">Re-set delay</label>:</dt>
<dd>
<input type="text" name="newDelay" id="newDelay" maxlength="2" size="6" value="#[curDelay]#" style="text-align: right;" /> minutes
<input type="submit" name="resetDelay" value="Set new delay" />
</dd>
<dt>#(status)# <dt>#(status)#
<input type="submit" name="stop" value="Stop Thread" />:: <input type="submit" name="stop" value="Stop Thread" />::
<input type="submit" name="restart" value="Restart Thread" />:: <input type="submit" name="restart" value="Restart Thread" />::

38
htroot/CrawlURLFetch_p.java
Unescape View File

@ -224,6 +224,15 @@ public class CrawlURLFetch_p {
prop.put("threadError", ERR_THREAD_RESUME); prop.put("threadError", ERR_THREAD_RESUME);
} }
} }
else if (post.containsKey("resetDelay")) {
final long frequency = getDate(post.get("newDelay", ""), "minutes");
if (frequency == -1) {
prop.put("freqError", ERR_DATE);
} else {
fetcher.delay = frequency;
}
}
prop.put("LOCATION", "/CrawlURLFetch_p.html");
} }
if (fetcher != null) { if (fetcher != null) {
@ -238,6 +247,7 @@ public class CrawlURLFetch_p {
prop.put("runs_lastFetchedURLs", fetcher.lastFetchedURLs); prop.put("runs_lastFetchedURLs", fetcher.lastFetchedURLs);
prop.put("runs_lastServerResponse", (fetcher.lastServerResponse == null) prop.put("runs_lastServerResponse", (fetcher.lastServerResponse == null)
? "" : fetcher.lastServerResponse); ? "" : fetcher.lastServerResponse);
prop.put("runs_curDelay", (int)(fetcher.delay / 60000));
Iterator it = fetcher.failed.keySet().iterator(); Iterator it = fetcher.failed.keySet().iterator();
int i = 0; int i = 0;
@ -265,28 +275,34 @@ public class CrawlURLFetch_p {
private static int listPeers(serverObjects prop, boolean checkURLCount, httpRemoteProxyConfig theRemoteProxyConfig) { private static int listPeers(serverObjects prop, boolean checkURLCount, httpRemoteProxyConfig theRemoteProxyConfig) {
int peerCount = 0; int peerCount = 0;
if (yacyCore.seedDB != null && yacyCore.seedDB.sizeConnected() > 0) {
prop.put("peersKnown", 1);
try {
TreeMap hostList = new TreeMap(); TreeMap hostList = new TreeMap();
String peername;
if (yacyCore.seedDB != null && yacyCore.seedDB.sizeConnected() > 0) {
final Enumeration e = yacyCore.seedDB.seedsConnected(true, false, null, yacyVersion.YACY_PROVIDES_CRAWLS_VIA_LIST_HTML); final Enumeration e = yacyCore.seedDB.seedsConnected(true, false, null, yacyVersion.YACY_PROVIDES_CRAWLS_VIA_LIST_HTML);
int dbsize;
while (e.hasMoreElements()) { while (e.hasMoreElements()) {
yacySeed seed = (yacySeed) e.nextElement(); yacySeed seed = (yacySeed) e.nextElement();
if (seed != null && (!checkURLCount || getURLs2Fetch(seed, theRemoteProxyConfig) > 0)) if (seed != null && !seed.hash.equals(yacyCore.seedDB.mySeed.hash)) {
hostList.put(seed.get(yacySeed.NAME, "nameless"), seed.hash); peername = seed.get(yacySeed.NAME, "nameless");
if (checkURLCount && (dbsize = getURLs2Fetch(seed, theRemoteProxyConfig)) > 0) {
hostList.put(peername + " (" + dbsize + ")", seed.hash);
} else {
hostList.put(peername, seed.hash);
}
}
}
} }
String peername; if (hostList.size() > 0) {
while ((peername = (String) hostList.firstKey()) != null) { while (!hostList.isEmpty() && (peername = (String) hostList.firstKey()) != null) {
final String hash = (String) hostList.get(peername); final String hash = (String) hostList.get(peername);
if (hash.equals(yacyCore.seedDB.mySeed.hash)) continue;
prop.put("peersKnown_peers_" + peerCount + "_hash", hash); prop.put("peersKnown_peers_" + peerCount + "_hash", hash);
prop.put("peersKnown_peers_" + peerCount + "_name", peername); prop.put("peersKnown_peers_" + peerCount + "_name", peername);
hostList.remove(peername); hostList.remove(peername);
peerCount++; peerCount++;
} }
} catch (Exception e) { /* no comment :P */ }
prop.put("peersKnown_peers", peerCount); prop.put("peersKnown_peers", peerCount);
prop.put("peersKnown", 1);
} else { } else {
prop.put("peersKnown", 0); prop.put("peersKnown", 0);
} }
@ -304,7 +320,7 @@ public class CrawlURLFetch_p {
if (answer.matches("\\d+")) if (answer.matches("\\d+"))
return Integer.parseInt(answer); return Integer.parseInt(answer);
else { else {
System.err.println("RETRIEVED INVALID ANSWER FROM " + seed.getName() + ": '" + answer + "'"); serverLog.logFine("URLFETCHER", "Retrieved invalid answer from " + seed.getName() + ": '" + answer + "'");
return -1; return -1;
} }
} catch (MalformedURLException e) { } catch (MalformedURLException e) {
@ -348,7 +364,7 @@ public class CrawlURLFetch_p {
public final URL url; public final URL url;
public final int count; public final int count;
public final long delay; public long delay;
public final plasmaSwitchboard sb; public final plasmaSwitchboard sb;
public final plasmaCrawlProfile.entry profile; public final plasmaCrawlProfile.entry profile;

20
htroot/yacy/list.java
Unescape View File

@ -51,6 +51,7 @@
// contains contributions by [FB] to support listing URLs for URL Fetcher // contains contributions by [FB] to support listing URLs for URL Fetcher
import java.io.File; import java.io.File;
import java.net.InetAddress;
import de.anomic.data.URLFetcherStack; import de.anomic.data.URLFetcherStack;
import de.anomic.data.listManager; import de.anomic.data.listManager;
@ -75,7 +76,13 @@ public final class list {
final String col = post.get("col", ""); final String col = post.get("col", "");
final File listsPath = new File(ss.getRootPath(),ss.getConfig("listsPath", "DATA/LISTS")); final File listsPath = new File(ss.getRootPath(),ss.getConfig("listsPath", "DATA/LISTS"));
final String otherPeerName = yacyCore.seedDB.get(post.get("iam", null)).get(yacySeed.NAME, "unknown");
String otherPeerName = null;
if (post.containsKey("iam")) {
yacySeed bla = yacyCore.seedDB.get(post.get("iam", ""));
if (bla != null) otherPeerName = bla.getName();
}
if (otherPeerName == null) otherPeerName = (String)header.get(httpHeader.CONNECTION_PROP_CLIENTIP);
if (col.equals("black")) { if (col.equals("black")) {
final StringBuffer out = new StringBuffer(); final StringBuffer out = new StringBuffer();
@ -101,6 +108,7 @@ public final class list {
if (display.equals("list")) { if (display.equals("list")) {
// list urls from remote crawler queue for other peers // list urls from remote crawler queue for other peers
final int count = Math.min(post.getInt("count", 50), CrawlURLFetchStack_p.maxURLsPerFetch); final int count = Math.min(post.getInt("count", 50), CrawlURLFetchStack_p.maxURLsPerFetch);
if (count > 0 && db.size() > 0) { if (count > 0 && db.size() > 0) {
final StringBuffer sb = new StringBuffer(); final StringBuffer sb = new StringBuffer();
@ -112,11 +120,15 @@ public final class list {
cnt++; cnt++;
} }
prop.put("list", sb); prop.put("list", sb);
CrawlURLFetchStack_p.fetchMap.put(otherPeerName, new Integer(cnt)); CrawlURLFetchStack_p.fetchMap.put(
serverLog.logInfo("URLFETCHER", "sent " + cnt + " URLs to peer " + otherPeerName); otherPeerName,
new Integer(((CrawlURLFetchStack_p.fetchMap.get(otherPeerName) == null)
? 0
: ((Integer)CrawlURLFetchStack_p.fetchMap.get(otherPeerName)).intValue()) + cnt));
serverLog.logInfo("URLFETCHER", "sent " + cnt + " URLs to " + otherPeerName);
} else { } else {
prop.put("list", ""); prop.put("list", "");
serverLog.logInfo("URLFETCHER", "couldn't satisfy URL request of " + otherPeerName + ": stack is empty"); serverLog.logInfo("URLFETCHER", "couldn't satisfy URL request from " + otherPeerName + ": stack is empty");
} }
} else if (display.equals("count")) { } else if (display.equals("count")) {
prop.put("list", db.size()); prop.put("list", db.size());

2
source/de/anomic/http/httpHeader.java
Unescape View File

@ -237,6 +237,8 @@ public final class httpHeader extends TreeMap implements Map {
public static final String CONNECTION_PROP_PREV_REQUESTLINE = "PREVREQUESTLINE"; public static final String CONNECTION_PROP_PREV_REQUESTLINE = "PREVREQUESTLINE";
public static final String CONNECTION_PROP_REQUEST_START = "REQUEST_START"; public static final String CONNECTION_PROP_REQUEST_START = "REQUEST_START";
public static final String CONNECTION_PROP_REQUEST_END = "REQUEST_END"; public static final String CONNECTION_PROP_REQUEST_END = "REQUEST_END";
public static final String CONNECTION_PROP_INPUTSTREAM = "INPUTSTREAM";
public static final String CONNECTION_PROP_OUTPUTSTREAM = "OUTPUTSTREAM";
/* PROPERTIES: Client -> Proxy */ /* PROPERTIES: Client -> Proxy */
public static final String CONNECTION_PROP_CLIENT_REQUEST_HEADER = "CLIENT_REQUEST_HEADER"; public static final String CONNECTION_PROP_CLIENT_REQUEST_HEADER = "CLIENT_REQUEST_HEADER";

30
source/de/anomic/http/httpdFileHandler.java
Unescape View File

@ -491,8 +491,8 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http
// call an image-servlet to produce an on-the-fly - generated image // call an image-servlet to produce an on-the-fly - generated image
Object img = null; Object img = null;
try { try {
requestHeader.put("CLIENTIP", conProp.getProperty("CLIENTIP")); requestHeader.put(httpHeader.CONNECTION_PROP_CLIENTIP, conProp.getProperty("CLIENTIP"));
requestHeader.put("PATH", path); requestHeader.put(httpHeader.CONNECTION_PROP_PATH, path);
// in case that there are no args given, args = null or empty hashmap // in case that there are no args given, args = null or empty hashmap
img = invokeServlet(targetClass, requestHeader, args); img = invokeServlet(targetClass, requestHeader, args);
} catch (InvocationTargetException e) { } catch (InvocationTargetException e) {
@ -568,10 +568,10 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http
} }
} else if ((targetClass != null) && (path.endsWith(".stream"))) { } else if ((targetClass != null) && (path.endsWith(".stream"))) {
// call rewrite-class // call rewrite-class
requestHeader.put("CLIENTIP", conProp.getProperty("CLIENTIP")); requestHeader.put(httpHeader.CONNECTION_PROP_CLIENTIP, conProp.getProperty("CLIENTIP"));
requestHeader.put("PATH", path); requestHeader.put(httpHeader.CONNECTION_PROP_PATH, path);
requestHeader.put("INPUTSTREAM", body); requestHeader.put(httpHeader.CONNECTION_PROP_INPUTSTREAM, body);
requestHeader.put("OUTPUTSTREAM", out); requestHeader.put(httpHeader.CONNECTION_PROP_OUTPUTSTREAM, out);
httpd.sendRespondHeader(this.connectionProperties, out, httpVersion, 200, null); httpd.sendRespondHeader(this.connectionProperties, out, httpVersion, 200, null);
@ -609,8 +609,8 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http
} else { } else {
// CGI-class: call the class to create a property for rewriting // CGI-class: call the class to create a property for rewriting
try { try {
requestHeader.put("CLIENTIP", conProp.getProperty("CLIENTIP")); requestHeader.put(httpHeader.CONNECTION_PROP_CLIENTIP, conProp.getProperty("CLIENTIP"));
requestHeader.put("PATH", path); requestHeader.put(httpHeader.CONNECTION_PROP_PATH, path);
// in case that there are no args given, args = null or empty hashmap // in case that there are no args given, args = null or empty hashmap
Object tmp = invokeServlet(targetClass, requestHeader, args); Object tmp = invokeServlet(targetClass, requestHeader, args);
if(tmp instanceof servletProperties){ if(tmp instanceof servletProperties){
@ -621,7 +621,7 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http
// if no args given , then tp will be an empty Hashtable object (not null) // if no args given , then tp will be an empty Hashtable object (not null)
if (tp == null) tp = new servletProperties(); if (tp == null) tp = new servletProperties();
// check if the servlets requests authentification // check if the servlets requests authentification
if (tp.containsKey("AUTHENTICATE")) { if (tp.containsKey(servletProperties.ACTION_AUTHENTICATE)) {
// handle brute-force protection // handle brute-force protection
if (authorization != null) { if (authorization != null) {
String clientIP = conProp.getProperty("CLIENTIP", "unknown-host"); String clientIP = conProp.getProperty("CLIENTIP", "unknown-host");
@ -634,11 +634,11 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http
} }
// send authentication request to browser // send authentication request to browser
httpHeader headers = getDefaultHeaders(path); httpHeader headers = getDefaultHeaders(path);
headers.put(httpHeader.WWW_AUTHENTICATE,"Basic realm=\"" + tp.get("AUTHENTICATE", "") + "\""); headers.put(httpHeader.WWW_AUTHENTICATE,"Basic realm=\"" + tp.get(servletProperties.ACTION_AUTHENTICATE, "") + "\"");
httpd.sendRespondHeader(conProp,out,httpVersion,401,headers); httpd.sendRespondHeader(conProp,out,httpVersion,401,headers);
return; return;
} else if (tp.containsKey("LOCATION")) { } else if (tp.containsKey(servletProperties.ACTION_LOCATION)) {
String location = tp.get("LOCATION",""); String location = tp.get(servletProperties.ACTION_LOCATION, "");
if (location.length() == 0) location = path; if (location.length() == 0) location = path;
httpHeader headers = getDefaultHeaders(path); httpHeader headers = getDefaultHeaders(path);
@ -648,9 +648,9 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http
return; return;
} }
// add the application version, the uptime and the client name to every rewrite table // add the application version, the uptime and the client name to every rewrite table
tp.put("version", switchboard.getConfig("version", "")); tp.put(servletProperties.PEER_STAT_VERSION, switchboard.getConfig("version", ""));
tp.put("uptime", ((System.currentTimeMillis() - Long.parseLong(switchboard.getConfig("startupTime","0"))) / 1000) / 60); // uptime in minutes tp.put(servletProperties.PEER_STAT_UPTIME, ((System.currentTimeMillis() - Long.parseLong(switchboard.getConfig("startupTime","0"))) / 1000) / 60); // uptime in minutes
tp.put("clientname", switchboard.getConfig("peerName", "anomic")); tp.put(servletProperties.PEER_STAT_CLIENTNAME, switchboard.getConfig("peerName", "anomic"));
//System.out.println("respond props: " + ((tp == null) ? "null" : tp.toString())); // debug //System.out.println("respond props: " + ((tp == null) ? "null" : tp.toString())); // debug
} catch (InvocationTargetException e) { } catch (InvocationTargetException e) {
if (e.getCause() instanceof InterruptedException) { if (e.getCause() instanceof InterruptedException) {

7
source/de/anomic/server/servletProperties.java
Unescape View File

@ -27,6 +27,13 @@ public class servletProperties extends serverObjects {
private static final long serialVersionUID = 1L; private static final long serialVersionUID = 1L;
public static final String ACTION_AUTHENTICATE = "AUTHENTICATE";
public static final String ACTION_LOCATION = "LOCATION";
public static final String PEER_STAT_VERSION = "version";
public static final String PEER_STAT_UPTIME = "uptime";
public static final String PEER_STAT_CLIENTNAME = "clientname";
private String prefix=""; private String prefix="";
private httpHeader outgoingHeader; private httpHeader outgoingHeader;

Write Preview
Loading…
Cancel
Save