boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

many bug-fixes

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@73 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 20 years ago
parent 48650c082c
commit c7c6aaf06e
14 changed files with 99 additions and 100 deletions
Show Stats Download Patch File Download Diff File

20
doc/News.html
Unescape View File

@ -39,19 +39,21 @@ globalheader();
<br><p>v0.37
<ul>
<li>YaCy's source code is now hosted in a subversion version control system on berlios: <a href="http://developer.berlios.de/projects/yacy/">yacy@berlios.de</a></li>
<li>YaCy's source code is now hosted in a Subversion/svn version control system on developer.berlios.de: <a href="http://developer.berlios.de/projects/yacy/">yacy@berlios.de</a></li>
<li>overall speed enhancements:</li>
<ul>
<li>Check on new peer names: must not occur already and may only contain letters, numbers and '_' or '-'.</li>
<li>New ThreadPool and performance enhancements from Martin Thelian</li>
<li>new Thread-Pools and performance enhancements from Martin Thelian: much faster http-server and more responsive web interface</li>
<li>fixed bug in database caching that prevented from caching at all; now database much faster. This also speeded up proxy mode (must read http-header from database)</li>
<li>modified thread control for non-blocking dequeueing</li>
<li>increased cache memory settings</li>
</ul>
<li></li>
<li></li>
<li></li>
<li></li>
<li>added a concept for external parsers; pdf an doc parser are integrated but not active yet.</li>
<li>fixed several bugs that caused thread-locks and 100% CPU load</li>
<li>fixed bug with cookie storage; changed handling of multiple cookies</li>
<li>check on new peer names: must not occur already and may only contain letters, numbers and '_' or '-'.</li>
<li>many minor bug fixes and spell corrections in interface</li>
</ul>
<br><p>v0.36_build20050326
<ul>
<li>Enhanced thread control and added performance menu: this can be used to steer scheduling tasks and for profiling.</li>

13
htroot/CookieMonitorIncoming_p.java
Unescape View File

@ -63,7 +63,9 @@ public class CookieMonitorIncoming_p {
boolean dark = true;
Iterator i = switchboard.incomingCookies.entrySet().iterator();
Map.Entry entry;
String host, client, cookie;
String host, client;
Object[] cookies;
String ucl;
Date date;
Object[] oa;
while ((entCount < maxCount) && (i.hasNext())) {
@ -73,14 +75,17 @@ public class CookieMonitorIncoming_p {
oa = (Object[]) entry.getValue();
date = (Date) oa[0];
client = (String) oa[1];
cookie = (String) oa[2];
cookies = (Object[]) oa[2];
ucl = "<ul>";
for (int j = 0; j < cookies.length; j++) ucl = ucl + "<li>" + ((String) cookies[j]) + "</li>";
ucl = ucl + "</ul>";
// put values in template
prop.put("list_" + entCount + "_dark", ((dark) ? 1 : 0) ); dark =! dark;
prop.put("list_" + entCount + "_host", host);
prop.put("list_" + entCount + "_date", httpc.dateString(date));
prop.put("list_" + entCount + "_client", client);
prop.put("list_" + entCount + "_cookie", cookie);
prop.put("list_" + entCount + "_cookie", ucl);
// next
entCount++;

13
htroot/CookieMonitorOutgoing_p.java
Unescape View File

@ -63,7 +63,9 @@ public class CookieMonitorOutgoing_p {
boolean dark = true;
Iterator i = switchboard.outgoingCookies.entrySet().iterator();
Map.Entry entry;
String host, client, cookie;
String host, client;
Object[] cookies;
String ucl;
Date date;
Object[] oa;
while ((entCount < maxCount) && (i.hasNext())) {
@ -73,14 +75,17 @@ public class CookieMonitorOutgoing_p {
oa = (Object[]) entry.getValue();
date = (Date) oa[0];
client = (String) oa[1];
cookie = (String) oa[2];
cookies = (Object[]) oa[2];
ucl = "<ul>";
for (int j = 0; j < cookies.length; j++) ucl = ucl + "<li>" + ((String) cookies[j]) + "</li>";
ucl = ucl + "</ul>";
// put values in template
prop.put("list_" + entCount + "_dark", ((dark) ? 1 : 0) ); dark =! dark;
prop.put("list_" + entCount + "_host", host);
prop.put("list_" + entCount + "_date", httpc.dateString(date));
prop.put("list_" + entCount + "_client", client);
prop.put("list_" + entCount + "_cookie", cookie);
prop.put("list_" + entCount + "_cookie", ucl);
// next
entCount++;

1
htroot/env/templates/metas.template vendored
Unescape View File

@ -1,4 +1,5 @@
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
<link rel="shortcut icon" href="favicon.ico">
<meta name="Content-Language" content="English, Englisch">
<meta name="keywords" content="Anomic HTTP Proxy search engine spider indexer java network open free download Mac Windwos Software development">
<meta name="description" content="Anomic Software HTTP Proxy Freeware Home Page">

BIN
htroot/favicon.ico
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 1.3 KiB

2
makerelease.sh
Unescape View File

@ -45,7 +45,7 @@
# Contributions and changes to the program code must be marked as such.
# define variables
version='0.3681'
version='0.369'
datestr=`date +%Y%m%d`
#release='yacy_v'$version'_'$datestr
release='yacy_dev_v'$version'_'$datestr

30
source/de/anomic/http/httpHeader.java
Unescape View File

@ -111,7 +111,7 @@ public final class httpHeader extends TreeMap implements Map {
// we override the put method to make use of the reverseMappingCache
public Object put(Object key, Object value) {
String k = (String) key;
String upperK = k.toUpperCase();
String upperK = k.toUpperCase();
if (reverseMappingCache == null) {
return super.put(k, value);
} else {
@ -127,12 +127,38 @@ public final class httpHeader extends TreeMap implements Map {
}
}
// a convenience method to access the map with fail-over deafults
// to make the occurrence of multiple keys possible, we add them using a counter
public Object add(Object key, Object value) {
int c = keyCount((String) key);
if (c == 0) return put(key, value); else return put("*" + key + "-" + c, value);
}
public int keyCount(String key) {
if (!(containsKey(key))) return 0;
int c = 1;
while (containsKey("*" + key + "-" + c)) c++;
return c;
}
// a convenience method to access the map with fail-over defaults
public Object get(Object key, Object dflt) {
Object result = get(key);
if (result == null) return dflt; else return result;
}
// return multiple results
public Object getSingle(Object key, int count) {
if (count == 0) return get(key, null);
return get("*" + key + "-" + count, null);
}
public Object[] getMultiple(String key) {
int count = keyCount(key);
Object[] result = new Object[count];
for (int i = 0; i < count; i++) result[i] = getSingle(key, i);
return result;
}
// convenience methods for storing and loading to a file system
public void store(File f) throws IOException {
FileOutputStream fos = new FileOutputStream(f);

31
source/de/anomic/http/httpc.java
Unescape View File

@ -391,7 +391,6 @@ public final class httpc {
// at this point we should have a valid response. read in the header properties
String key = "";
String value = "";
while ((b = serverCore.receive(clientInput, readLineBuffer, timeout, terminalMaxLength, false)) != null) {
if (b.length == 0) break;
buffer = new String(b);
@ -406,16 +405,7 @@ public final class httpc {
// create new entry
p = buffer.indexOf(":");
if (p > 0) {
key = buffer.substring(0, p).trim();
value = (String) responseHeader.get(key);
// check if the header occurred already
if (value == null) {
// create new entry
responseHeader.put(key, buffer.substring(p + 1).trim());
} else {
// attach to old entry
responseHeader.put(key, value + "#" + buffer.substring(p + 1).trim());
}
responseHeader.add(buffer.substring(0, p).trim(), buffer.substring(p + 1).trim());
} else {
serverLog.logError("HTTPC", "RESPONSE PARSE ERROR: HOST='" + host + "', PATH='" + requestPath + "', STATUS='" + status + "'");
serverLog.logError("HTTPC", "..............BUFFER: " + buffer);
@ -640,19 +630,18 @@ public final class httpc {
Iterator i = header.keySet().iterator();
String key;
String value;
int pos;
int count;
char tag;
while (i.hasNext()) {
key = (String) i.next();
value = (String) header.get(key);
while ((pos = value.lastIndexOf("#")) >= 0) {
// special handling is needed if a key appeared several times, which is valid.
// all lines with same key are combined in one value, separated by a "#"
serverCore.send(clientOutput, key + ": " + value.substring(pos + 1).trim());
//System.out.println("**+" + key + ": " + value.substring(pos + 1).trim()); // debug
value = value.substring(0, pos).trim();
tag = key.charAt(0);
if ((tag != '*') && (tag != '#')) {
count = header.keyCount(key);
for (int j = 0; j < count; j++) {
serverCore.send(clientOutput, key + ": " + ((String) header.getSingle(key, j)).trim());
}
//System.out.println("#" + key + ": " + value);
}
serverCore.send(clientOutput, key + ": " + value);
//System.out.println("***" + key + ": " + value); // debug
}
// send terminating line

13
source/de/anomic/http/httpd.java
Unescape View File

@ -181,8 +181,6 @@ public final class httpd implements serverHandler {
httpHeader header = new httpHeader(reverseMappingCache);
int p;
String line;
String key;
String value;
while ((line = readLine()) != null) {
if (line.length() == 0) break; // this seperates the header of the HTTP request from the body
//System.out.println("***" + line); // debug
@ -190,16 +188,7 @@ public final class httpd implements serverHandler {
p = line.indexOf(":");
if (p >= 0) {
// store a property
key = line.substring(0, p).trim();
value = (String) header.get(key);
// check if the header occurred already
if (value == null) {
// create new entry
header.put(key, line.substring(p + 1).trim());
} else {
// value can occur double times, attach with '#' - separator
header.put(key, value + "#" + line.substring(p + 1).trim());
}
header.add(line.substring(0, p).trim(), line.substring(p + 1).trim());
}
}
return header;

8
source/de/anomic/http/httpdFileHandler.java
Unescape View File

@ -353,10 +353,10 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http
String mimeType = mimeTable.getProperty(conProp.getProperty("EXT",""),"text/html");
byte[] result;
if (path.endsWith("html") ||
path.endsWith("xml") ||
path.endsWith("rss") ||
path.endsWith("csv") ||
path.endsWith("pac")) {
path.endsWith("xml") ||
path.endsWith("rss") ||
path.endsWith("csv") ||
path.endsWith("pac")) {
rc = rewriteClassFile(file);
if (rc != null) {
// CGI-class: call the class to create a property for rewriting

50
source/de/anomic/http/httpdProxyHandler.java
Unescape View File

@ -223,8 +223,6 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
}
public void handleOutgoingCookies(httpHeader requestHeader, String targethost, String clienthost) {
// request header may have double-entries: they are accumulated in one entry
// by the httpd and separated by a "#" in the value field
/*
The syntax for the header is:
@ -238,14 +236,12 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
domain = "$Domain" "=" value
*/
if (requestHeader.containsKey("Cookie")) {
Object[] entry = new Object[]{new Date(), clienthost, requestHeader.get("Cookie")};
Object[] entry = new Object[]{new Date(), clienthost, requestHeader.getMultiple("Cookie")};
switchboard.outgoingCookies.put(targethost, entry);
}
}
public void handleIncomingCookies(httpHeader respondHeader, String serverhost, String targetclient) {
// respond header may have double-entries: they are accumulated in one entry
// by the httpc and separated by a "#" in the value field
/*
The syntax for the Set-Cookie response header is
@ -262,7 +258,7 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
| "Version" "=" 1*DIGIT
*/
if (respondHeader.containsKey("Set-Cookie")) {
Object[] entry = new Object[]{new Date(), targetclient, respondHeader.get("Set-Cookie")};
Object[] entry = new Object[]{new Date(), targetclient, respondHeader.getMultiple("Set-Cookie")};
switchboard.incomingCookies.put(serverhost, entry);
}
}
@ -414,8 +410,8 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
// make a transformer
if ((!(transformer.isIdentityTransformer())) &&
((ext == null) || (!(switchboard.extensionBlack.contains(ext)))) &&
((cachedResponseHeader == null) || (httpd.isTextMime(cachedResponseHeader.mime(), switchboard.mimeWhite)))) {
((ext == null) || (!(switchboard.extensionBlack.contains(ext)))) &&
((cachedResponseHeader == null) || (httpd.isTextMime(cachedResponseHeader.mime(), switchboard.mimeWhite)))) {
hfos = new htmlFilterOutputStream(respond, null, transformer, (ext.length() == 0));
} else {
hfos = respond;
@ -931,12 +927,10 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
if (!(header.containsKey("date"))) header.put("Date", httpc.dateString(httpc.nowDate()));
if (!(header.containsKey("content-type"))) header.put("Content-type", "text/html"); // fix this
StringBuffer headerStringBuffer = new StringBuffer(200);
StringBuffer headerStringBuffer = new StringBuffer(200);
// write status line
headerStringBuffer.append("HTTP/1.1 ")
.append(status)
.append("\r\n");
headerStringBuffer.append("HTTP/1.1 ").append(status).append("\r\n");
//System.out.println("HEADER: PROXY TO CLIENT = " + header.toString()); // DEBUG
@ -944,34 +938,22 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
Iterator i = header.keySet().iterator();
String key;
String value;
int pos;
char tag;
int count;
//System.out.println("vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv");
while (i.hasNext()) {
key = (String) i.next();
if (!(key.startsWith("#"))) { // '#' in key is reserved for proxy attributes as artificial header values
value = (String) header.get(key);
if (!(key.equals("Location"))) while ((pos = value.lastIndexOf("#")) >= 0) {
// special handling is needed if a key appeared several times, which is valid.
// all lines with same key are combined in one value, separated by a "#"
headerStringBuffer
.append(key)
.append(": ")
.append(value.substring(pos + 1).trim())
.append("\r\n");
//System.out.println("#" + key + ": " + value.substring(pos + 1).trim());
value = value.substring(0, pos).trim();
}
headerStringBuffer
.append(key)
.append(": ")
.append(value)
.append("\r\n");
//System.out.println("#" + key + ": " + value);
tag = key.charAt(0);
if ((tag != '*') && (tag != '#')) { // '#' in key is reserved for proxy attributes as artificial header values
count = header.keyCount(key);
for (int j = 0; j < count; j++) {
headerStringBuffer.append(key).append(": ").append((String) header.getSingle(key, j)).append("\r\n");
}
//System.out.println("#" + key + ": " + value);
}
}
headerStringBuffer.append("\r\n");
headerStringBuffer.append("\r\n");
// end header
respond.write(headerStringBuffer.toString().getBytes());
respond.flush();

7
source/de/anomic/kelondro/kelondroMScoreCluster.java
Unescape View File

@ -166,8 +166,6 @@ public class kelondroMScoreCluster {
c = cs.longValue();
gcount -= (c & 0xFFFFFFFF00000000L) >> 32;
en = (int) (c & 0xFFFFFFFFL);
// decrease overall counter
gcount -= c;
}
// set new value
@ -190,8 +188,9 @@ public class kelondroMScoreCluster {
keyrefDB.remove(cs);
refkeyDB.remove(obj);
// decrease overall counter
gcount -= cs.longValue();
return (int) ((cs.longValue() & 0xFFFFFFFF00000000L) >> 32);
long oldScore = (cs.longValue() & 0xFFFFFFFF00000000L) >> 32;
gcount -= oldScore;
return (int) oldScore;
}
}

2
source/de/anomic/plasma/plasmaCrawlWorker.java
Unescape View File

@ -183,7 +183,7 @@ public final class plasmaCrawlWorker extends Thread {
this.setName(this.threadBaseName + "_" + this.url);
load(this.url, this.referer, this.initiator, this.depth, this.profile);
} catch (IOException e) {
throw e;
//throw e;
}
finally {
this.done = true;

9
source/de/anomic/yacy/yacyClient.java
Unescape View File

@ -74,8 +74,9 @@ public class yacyClient {
HashMap result = null;
try {
/*
URL url = new URL("http://" + address + "/yacy/hello.html?iam=" + yacyCore.seedCache.mySeed.hash +
"&pattern=&count=20" +
URL url = new URL("http://" + address + "/yacy/hello.html?iam=" +
yacyCore.seedCache.mySeed.hash +
"&pattern=&count=20" +
"&key=" + key + "&seed=" + yacyCore.seedCache.mySeed.genSeedStr(key));
yacyCore.log.logDebug("HELLO to URL " + url.toString());
result = nxTools.table(httpc.wget(url,
@ -90,7 +91,7 @@ public class yacyClient {
obj.put("key", key);
obj.put("mytime", yacyCore.universalDateShortString());
obj.put("seed", yacyCore.seedDB.mySeed.genSeedStr(key));
result = nxTools.table(httpc.wput(url,
result = nxTools.table(httpc.wput(url,
20000, null, null,
yacyCore.seedDB.sb.remoteProxyHost,
yacyCore.seedDB.sb.remoteProxyPort,
@ -196,7 +197,7 @@ public class yacyClient {
String resp = (String) result.get("response");
if (resp == null) return -1; else return Integer.parseInt(resp);
} catch (Exception e) {
//yacyCore.log.logError("yacyClient.queryUrlCount error asking peer '" + target.getName() + "':" + e.toString());
yacyCore.log.logError("yacyClient.queryUrlCount error asking peer '" + target.getName() + "':" + e.toString());
return -1;
}
}

Write Preview
Loading…
Cancel
Save