diff --git a/doc/News.html b/doc/News.html
index 3961c8615..1a04044e4 100644
--- a/doc/News.html
+++ b/doc/News.html
@@ -39,19 +39,21 @@ globalheader();
v0.37
-YaCy's source code is now hosted in a subversion version control system on berlios: yacy@berlios.de
+YaCy's source code is now hosted in a Subversion/svn version control system on developer.berlios.de: yacy@berlios.de
+overall speed enhancements:
- Check on new peer names: must not occur already and may only contain letters, numbers and '_' or '-'.
- New ThreadPool and performance enhancements from Martin Thelian
+ new Thread-Pools and performance enhancements from Martin Thelian: much faster http-server and more responsive web interface
+ fixed bug in database caching that prevented from caching at all; now database much faster. This also speeded up proxy mode (must read http-header from database)
+ modified thread control for non-blocking dequeueing
+ increased cache memory settings
-
-
-
-
+added a concept for external parsers; pdf an doc parser are integrated but not active yet.
+fixed several bugs that caused thread-locks and 100% CPU load
+fixed bug with cookie storage; changed handling of multiple cookies
+check on new peer names: must not occur already and may only contain letters, numbers and '_' or '-'.
+many minor bug fixes and spell corrections in interface
-
-
v0.36_build20050326
Enhanced thread control and added performance menu: this can be used to steer scheduling tasks and for profiling.
diff --git a/htroot/CookieMonitorIncoming_p.java b/htroot/CookieMonitorIncoming_p.java
index 137e0a6f0..121038d40 100644
--- a/htroot/CookieMonitorIncoming_p.java
+++ b/htroot/CookieMonitorIncoming_p.java
@@ -63,7 +63,9 @@ public class CookieMonitorIncoming_p {
boolean dark = true;
Iterator i = switchboard.incomingCookies.entrySet().iterator();
Map.Entry entry;
- String host, client, cookie;
+ String host, client;
+ Object[] cookies;
+ String ucl;
Date date;
Object[] oa;
while ((entCount < maxCount) && (i.hasNext())) {
@@ -73,14 +75,17 @@ public class CookieMonitorIncoming_p {
oa = (Object[]) entry.getValue();
date = (Date) oa[0];
client = (String) oa[1];
- cookie = (String) oa[2];
-
+ cookies = (Object[]) oa[2];
+ ucl = "";
+ for (int j = 0; j < cookies.length; j++) ucl = ucl + "" + ((String) cookies[j]) + " ";
+ ucl = ucl + " ";
+
// put values in template
prop.put("list_" + entCount + "_dark", ((dark) ? 1 : 0) ); dark =! dark;
prop.put("list_" + entCount + "_host", host);
prop.put("list_" + entCount + "_date", httpc.dateString(date));
prop.put("list_" + entCount + "_client", client);
- prop.put("list_" + entCount + "_cookie", cookie);
+ prop.put("list_" + entCount + "_cookie", ucl);
// next
entCount++;
diff --git a/htroot/CookieMonitorOutgoing_p.java b/htroot/CookieMonitorOutgoing_p.java
index c7b882335..25e5e5ed6 100644
--- a/htroot/CookieMonitorOutgoing_p.java
+++ b/htroot/CookieMonitorOutgoing_p.java
@@ -63,7 +63,9 @@ public class CookieMonitorOutgoing_p {
boolean dark = true;
Iterator i = switchboard.outgoingCookies.entrySet().iterator();
Map.Entry entry;
- String host, client, cookie;
+ String host, client;
+ Object[] cookies;
+ String ucl;
Date date;
Object[] oa;
while ((entCount < maxCount) && (i.hasNext())) {
@@ -73,14 +75,17 @@ public class CookieMonitorOutgoing_p {
oa = (Object[]) entry.getValue();
date = (Date) oa[0];
client = (String) oa[1];
- cookie = (String) oa[2];
-
+ cookies = (Object[]) oa[2];
+ ucl = "";
+ for (int j = 0; j < cookies.length; j++) ucl = ucl + "" + ((String) cookies[j]) + " ";
+ ucl = ucl + " ";
+
// put values in template
prop.put("list_" + entCount + "_dark", ((dark) ? 1 : 0) ); dark =! dark;
prop.put("list_" + entCount + "_host", host);
prop.put("list_" + entCount + "_date", httpc.dateString(date));
prop.put("list_" + entCount + "_client", client);
- prop.put("list_" + entCount + "_cookie", cookie);
+ prop.put("list_" + entCount + "_cookie", ucl);
// next
entCount++;
diff --git a/htroot/env/templates/metas.template b/htroot/env/templates/metas.template
index 622c8d2a3..9356e37ad 100644
--- a/htroot/env/templates/metas.template
+++ b/htroot/env/templates/metas.template
@@ -1,4 +1,5 @@
+
diff --git a/htroot/favicon.ico b/htroot/favicon.ico
new file mode 100644
index 000000000..4d6424dd9
Binary files /dev/null and b/htroot/favicon.ico differ
diff --git a/makerelease.sh b/makerelease.sh
index b4d8ebb39..a6eb1cb41 100755
--- a/makerelease.sh
+++ b/makerelease.sh
@@ -45,7 +45,7 @@
# Contributions and changes to the program code must be marked as such.
# define variables
-version='0.3681'
+version='0.369'
datestr=`date +%Y%m%d`
#release='yacy_v'$version'_'$datestr
release='yacy_dev_v'$version'_'$datestr
diff --git a/source/de/anomic/http/httpHeader.java b/source/de/anomic/http/httpHeader.java
index c043d6cb3..0de74f7ea 100644
--- a/source/de/anomic/http/httpHeader.java
+++ b/source/de/anomic/http/httpHeader.java
@@ -111,7 +111,7 @@ public final class httpHeader extends TreeMap implements Map {
// we override the put method to make use of the reverseMappingCache
public Object put(Object key, Object value) {
String k = (String) key;
- String upperK = k.toUpperCase();
+ String upperK = k.toUpperCase();
if (reverseMappingCache == null) {
return super.put(k, value);
} else {
@@ -127,12 +127,38 @@ public final class httpHeader extends TreeMap implements Map {
}
}
- // a convenience method to access the map with fail-over deafults
+ // to make the occurrence of multiple keys possible, we add them using a counter
+ public Object add(Object key, Object value) {
+ int c = keyCount((String) key);
+ if (c == 0) return put(key, value); else return put("*" + key + "-" + c, value);
+ }
+
+ public int keyCount(String key) {
+ if (!(containsKey(key))) return 0;
+ int c = 1;
+ while (containsKey("*" + key + "-" + c)) c++;
+ return c;
+ }
+
+ // a convenience method to access the map with fail-over defaults
public Object get(Object key, Object dflt) {
Object result = get(key);
if (result == null) return dflt; else return result;
}
+ // return multiple results
+ public Object getSingle(Object key, int count) {
+ if (count == 0) return get(key, null);
+ return get("*" + key + "-" + count, null);
+ }
+
+ public Object[] getMultiple(String key) {
+ int count = keyCount(key);
+ Object[] result = new Object[count];
+ for (int i = 0; i < count; i++) result[i] = getSingle(key, i);
+ return result;
+ }
+
// convenience methods for storing and loading to a file system
public void store(File f) throws IOException {
FileOutputStream fos = new FileOutputStream(f);
diff --git a/source/de/anomic/http/httpc.java b/source/de/anomic/http/httpc.java
index bcc42d858..f5d4fceea 100644
--- a/source/de/anomic/http/httpc.java
+++ b/source/de/anomic/http/httpc.java
@@ -391,7 +391,6 @@ public final class httpc {
// at this point we should have a valid response. read in the header properties
String key = "";
- String value = "";
while ((b = serverCore.receive(clientInput, readLineBuffer, timeout, terminalMaxLength, false)) != null) {
if (b.length == 0) break;
buffer = new String(b);
@@ -406,16 +405,7 @@ public final class httpc {
// create new entry
p = buffer.indexOf(":");
if (p > 0) {
- key = buffer.substring(0, p).trim();
- value = (String) responseHeader.get(key);
- // check if the header occurred already
- if (value == null) {
- // create new entry
- responseHeader.put(key, buffer.substring(p + 1).trim());
- } else {
- // attach to old entry
- responseHeader.put(key, value + "#" + buffer.substring(p + 1).trim());
- }
+ responseHeader.add(buffer.substring(0, p).trim(), buffer.substring(p + 1).trim());
} else {
serverLog.logError("HTTPC", "RESPONSE PARSE ERROR: HOST='" + host + "', PATH='" + requestPath + "', STATUS='" + status + "'");
serverLog.logError("HTTPC", "..............BUFFER: " + buffer);
@@ -640,19 +630,18 @@ public final class httpc {
Iterator i = header.keySet().iterator();
String key;
String value;
- int pos;
+ int count;
+ char tag;
while (i.hasNext()) {
key = (String) i.next();
- value = (String) header.get(key);
- while ((pos = value.lastIndexOf("#")) >= 0) {
- // special handling is needed if a key appeared several times, which is valid.
- // all lines with same key are combined in one value, separated by a "#"
- serverCore.send(clientOutput, key + ": " + value.substring(pos + 1).trim());
- //System.out.println("**+" + key + ": " + value.substring(pos + 1).trim()); // debug
- value = value.substring(0, pos).trim();
+ tag = key.charAt(0);
+ if ((tag != '*') && (tag != '#')) {
+ count = header.keyCount(key);
+ for (int j = 0; j < count; j++) {
+ serverCore.send(clientOutput, key + ": " + ((String) header.getSingle(key, j)).trim());
+ }
+ //System.out.println("#" + key + ": " + value);
}
- serverCore.send(clientOutput, key + ": " + value);
- //System.out.println("***" + key + ": " + value); // debug
}
// send terminating line
diff --git a/source/de/anomic/http/httpd.java b/source/de/anomic/http/httpd.java
index 59276e64f..2dd6ecf2f 100644
--- a/source/de/anomic/http/httpd.java
+++ b/source/de/anomic/http/httpd.java
@@ -181,8 +181,6 @@ public final class httpd implements serverHandler {
httpHeader header = new httpHeader(reverseMappingCache);
int p;
String line;
- String key;
- String value;
while ((line = readLine()) != null) {
if (line.length() == 0) break; // this seperates the header of the HTTP request from the body
//System.out.println("***" + line); // debug
@@ -190,16 +188,7 @@ public final class httpd implements serverHandler {
p = line.indexOf(":");
if (p >= 0) {
// store a property
- key = line.substring(0, p).trim();
- value = (String) header.get(key);
- // check if the header occurred already
- if (value == null) {
- // create new entry
- header.put(key, line.substring(p + 1).trim());
- } else {
- // value can occur double times, attach with '#' - separator
- header.put(key, value + "#" + line.substring(p + 1).trim());
- }
+ header.add(line.substring(0, p).trim(), line.substring(p + 1).trim());
}
}
return header;
diff --git a/source/de/anomic/http/httpdFileHandler.java b/source/de/anomic/http/httpdFileHandler.java
index 4badd3624..2be6344a5 100644
--- a/source/de/anomic/http/httpdFileHandler.java
+++ b/source/de/anomic/http/httpdFileHandler.java
@@ -353,10 +353,10 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http
String mimeType = mimeTable.getProperty(conProp.getProperty("EXT",""),"text/html");
byte[] result;
if (path.endsWith("html") ||
- path.endsWith("xml") ||
- path.endsWith("rss") ||
- path.endsWith("csv") ||
- path.endsWith("pac")) {
+ path.endsWith("xml") ||
+ path.endsWith("rss") ||
+ path.endsWith("csv") ||
+ path.endsWith("pac")) {
rc = rewriteClassFile(file);
if (rc != null) {
// CGI-class: call the class to create a property for rewriting
diff --git a/source/de/anomic/http/httpdProxyHandler.java b/source/de/anomic/http/httpdProxyHandler.java
index 980b7945e..aac8762fb 100644
--- a/source/de/anomic/http/httpdProxyHandler.java
+++ b/source/de/anomic/http/httpdProxyHandler.java
@@ -223,8 +223,6 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
}
public void handleOutgoingCookies(httpHeader requestHeader, String targethost, String clienthost) {
- // request header may have double-entries: they are accumulated in one entry
- // by the httpd and separated by a "#" in the value field
/*
The syntax for the header is:
@@ -238,14 +236,12 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
domain = "$Domain" "=" value
*/
if (requestHeader.containsKey("Cookie")) {
- Object[] entry = new Object[]{new Date(), clienthost, requestHeader.get("Cookie")};
+ Object[] entry = new Object[]{new Date(), clienthost, requestHeader.getMultiple("Cookie")};
switchboard.outgoingCookies.put(targethost, entry);
}
}
public void handleIncomingCookies(httpHeader respondHeader, String serverhost, String targetclient) {
- // respond header may have double-entries: they are accumulated in one entry
- // by the httpc and separated by a "#" in the value field
/*
The syntax for the Set-Cookie response header is
@@ -262,7 +258,7 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
| "Version" "=" 1*DIGIT
*/
if (respondHeader.containsKey("Set-Cookie")) {
- Object[] entry = new Object[]{new Date(), targetclient, respondHeader.get("Set-Cookie")};
+ Object[] entry = new Object[]{new Date(), targetclient, respondHeader.getMultiple("Set-Cookie")};
switchboard.incomingCookies.put(serverhost, entry);
}
}
@@ -414,8 +410,8 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
// make a transformer
if ((!(transformer.isIdentityTransformer())) &&
- ((ext == null) || (!(switchboard.extensionBlack.contains(ext)))) &&
- ((cachedResponseHeader == null) || (httpd.isTextMime(cachedResponseHeader.mime(), switchboard.mimeWhite)))) {
+ ((ext == null) || (!(switchboard.extensionBlack.contains(ext)))) &&
+ ((cachedResponseHeader == null) || (httpd.isTextMime(cachedResponseHeader.mime(), switchboard.mimeWhite)))) {
hfos = new htmlFilterOutputStream(respond, null, transformer, (ext.length() == 0));
} else {
hfos = respond;
@@ -931,12 +927,10 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
if (!(header.containsKey("date"))) header.put("Date", httpc.dateString(httpc.nowDate()));
if (!(header.containsKey("content-type"))) header.put("Content-type", "text/html"); // fix this
- StringBuffer headerStringBuffer = new StringBuffer(200);
+ StringBuffer headerStringBuffer = new StringBuffer(200);
// write status line
- headerStringBuffer.append("HTTP/1.1 ")
- .append(status)
- .append("\r\n");
+ headerStringBuffer.append("HTTP/1.1 ").append(status).append("\r\n");
//System.out.println("HEADER: PROXY TO CLIENT = " + header.toString()); // DEBUG
@@ -944,34 +938,22 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
Iterator i = header.keySet().iterator();
String key;
String value;
- int pos;
+ char tag;
+ int count;
//System.out.println("vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv");
while (i.hasNext()) {
key = (String) i.next();
- if (!(key.startsWith("#"))) { // '#' in key is reserved for proxy attributes as artificial header values
- value = (String) header.get(key);
- if (!(key.equals("Location"))) while ((pos = value.lastIndexOf("#")) >= 0) {
- // special handling is needed if a key appeared several times, which is valid.
- // all lines with same key are combined in one value, separated by a "#"
- headerStringBuffer
- .append(key)
- .append(": ")
- .append(value.substring(pos + 1).trim())
- .append("\r\n");
- //System.out.println("#" + key + ": " + value.substring(pos + 1).trim());
- value = value.substring(0, pos).trim();
- }
- headerStringBuffer
- .append(key)
- .append(": ")
- .append(value)
- .append("\r\n");
- //System.out.println("#" + key + ": " + value);
+ tag = key.charAt(0);
+ if ((tag != '*') && (tag != '#')) { // '#' in key is reserved for proxy attributes as artificial header values
+ count = header.keyCount(key);
+ for (int j = 0; j < count; j++) {
+ headerStringBuffer.append(key).append(": ").append((String) header.getSingle(key, j)).append("\r\n");
+ }
+ //System.out.println("#" + key + ": " + value);
}
}
+ headerStringBuffer.append("\r\n");
- headerStringBuffer.append("\r\n");
-
// end header
respond.write(headerStringBuffer.toString().getBytes());
respond.flush();
diff --git a/source/de/anomic/kelondro/kelondroMScoreCluster.java b/source/de/anomic/kelondro/kelondroMScoreCluster.java
index f1898ce0b..a9e84cef5 100644
--- a/source/de/anomic/kelondro/kelondroMScoreCluster.java
+++ b/source/de/anomic/kelondro/kelondroMScoreCluster.java
@@ -166,8 +166,6 @@ public class kelondroMScoreCluster {
c = cs.longValue();
gcount -= (c & 0xFFFFFFFF00000000L) >> 32;
en = (int) (c & 0xFFFFFFFFL);
- // decrease overall counter
- gcount -= c;
}
// set new value
@@ -190,8 +188,9 @@ public class kelondroMScoreCluster {
keyrefDB.remove(cs);
refkeyDB.remove(obj);
// decrease overall counter
- gcount -= cs.longValue();
- return (int) ((cs.longValue() & 0xFFFFFFFF00000000L) >> 32);
+ long oldScore = (cs.longValue() & 0xFFFFFFFF00000000L) >> 32;
+ gcount -= oldScore;
+ return (int) oldScore;
}
}
diff --git a/source/de/anomic/plasma/plasmaCrawlWorker.java b/source/de/anomic/plasma/plasmaCrawlWorker.java
index fbe38eef8..d5a9926d0 100644
--- a/source/de/anomic/plasma/plasmaCrawlWorker.java
+++ b/source/de/anomic/plasma/plasmaCrawlWorker.java
@@ -183,7 +183,7 @@ public final class plasmaCrawlWorker extends Thread {
this.setName(this.threadBaseName + "_" + this.url);
load(this.url, this.referer, this.initiator, this.depth, this.profile);
} catch (IOException e) {
- throw e;
+ //throw e;
}
finally {
this.done = true;
diff --git a/source/de/anomic/yacy/yacyClient.java b/source/de/anomic/yacy/yacyClient.java
index e943b4e8e..bfc45757a 100644
--- a/source/de/anomic/yacy/yacyClient.java
+++ b/source/de/anomic/yacy/yacyClient.java
@@ -74,8 +74,9 @@ public class yacyClient {
HashMap result = null;
try {
/*
- URL url = new URL("http://" + address + "/yacy/hello.html?iam=" + yacyCore.seedCache.mySeed.hash +
- "&pattern=&count=20" +
+ URL url = new URL("http://" + address + "/yacy/hello.html?iam=" +
+ yacyCore.seedCache.mySeed.hash +
+ "&pattern=&count=20" +
"&key=" + key + "&seed=" + yacyCore.seedCache.mySeed.genSeedStr(key));
yacyCore.log.logDebug("HELLO to URL " + url.toString());
result = nxTools.table(httpc.wget(url,
@@ -90,7 +91,7 @@ public class yacyClient {
obj.put("key", key);
obj.put("mytime", yacyCore.universalDateShortString());
obj.put("seed", yacyCore.seedDB.mySeed.genSeedStr(key));
- result = nxTools.table(httpc.wput(url,
+ result = nxTools.table(httpc.wput(url,
20000, null, null,
yacyCore.seedDB.sb.remoteProxyHost,
yacyCore.seedDB.sb.remoteProxyPort,
@@ -196,7 +197,7 @@ public class yacyClient {
String resp = (String) result.get("response");
if (resp == null) return -1; else return Integer.parseInt(resp);
} catch (Exception e) {
- //yacyCore.log.logError("yacyClient.queryUrlCount error asking peer '" + target.getName() + "':" + e.toString());
+ yacyCore.log.logError("yacyClient.queryUrlCount error asking peer '" + target.getName() + "':" + e.toString());
return -1;
}
}