many bug-fixes

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@73 6c8d7289-2bf4-0310-a012-ef5d649a1542

doc/News.html

@@ -39,19 +39,21 @@ globalheader();
 
 <br><p>v0.37
 <ul>
-<li>YaCy's source code is now hosted in a subversion version control system on berlios: <a href="http://developer.berlios.de/projects/yacy/">yacy@berlios.de</a></li>
+<li>YaCy's source code is now hosted in a Subversion/svn version control system on developer.berlios.de: <a href="http://developer.berlios.de/projects/yacy/">yacy@berlios.de</a></li>
+<li>overall speed enhancements:</li>
   <ul>
-  <li>Check on new peer names: must not occur already and may only contain letters, numbers and '_' or '-'.</li>
+  <li>new Thread-Pools and performance enhancements from Martin Thelian: much faster http-server and more responsive web interface</li>
-  <li>New ThreadPool and performance enhancements from Martin Thelian</li>
+  <li>fixed bug in database caching that prevented from caching at all; now database much faster. This also speeded up proxy mode (must read http-header from database)</li>
+  <li>modified thread control for non-blocking dequeueing</li>
+  <li>increased cache memory settings</li>
   </ul>
-<li></li>
+<li>added a concept for external parsers; pdf an doc parser are integrated but not active yet.</li>
-<li></li>
+<li>fixed several bugs that caused thread-locks and 100% CPU load</li>
-<li></li>
+<li>fixed bug with cookie storage; changed handling of multiple cookies</li>
-<li></li>
+<li>check on new peer names: must not occur already and may only contain letters, numbers and '_' or '-'.</li>
+<li>many minor bug fixes and spell corrections in interface</li>
 </ul>
 
-
-
 <br><p>v0.36_build20050326
 <ul>
 <li>Enhanced thread control and added performance menu: this can be used to steer scheduling tasks and for profiling.</li>

htroot/CookieMonitorIncoming_p.java

@@ -63,7 +63,9 @@ public class CookieMonitorIncoming_p {
         boolean dark = true;
         Iterator i = switchboard.incomingCookies.entrySet().iterator();
         Map.Entry entry;
-        String host, client, cookie;
+        String host, client;
+        Object[] cookies;
+        String ucl;
         Date date;
         Object[] oa;
         while ((entCount < maxCount) && (i.hasNext())) {
@@ -73,14 +75,17 @@ public class CookieMonitorIncoming_p {
             oa = (Object[]) entry.getValue();
             date = (Date) oa[0];
             client = (String) oa[1];
-            cookie = (String) oa[2];
+            cookies = (Object[]) oa[2];
+            ucl = "<ul>";
+            for (int j = 0; j < cookies.length; j++) ucl = ucl + "<li>" + ((String) cookies[j]) + "</li>";
+            ucl = ucl + "</ul>";
             
             // put values in template
             prop.put("list_" + entCount + "_dark", ((dark) ? 1 : 0) ); dark =! dark;
             prop.put("list_" + entCount + "_host", host);
             prop.put("list_" + entCount + "_date", httpc.dateString(date));
             prop.put("list_" + entCount + "_client", client);
-            prop.put("list_" + entCount + "_cookie", cookie);
+            prop.put("list_" + entCount + "_cookie", ucl);
             
             // next
             entCount++;

htroot/CookieMonitorOutgoing_p.java

@@ -63,7 +63,9 @@ public class CookieMonitorOutgoing_p {
         boolean dark = true;
         Iterator i = switchboard.outgoingCookies.entrySet().iterator();
         Map.Entry entry;
-        String host, client, cookie;
+        String host, client;
+        Object[] cookies;
+        String ucl;
         Date date;
         Object[] oa;
         while ((entCount < maxCount) && (i.hasNext())) {
@@ -73,14 +75,17 @@ public class CookieMonitorOutgoing_p {
             oa = (Object[]) entry.getValue();
             date = (Date) oa[0];
             client = (String) oa[1];
-            cookie = (String) oa[2];
+            cookies = (Object[]) oa[2];
+            ucl = "<ul>";
+            for (int j = 0; j < cookies.length; j++) ucl = ucl + "<li>" + ((String) cookies[j]) + "</li>";
+            ucl = ucl + "</ul>";
             
             // put values in template
             prop.put("list_" + entCount + "_dark", ((dark) ? 1 : 0) ); dark =! dark;
             prop.put("list_" + entCount + "_host", host);
             prop.put("list_" + entCount + "_date", httpc.dateString(date));
             prop.put("list_" + entCount + "_client", client);
-            prop.put("list_" + entCount + "_cookie", cookie);
+            prop.put("list_" + entCount + "_cookie", ucl);
             
             // next
             entCount++;

htroot/env/templates/metas.template

@@ -1,4 +1,5 @@
 <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
+<link rel="shortcut icon" href="favicon.ico">
 <meta name="Content-Language" content="English, Englisch">
 <meta name="keywords" content="Anomic HTTP Proxy search engine spider indexer java network open free download Mac Windwos Software development">
 <meta name="description" content="Anomic Software HTTP Proxy Freeware Home Page">

makerelease.sh

@@ -45,7 +45,7 @@
 # Contributions and changes to the program code must be marked as such.
 
 # define variables
-version='0.3681'
+version='0.369'
 datestr=`date +%Y%m%d`
 #release='yacy_v'$version'_'$datestr
 release='yacy_dev_v'$version'_'$datestr

source/de/anomic/http/httpHeader.java

@@ -127,12 +127,38 @@ public final class httpHeader extends TreeMap implements Map {
 	}
     }
 
-    // a convenience method to access the map with fail-over deafults
+    // to make the occurrence of multiple keys possible, we add them using a counter
+    public Object add(Object key, Object value) {
+        int c = keyCount((String) key);
+        if (c == 0) return put(key, value); else return put("*" + key + "-" + c, value);
+    }
+    
+    public int keyCount(String key) {
+        if (!(containsKey(key))) return 0;
+        int c = 1;
+        while (containsKey("*" + key + "-" + c)) c++;
+        return c;
+    }
+    
+    // a convenience method to access the map with fail-over defaults
     public Object get(Object key, Object dflt) {
 	Object result = get(key);
 	if (result == null) return dflt; else return result;
     }
 
+    // return multiple results
+    public Object getSingle(Object key, int count) {
+        if (count == 0) return get(key, null);
+        return get("*" + key + "-" + count, null);
+    }
+    
+    public Object[] getMultiple(String key) {
+        int count = keyCount(key);
+        Object[] result = new Object[count];
+        for (int i = 0; i < count; i++) result[i] = getSingle(key, i);
+        return result;
+    }
+    
     // convenience methods for storing and loading to a file system
     public void store(File f) throws IOException {
 	FileOutputStream fos = new FileOutputStream(f);

source/de/anomic/http/httpc.java

@@ -391,7 +391,6 @@ public final class httpc {
 
 	    // at this point we should have a valid response. read in the header properties
 	    String key = "";
-	    String value = "";
 	    while ((b = serverCore.receive(clientInput, readLineBuffer, timeout, terminalMaxLength, false)) != null) {
 		if (b.length == 0) break;
 		buffer = new String(b);
@@ -406,16 +405,7 @@ public final class httpc {
 		    // create new entry
 		    p = buffer.indexOf(":");
 		    if (p > 0) {
-			key = buffer.substring(0, p).trim();
+			responseHeader.add(buffer.substring(0, p).trim(), buffer.substring(p + 1).trim());
-			value = (String) responseHeader.get(key);
-			// check if the header occurred already
-			if (value == null) {
-			    // create new entry
-			    responseHeader.put(key, buffer.substring(p + 1).trim());
-			} else {
-			    // attach to old entry
-			    responseHeader.put(key, value + "#" + buffer.substring(p + 1).trim());
-			}
 		    } else {
 			serverLog.logError("HTTPC", "RESPONSE PARSE ERROR: HOST='" + host + "', PATH='" + requestPath + "', STATUS='" + status + "'");
 			serverLog.logError("HTTPC", "..............BUFFER: " + buffer);
@@ -640,19 +630,18 @@ public final class httpc {
 	Iterator i = header.keySet().iterator();
 	String key;
 	String value;
-	int pos;
+	int count;
+        char tag;
 	while (i.hasNext()) {
 	    key = (String) i.next();
-	    value = (String) header.get(key);
+            tag = key.charAt(0);
-	    while ((pos = value.lastIndexOf("#")) >= 0) {
+            if ((tag != '*') && (tag != '#')) {
-		// special handling is needed if a key appeared several times, which is valid.
+                count = header.keyCount(key);
-		// all lines with same key are combined in one value, separated by a "#"
+                for (int j = 0; j < count; j++) {
-		serverCore.send(clientOutput, key + ": " + value.substring(pos + 1).trim());
+                    serverCore.send(clientOutput, key + ": " + ((String) header.getSingle(key, j)).trim());
-		//System.out.println("**+" + key + ": " + value.substring(pos + 1).trim()); // debug
+                }
-		value = value.substring(0, pos).trim();
+                //System.out.println("#" + key + ": " + value);
-	    }
+	    }
-	    serverCore.send(clientOutput, key + ": " + value);
-	    //System.out.println("***" + key + ": " + value); // debug
 	}
 
 	// send terminating line

source/de/anomic/http/httpd.java

@@ -181,8 +181,6 @@ public final class httpd implements serverHandler {
     	httpHeader header = new httpHeader(reverseMappingCache);
     	int p;
     	String line;
-    	String key;
-    	String value;
     	while ((line = readLine()) != null) {
     	    if (line.length() == 0) break; // this seperates the header of the HTTP request from the body
     	    //System.out.println("***" + line); // debug
@@ -190,16 +188,7 @@ public final class httpd implements serverHandler {
     	    p = line.indexOf(":");
     	    if (p >= 0) {
     		// store a property
-    		key = line.substring(0, p).trim();
+    		header.add(line.substring(0, p).trim(), line.substring(p + 1).trim());
-    		value = (String) header.get(key);
-    		// check if the header occurred already
-    		if (value == null) {
-    		    // create new entry
-    		    header.put(key, line.substring(p + 1).trim());
-    		} else {
-    		    // value can occur double times, attach with '#' - separator
-    		    header.put(key, value + "#" + line.substring(p + 1).trim());
-    		}
     	    }
     	}
     	return header;

source/de/anomic/http/httpdProxyHandler.java

@@ -223,8 +223,6 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
     }
     
     public void handleOutgoingCookies(httpHeader requestHeader, String targethost, String clienthost) {
-        // request header may have double-entries: they are accumulated in one entry
-        // by the httpd and separated by a "#" in the value field
         /*
         The syntax for the header is:
  
@@ -238,14 +236,12 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
         domain          =       "$Domain" "=" value
         */
         if (requestHeader.containsKey("Cookie")) {
-            Object[] entry = new Object[]{new Date(), clienthost, requestHeader.get("Cookie")};
+            Object[] entry = new Object[]{new Date(), clienthost, requestHeader.getMultiple("Cookie")};
             switchboard.outgoingCookies.put(targethost, entry);
         }
     }
     
     public void handleIncomingCookies(httpHeader respondHeader, String serverhost, String targetclient) {
-        // respond header may have double-entries: they are accumulated in one entry
-        // by the httpc and separated by a "#" in the value field
         /*
         The syntax for the Set-Cookie response header is 
 
@@ -262,7 +258,7 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
                         |       "Version" "=" 1*DIGIT
         */
         if (respondHeader.containsKey("Set-Cookie")) {
-            Object[] entry = new Object[]{new Date(), targetclient, respondHeader.get("Set-Cookie")};
+            Object[] entry = new Object[]{new Date(), targetclient, respondHeader.getMultiple("Set-Cookie")};
             switchboard.incomingCookies.put(serverhost, entry);
         }
     }
@@ -934,9 +930,7 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
 	StringBuffer headerStringBuffer = new StringBuffer(200);
     
 	// write status line
-    headerStringBuffer.append("HTTP/1.1 ")
+	headerStringBuffer.append("HTTP/1.1 ").append(status).append("\r\n");
-                      .append(status)
-                      .append("\r\n");
 
 	//System.out.println("HEADER: PROXY TO CLIENT = " + header.toString()); // DEBUG
 
@@ -944,32 +938,20 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
 	Iterator i = header.keySet().iterator();
 	String key;
 	String value;
-	int pos;
+        char tag;
+        int count;
 	//System.out.println("vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv");
 	while (i.hasNext()) {
 	    key = (String) i.next();
-	    if (!(key.startsWith("#"))) { // '#' in key is reserved for proxy attributes as artificial header values
+            tag = key.charAt(0);
-		value = (String) header.get(key);
+	    if ((tag != '*') && (tag != '#')) { // '#' in key is reserved for proxy attributes as artificial header values
-		if (!(key.equals("Location"))) while ((pos = value.lastIndexOf("#")) >= 0) {
+                count = header.keyCount(key);
-		    // special handling is needed if a key appeared several times, which is valid.
+                for (int j = 0; j < count; j++) {
-		    // all lines with same key are combined in one value, separated by a "#"
+                    headerStringBuffer.append(key).append(": ").append((String) header.getSingle(key, j)).append("\r\n");  
-            headerStringBuffer
+                }
-                .append(key)
-                .append(": ")
-                .append(value.substring(pos + 1).trim())
-                .append("\r\n");
-		    //System.out.println("#" + key + ": " + value.substring(pos + 1).trim());
-		    value = value.substring(0, pos).trim();
-		}
-        headerStringBuffer
-            .append(key)
-            .append(": ")
-            .append(value)
-            .append("\r\n");        
                 //System.out.println("#" + key + ": " + value);
 	    }
 	}
-
         headerStringBuffer.append("\r\n");
 
 	// end header

source/de/anomic/kelondro/kelondroMScoreCluster.java

@@ -166,8 +166,6 @@ public class kelondroMScoreCluster {
             c = cs.longValue();
             gcount -= (c & 0xFFFFFFFF00000000L) >> 32;
             en = (int) (c & 0xFFFFFFFFL);
-            // decrease overall counter
-            gcount -= c;
         }
         
         // set new value
@@ -190,8 +188,9 @@ public class kelondroMScoreCluster {
             keyrefDB.remove(cs);
             refkeyDB.remove(obj);
             // decrease overall counter
-            gcount -= cs.longValue();
+            long oldScore = (cs.longValue() & 0xFFFFFFFF00000000L) >> 32;
-            return (int) ((cs.longValue() & 0xFFFFFFFF00000000L) >> 32);
+            gcount -= oldScore;
+            return (int) oldScore;
         }
     }
 

source/de/anomic/plasma/plasmaCrawlWorker.java

@@ -183,7 +183,7 @@ public final class plasmaCrawlWorker extends Thread {
             this.setName(this.threadBaseName + "_" + this.url);
             load(this.url, this.referer, this.initiator, this.depth, this.profile);
         } catch (IOException e) {
-            throw e;
+            //throw e;
         }
         finally {
             this.done = true;

source/de/anomic/yacy/yacyClient.java

@@ -74,7 +74,8 @@ public class yacyClient {
 	HashMap result = null;
 	try {
 	    /*
-	    URL url = new URL("http://" + address + "/yacy/hello.html?iam=" + yacyCore.seedCache.mySeed.hash +
+	    URL url = new URL("http://" + address + "/yacy/hello.html?iam=" +
+	                      yacyCore.seedCache.mySeed.hash +
                               "&pattern=&count=20" + 
 			      "&key=" + key + "&seed=" + yacyCore.seedCache.mySeed.genSeedStr(key));
 	    yacyCore.log.logDebug("HELLO to URL " + url.toString());
@@ -196,7 +197,7 @@ public class yacyClient {
             String resp = (String) result.get("response");
             if (resp == null) return -1; else return Integer.parseInt(resp);
 	} catch (Exception e) {
-            //yacyCore.log.logError("yacyClient.queryUrlCount error asking peer '" + target.getName() + "':" + e.toString());
+            yacyCore.log.logError("yacyClient.queryUrlCount error asking peer '" + target.getName() + "':" + e.toString());
 	    return -1;
 	}
     }