performance hacks

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5376 6c8d7289-2bf4-0310-a012-ef5d649a1542

bin/searchtest.sh

@@ -4,7 +4,7 @@ S=`date "+%s"`
 C=0
 for N in `cat searchtest.words`; do 
   echo search for $N:
-  ./localsearch.sh $N
+  ./localsearch.sh $N > /dev/null
   C=$(($C+1))
 done
 T=`date "+%s"`

build.properties

@@ -3,7 +3,7 @@ javacSource=1.5
 javacTarget=1.5
 
 # Release Configuration
-releaseVersion=0.615
+releaseVersion=0.616
 stdReleaseFile=yacy_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz
 embReleaseFile=yacy_emb_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz
 proReleaseFile=yacy_pro_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz

htroot/Blog.java

@@ -36,9 +36,9 @@ import java.util.Iterator;
 
 import de.anomic.data.blogBoard;
 import de.anomic.data.userDB;
-import de.anomic.http.HttpClient;
 import de.anomic.http.httpRequestHeader;
 import de.anomic.plasma.plasmaSwitchboard;
+import de.anomic.server.serverDate;
 import de.anomic.server.serverObjects;
 import de.anomic.server.serverSwitch;
 import de.anomic.yacy.yacyNewsPool;
@@ -344,7 +344,7 @@ public class Blog {
         }
 
         prop.put("mode_entries_" + number + "_date", dateString(entry.getDate()));
-        prop.put("mode_entries_" + number + "_rfc822date", HttpClient.dateString(entry.getDate()));
+        prop.put("mode_entries_" + number + "_rfc822date", serverDate.formatRFC1123(entry.getDate()));
         prop.put("mode_entries_" + number + "_pageid", entry.getKey());
         prop.put("mode_entries_" + number + "_address", address);
         prop.put("mode_entries_" + number + "_ip", entry.getIp());

htroot/Bookmarks.java

@@ -40,7 +40,6 @@ import de.anomic.data.bookmarksDB;
 import de.anomic.data.listManager;
 import de.anomic.data.userDB;
 import de.anomic.data.bookmarksDB.Tag;
-import de.anomic.http.HttpClient;
 import de.anomic.http.httpRequestHeader;
 import de.anomic.index.indexURLReference;
 import de.anomic.plasma.plasmaParserDocument;
@@ -294,7 +293,7 @@ public class Bookmarks {
        			prop.putHTML("bookmarks_"+count+"_title", bookmark.getTitle());
        			prop.putHTML("bookmarks_"+count+"_description", bookmark.getDescription());
        			prop.put("bookmarks_"+count+"_date", serverDate.formatISO8601(new Date(bookmark.getTimeStamp())));
-       			prop.put("bookmarks_"+count+"_rfc822date", HttpClient.dateString(new Date(bookmark.getTimeStamp())));
+       			prop.put("bookmarks_"+count+"_rfc822date", serverDate.formatRFC1123(new Date(bookmark.getTimeStamp())));
        			prop.put("bookmarks_"+count+"_public", (bookmark.getPublic() ? "1" : "0"));
             
        			//List Tags.

htroot/CookieMonitorIncoming_p.java

@@ -28,9 +28,9 @@ import java.util.Date;
 import java.util.Iterator;
 import java.util.Map;
 
-import de.anomic.http.HttpClient;
 import de.anomic.http.httpRequestHeader;
 import de.anomic.plasma.plasmaSwitchboard;
+import de.anomic.server.serverDate;
 import de.anomic.server.serverObjects;
 import de.anomic.server.serverSwitch;
 
@@ -78,7 +78,7 @@ public class CookieMonitorIncoming_p {
             prop.put("list_" + entCount + "_dark", dark ? "1" : "0");
             dark =! dark;
             prop.put("list_" + entCount + "_host", host);
-            prop.put("list_" + entCount + "_date", HttpClient.dateString(date));
+            prop.put("list_" + entCount + "_date", serverDate.formatRFC1123(date));
             prop.put("list_" + entCount + "_client", client);
             while (tmpCount < cookies.length){
                 prop.putHTML("list_" + entCount + "_cookies_" + tmpCount + "_item", ((String) cookies[tmpCount]));

htroot/CookieMonitorOutgoing_p.java

@@ -28,9 +28,9 @@ import java.util.Date;
 import java.util.Iterator;
 import java.util.Map;
 
-import de.anomic.http.HttpClient;
 import de.anomic.http.httpRequestHeader;
 import de.anomic.plasma.plasmaSwitchboard;
+import de.anomic.server.serverDate;
 import de.anomic.server.serverObjects;
 import de.anomic.server.serverSwitch;
 
@@ -78,7 +78,7 @@ public class CookieMonitorOutgoing_p {
             prop.put("list_" + entCount + "_dark", dark ? "1" : "0" ); 
             dark =! dark;
             prop.put("list_" + entCount + "_host", host);
-            prop.put("list_" + entCount + "_date", HttpClient.dateString(date));
+            prop.put("list_" + entCount + "_date", serverDate.formatRFC1123(date));
             prop.put("list_" + entCount + "_client", client);
             while (tmpCount < cookies.length){
                 prop.putHTML("list_" + entCount + "_cookies_" + tmpCount + "_item", ((String) cookies[tmpCount]));

htroot/Messages_p.java

@@ -33,9 +33,9 @@ import java.util.Iterator;
 import java.util.TreeMap;
 
 import de.anomic.data.messageBoard;
-import de.anomic.http.HttpClient;
 import de.anomic.http.httpRequestHeader;
 import de.anomic.plasma.plasmaSwitchboard;
+import de.anomic.server.serverDate;
 import de.anomic.server.serverFileUtils;
 import de.anomic.server.serverObjects;
 import de.anomic.server.serverSwitch;
@@ -131,7 +131,7 @@ public class Messages_p {
                     	prop.put("mode_messages_"+count+"_peerAddress", peerAddress);
 
                     	// set the rfc822 date
-                    	prop.put("mode_messages_"+count+"_rfc822Date",HttpClient.dateString(message.date()));
+                    	prop.put("mode_messages_"+count+"_rfc822Date", serverDate.formatRFC1123(message.date()));
 
                     	// also write out the message body (needed for the RSS feed)
                         try {

htroot/News.java

@@ -29,7 +29,6 @@ import java.util.Iterator;
 import java.util.Map;
 import java.util.Map.Entry;
 
-import de.anomic.http.HttpClient;
 import de.anomic.http.httpRequestHeader;
 import de.anomic.plasma.plasmaSwitchboard;
 import de.anomic.server.serverDate;
@@ -117,7 +116,7 @@ public class News {
                     prop.put("table_list_" + i + "_id", record.id());
                     prop.putHTML("table_list_" + i + "_ori", (seed == null) ? record.originator() : seed.getName());
                     prop.put("table_list_" + i + "_cre", serverDate.formatShortSecond(record.created()));
-                    prop.put("table_list_" + i + "_crerfcdate", HttpClient.dateString(record.created()));
+                    prop.put("table_list_" + i + "_crerfcdate", serverDate.formatRFC1123(record.created()));
                     prop.putHTML("table_list_" + i + "_cat", category);
                     prop.put("table_list_" + i + "_rec", (record.received() == null) ? "-" : serverDate.formatShortSecond(record.received()));
                     prop.put("table_list_" + i + "_dis", record.distributed());

htroot/Threaddump_p.java

@@ -34,6 +34,8 @@ import java.io.IOException;
 import java.util.Date;
 import java.util.Map;
 import java.util.Map.Entry;
+import java.util.ArrayList;
+import java.util.HashMap;
 
 import de.anomic.htmlFilter.htmlFilterCharacterCoding;
 import de.anomic.http.httpRequestHeader;
@@ -97,6 +99,8 @@ public class Threaddump_p {
         final File classPath = new File(rootPath, "source");
   
         Thread thread;
+        // collect single dumps
+        HashMap<String, ArrayList<String>> dumps = new HashMap<String, ArrayList<String>>();
         for (final Entry<Thread, StackTraceElement[]> entry: stackTraces.entrySet()) {
             thread = entry.getKey();
             final StackTraceElement[] stackTraceElements = entry.getValue();
@@ -104,7 +108,8 @@ public class Threaddump_p {
             String line;
             String tracename = "";
             File classFile;
-            if ((stateIn.equals(thread.getState()))  && (stackTraceElements.length > 0)) {
+            if ((stateIn.equals(thread.getState())) && (stackTraceElements.length > 0)) {
+                StringBuffer sb = new StringBuffer();
                 if (plain) {
                     classFile = getClassFile(classPath, stackTraceElements[stackTraceElements.length - 1].getClassName());
                     tracename = classFile.getName();
@@ -113,7 +118,7 @@ public class Threaddump_p {
                     while (tracename.length() < 20) tracename = tracename + "_";
                     tracename = "[" + tracename + "] ";                
                 }                
-                bufferappend(buffer, plain, tracename + "Thread= " + thread.getName() + " " + (thread.isDaemon()?"daemon":"") + " id=" + thread.getId() + " " + thread.getState().toString());
+                String threadtitle = tracename + "Thread= " + thread.getName() + " " + (thread.isDaemon()?"daemon":"") + " id=" + thread.getId() + " " + thread.getState().toString();
                 for (int i = 0; i < stackTraceElements.length; i++) {
                     ste = stackTraceElements[i];
                     if (i == 0) {
@@ -122,14 +127,26 @@ public class Threaddump_p {
                         line = null;
                     }
                     if ((line != null) && (line.length() > 0)) {
-                        bufferappend(buffer, plain, tracename + "at " + htmlFilterCharacterCoding.unicode2html(ste.toString(), true) + " [" + line.trim() + "]");
+                        bufferappend(sb, plain, tracename + "at " + htmlFilterCharacterCoding.unicode2html(ste.toString(), true) + " [" + line.trim() + "]");
                     } else {
-                        bufferappend(buffer, plain, tracename + "at " + htmlFilterCharacterCoding.unicode2html(ste.toString(), true));
+                        bufferappend(sb, plain, tracename + "at " + htmlFilterCharacterCoding.unicode2html(ste.toString(), true));
                     }
                 }
-                bufferappend(buffer, plain, "");
+                String threaddump = sb.toString();
+                ArrayList<String> threads = dumps.get(threaddump);
+                if (threads == null) threads = new ArrayList<String>();
+                threads.add(threadtitle);
+                dumps.put(threaddump, threads);
             }
         }
+        
+        // write dumps
+        for (final Entry<String, ArrayList<String>> entry: dumps.entrySet()) {
+            ArrayList<String> threads = entry.getValue();
+            for (int i = 0; i < threads.size(); i++) bufferappend(buffer, plain, threads.get(i));
+            bufferappend(buffer, plain, entry.getKey());
+            bufferappend(buffer, plain, "");
+        }
         bufferappend(buffer, plain, "");
     }
     

htroot/yacysearch.java

@@ -385,6 +385,7 @@ public class yacysearch {
                 prop.put("results_" + i + "_item", offset + i);
                 prop.put("results_" + i + "_eventID", theQuery.id(false));
                 prop.put("results_" + i + "_display", display);
+                prop.put("results_" + i + "_nl", (i < theQuery.displayResults() - 1) ? 1 : 0);
             }
             prop.put("results", theQuery.displayResults());
             prop.put("resultTable", (contentdomCode <= 1) ? "0" : "1");

htroot/yacysearch.json

@@ -18,8 +18,9 @@
     "items": [
 #{results}#
 <!--#include virtual="yacysearchitem.json?item=#[item]#&eventID=#[eventID]#" -->
+#(nl)#::    ,#(/nl)#
 #{/results}#
-    {}],
+    ],
 <!--#include virtual="yacysearchtrailer.json?eventID=#[eventID]#" -->
   }]
 }

htroot/yacysearch.rss

@@ -20,9 +20,9 @@
 		<atom:link rel="related" href="opensearchdescription.xml" type="application/opensearchdescription+xml"/>
 		<opensearch:Query role="request" searchTerms="#[rss_queryenc]#" />
 		
-		#{results}#
+#{results}#
 <!--#include virtual="yacysearchitem.xml?item=#[item]#&eventID=#[eventID]#" -->
-		#{/results}#
+#{/results}#
 		
 <!--#include virtual="yacysearchtrailer.xml?eventID=#[eventID]#" -->
 	</channel>

htroot/yacysearchitem.json

@@ -4,4 +4,4 @@
       "description": "#[description]#",
       "pubDate": "#[date822]#",
       "guid": "#[urlhash]#"
-    },#(/content)#
+    }#(/content)#

htroot/yacysearchtrailer.java

@@ -101,12 +101,13 @@ public class yacysearchtrailer {
                     prop.put("words_" + hintcount + "_offset", "0");
                     prop.put("words_" + hintcount + "_contentdom", theQuery.contentdom());
                     prop.put("words_" + hintcount + "_resource", ((theQuery.isLocal()) ? "local" : "global"));
+                    prop.put("words_" + hintcount + "_nl", (iter.hasNext() && hintcount < MAX_TOPWORDS) ? 1 : 0);
                 }
-                prop.put("words", hintcount);
                 if (hintcount++ > MAX_TOPWORDS) {
                     break;
                 }
             }
+            prop.put("words", hintcount);
         }
         serverProfiling.update("SEARCH", new plasmaProfiling.searchEvent(theQuery.id(true), plasmaSearchEvent.FINALIZATION + "-" + "bottomline", 0, 0));
         

htroot/yacysearchtrailer.json

@@ -1,5 +1,5 @@
 "yacy$topwords": [
 #{words}#
-{ "word": "#[word]#" },
+{ "word": "#[word]#" }#(nl)#::,#(/nl)#
 #{/words}#
-{}]
+]

source/de/anomic/crawler/FTPLoader.java

@@ -32,7 +32,6 @@ import java.io.IOException;
 import java.io.PrintStream;
 import java.util.Date;
 
-import de.anomic.http.JakartaCommonsHttpClient;
 import de.anomic.http.httpRequestHeader;
 import de.anomic.http.httpResponseHeader;
 import de.anomic.http.httpdProxyCacheEntry;
@@ -41,6 +40,7 @@ import de.anomic.net.ftpc;
 import de.anomic.plasma.plasmaHTCache;
 import de.anomic.plasma.plasmaParser;
 import de.anomic.plasma.plasmaSwitchboard;
+import de.anomic.server.serverDate;
 import de.anomic.server.logging.serverLog;
 import de.anomic.yacy.yacyURL;
 
@@ -60,7 +60,7 @@ public class FTPLoader {
         httpRequestHeader requestHeader = new httpRequestHeader();
         requestHeader.put(httpRequestHeader.REFERER, sb.getURL(entry.referrerhash()).toNormalform(true, false));
         httpResponseHeader responseHeader = new httpResponseHeader();
-        responseHeader.put(httpResponseHeader.LAST_MODIFIED, JakartaCommonsHttpClient.date2String(fileDate));
+        responseHeader.put(httpResponseHeader.LAST_MODIFIED, serverDate.formatRFC1123(fileDate));
         responseHeader.put(httpResponseHeader.CONTENT_TYPE, mimeType);
         indexDocumentMetadata metadata = new httpdProxyCacheEntry(entry.depth(), entry.url(), entry.name(), "OK",
                 requestHeader, responseHeader,

source/de/anomic/crawler/NoticedURL.java

@@ -222,7 +222,7 @@ public class NoticedURL {
                     if (s > balancer.size()) continue;
                     final int aftersize = balancer.size();
                     balancer.clear(); // the balancer is broken and cannot shrink
-                    throw new IOException("entry is null, balancer cannot shrink (bevore pop = " + s + ", after pop = " + aftersize + "); reset of balancer");
+                    serverLog.logWarning("BALANCER", "entry is null, balancer cannot shrink (bevore pop = " + s + ", after pop = " + aftersize + "); reset of balancer");
                 }
                 return entry;
             }

source/de/anomic/crawler/RobotsTxt.java

@@ -40,7 +40,6 @@ import java.util.LinkedList;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 
-import de.anomic.http.HttpClient;
 import de.anomic.http.JakartaCommonsHttpClient;
 import de.anomic.http.JakartaCommonsHttpResponse;
 import de.anomic.http.httpRequestHeader;
@@ -51,6 +50,7 @@ import de.anomic.kelondro.kelondroException;
 import de.anomic.kelondro.kelondroMap;
 import de.anomic.kelondro.kelondroNaturalOrder;
 import de.anomic.server.serverByteBuffer;
+import de.anomic.server.serverDate;
 import de.anomic.server.serverFileUtils;
 import de.anomic.server.logging.serverLog;
 import de.anomic.yacy.yacyURL;
@@ -516,7 +516,7 @@ public class RobotsTxt {
             oldEtag = entry.getETag();
             reqHeaders = new httpRequestHeader();
             final Date modDate = entry.getModDate();
-            if (modDate != null) reqHeaders.put(httpRequestHeader.IF_MODIFIED_SINCE,HttpClient.dateString(entry.getModDate()));
+            if (modDate != null) reqHeaders.put(httpRequestHeader.IF_MODIFIED_SINCE, serverDate.formatRFC1123(entry.getModDate()));
             
         }
         

source/de/anomic/data/URLLicense.java

@@ -26,8 +26,9 @@
 
 package de.anomic.data;
 
-import java.util.HashMap;
 import java.util.Random;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentLinkedQueue;
 
 import de.anomic.yacy.yacyURL;
 
@@ -35,13 +36,19 @@ public class URLLicense {
 
     // this class defines a license-generation for URLs
     // it is used in case of snippet- and preview-Image-fetching to grant also non-authorized users the usage of a image-fetcher servlet
-
+    private static final int maxQueue = 500;
+    private static final long minCheck = 5000;
+    
     private final Random random;
-    private final HashMap<String, yacyURL> permissions;
+    private final ConcurrentHashMap<String, yacyURL> permissions;
-    private final int keylen;
+    private final ConcurrentLinkedQueue<String> aging;
+    private long lastCheck;
+    private int keylen;
     
     public URLLicense(final int keylen) {
-        this.permissions = new HashMap<String, yacyURL>();
+        this.permissions = new ConcurrentHashMap<String, yacyURL>();
+        this.aging = new ConcurrentLinkedQueue<String>();
+        this.lastCheck = System.currentTimeMillis();
         this.random = new Random(System.currentTimeMillis());
         this.keylen = keylen;
     }
@@ -52,8 +59,16 @@ public class URLLicense {
         while (license.length() < keylen) license += Integer.toHexString(random.nextInt());
         license = license.substring(0, keylen);
         // store reference to url with license key
-        synchronized (permissions) {
+        permissions.put(license, url);
-            permissions.put(license, url);
+        aging.add(license);
+        if (System.currentTimeMillis() - this.lastCheck > minCheck) {
+            // check aging
+            this.lastCheck = System.currentTimeMillis();
+            String s;
+            while (aging.size() > maxQueue) {
+                s = aging.poll();
+                if (s != null) permissions.remove(s);
+            }
         }
         // return the license key
         return license;
@@ -61,9 +76,7 @@ public class URLLicense {
     
     public yacyURL releaseLicense(final String license) {
         yacyURL url = null;
-        synchronized (permissions) {
+        url = permissions.remove(license);
-            url = permissions.remove(license);
-        }
         /*
         if (url == null) {
             System.out.println("DEBUG-URLLICENSE: no URL license present for code=" + license);

source/de/anomic/http/HttpClient.java

@@ -27,7 +27,6 @@
 package de.anomic.http;
 
 import java.io.IOException;
-import java.util.Date;
 
 import de.anomic.server.logging.serverLog;
 
@@ -66,18 +65,6 @@ public abstract class HttpClient {
     public static String getSystemOST() {
         return systemOST;
     }
-
-    /**
-     * for easy access
-     * 
-     * @see date2String(Date)
-     * @param date
-     * @return
-     */
-    public static String dateString(final Date date) {
-        return JakartaCommonsHttpClient.date2String(date);
-    }
-
     
     /**
      * Gets a page (as raw bytes) addressing vhost at host in uri with specified header and timeout

source/de/anomic/http/JakartaCommonsHttpClient.java

@@ -30,7 +30,6 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Date;
 import java.util.List;
 import java.util.Map.Entry;
 import java.util.zip.GZIPOutputStream;
@@ -58,7 +57,6 @@ import org.apache.commons.httpclient.params.DefaultHttpParams;
 import org.apache.commons.httpclient.params.HttpMethodParams;
 import org.apache.commons.httpclient.protocol.Protocol;
 import org.apache.commons.httpclient.protocol.ProtocolSocketFactory;
-import org.apache.commons.httpclient.util.DateUtil;
 
 import de.anomic.kelondro.kelondroBase64Order;
 import de.anomic.server.logging.serverLog;
@@ -585,20 +583,6 @@ public class JakartaCommonsHttpClient {
         return hostConfig;
     }
 
-    /**
-     * Returns the given date in an HTTP-usable format. (according to RFC1123/RFC822)
-     * 
-     * @param date The Date-Object to be converted.
-     * @return String with the date.
-     */
-    public static String date2String(final Date date) { // TODO: merge this method with serverDate
-        if (date == null) {
-            return "";
-        }
-
-        return DateUtil.formatDate(date);
-    }
-
     /**
      * close all connections
      */

source/de/anomic/http/httpSSI.java

@@ -39,23 +39,24 @@ public class httpSSI {
         writeSSI(in, 0, out, authorization, requesthost);
     }
     
-    public static void writeSSI(final serverByteBuffer in, final int off, final OutputStream out, final String authorization, final String requesthost) throws IOException {
+    public static void writeSSI(final serverByteBuffer in, int off, final OutputStream out, final String authorization, final String requesthost) throws IOException {
-        final int p = in.indexOf("<!--#".getBytes(), off);
+        int p = in.indexOf("<!--#".getBytes(), off);
-        if (p >= 0) {
+        int q;
-            final int q = in.indexOf("-->".getBytes(), p + 10);
+        while (p >= 0) {
+            q = in.indexOf("-->".getBytes(), p + 10);
             if (out instanceof httpChunkedOutputStream) {
                 ((httpChunkedOutputStream) out).write(in, off, p - off);
             } else {
                 out.write(in.getBytes(off, p - off));
             }
             parseSSI(in, p, q + 3 - p, out, authorization, requesthost);
-            writeSSI(in, q + 3, out, authorization, requesthost);
+            off = q + 3;
-        } else /* p < 0 */ {
+            p = in.indexOf("<!--#".getBytes(), off);
-            if (out instanceof httpChunkedOutputStream) {
+        }
-                ((httpChunkedOutputStream) out).write(in, off, in.length() - off);
+        if (out instanceof httpChunkedOutputStream) {
-            } else {
+            ((httpChunkedOutputStream) out).write(in, off, in.length() - off);
-                out.write(in.getBytes(off, in.length() - off));
+        } else {
-            }
+            out.write(in.getBytes(off, in.length() - off));
         }
     }
     

source/de/anomic/http/httpTemplate.java

@@ -402,11 +402,11 @@ public final class httpTemplate {
                             text.append(":".getBytes("UTF-8"));
                         }
                         if(!found){
-                            text.append((byte)bb);
+                            text.append((byte)bb);/*
                             if(pis.available()==0){
                                 serverLog.logSevere("TEMPLATE", "No Close Key found for #("+new String(key)+")# (by Index)");
                                 found=true;
-                            }
+                            }*/
                         }
                     }//while
                 }//if(byName) (else branch)

source/de/anomic/http/httpd.java

@@ -67,6 +67,7 @@ import de.anomic.plasma.plasmaSwitchboard;
 import de.anomic.server.serverByteBuffer;
 import de.anomic.server.serverCodings;
 import de.anomic.server.serverCore;
+import de.anomic.server.serverDate;
 import de.anomic.server.serverDomains;
 import de.anomic.server.serverFileUtils;
 import de.anomic.server.serverHandler;
@@ -1199,7 +1200,7 @@ public final class httpd implements serverHandler, Cloneable {
             
             // Generated Tue, 23 Aug 2005 11:19:14 GMT by brain.wg (squid/2.5.STABLE3)
             // adding some system information
-            final String systemDate = HttpClient.dateString(new Date());
+            final String systemDate = serverDate.formatRFC1123(new Date());
             tp.put("date", systemDate);
             
             // rewrite the file
@@ -1299,9 +1300,9 @@ public final class httpd implements serverHandler, Cloneable {
         final Date now = new Date(System.currentTimeMillis());
         
         headers.put(httpResponseHeader.SERVER, "AnomicHTTPD (www.anomic.de)");
-        headers.put(httpResponseHeader.DATE, HttpClient.dateString(now));
+        headers.put(httpResponseHeader.DATE, serverDate.formatRFC1123(now));
         if (moddate.after(now)) moddate = now;
-        headers.put(httpResponseHeader.LAST_MODIFIED, HttpClient.dateString(moddate));
+        headers.put(httpResponseHeader.LAST_MODIFIED, serverDate.formatRFC1123(moddate));
         
         if (nocache) {
             if (httpVersion.toUpperCase().equals(httpHeader.HTTP_VERSION_1_1)) headers.put(httpResponseHeader.CACHE_CONTROL, "no-cache");
@@ -1315,7 +1316,7 @@ public final class httpd implements serverHandler, Cloneable {
         headers.put(httpHeader.CONTENT_TYPE, contentType);  
         if (contentLength > 0)   headers.put(httpResponseHeader.CONTENT_LENGTH, Long.toString(contentLength));
         //if (cookie != null)      headers.put(httpResponseHeader.SET_COOKIE, cookie);
-        if (expires != null)     headers.put(httpResponseHeader.EXPIRES, HttpClient.dateString(expires));
+        if (expires != null)     headers.put(httpResponseHeader.EXPIRES, serverDate.formatRFC1123(expires));
         if (contentEnc != null)  headers.put(httpResponseHeader.CONTENT_ENCODING, contentEnc);
         if (transferEnc != null) headers.put(httpResponseHeader.TRANSFER_ENCODING, transferEnc);
         
@@ -1366,7 +1367,7 @@ public final class httpd implements serverHandler, Cloneable {
 
                 // prepare header
                 if (!responseHeader.containsKey(httpHeader.DATE)) 
-                    responseHeader.put(httpHeader.DATE, HttpClient.dateString(new Date()));
+                    responseHeader.put(httpHeader.DATE, serverDate.formatRFC1123(new Date()));
                 if (!responseHeader.containsKey(httpHeader.CONTENT_TYPE)) 
                     responseHeader.put(httpHeader.CONTENT_TYPE, "text/html; charset=UTF-8"); // fix this
                 if (!responseHeader.containsKey(httpRequestHeader.CONNECTION) && conProp.containsKey(httpHeader.CONNECTION_PROP_PERSISTENT))

source/de/anomic/http/httpdFileHandler.java

@@ -232,7 +232,7 @@ public final class httpdFileHandler {
             ext = path.substring(pos + 1).toLowerCase();
         }
         headers.put(httpResponseHeader.SERVER, "AnomicHTTPD (www.anomic.de)");
-        headers.put(httpResponseHeader.DATE, HttpClient.dateString(new Date()));
+        headers.put(httpResponseHeader.DATE, serverDate.formatRFC1123(new Date()));
         if(!(plasmaParser.mediaExtContains(ext))){
             headers.put(httpResponseHeader.PRAGMA, "no-cache");         
         }

source/de/anomic/http/httpdProxyHandler.java

@@ -87,6 +87,7 @@ import de.anomic.plasma.plasmaParser;
 import de.anomic.plasma.plasmaSwitchboard;
 import de.anomic.plasma.plasmaSwitchboardConstants;
 import de.anomic.server.serverCore;
+import de.anomic.server.serverDate;
 import de.anomic.server.serverDomains;
 import de.anomic.server.serverFileUtils;
 import de.anomic.server.serverObjects;
@@ -714,7 +715,7 @@ public final class httpdProxyHandler {
             modifyProxyHeaders(cachedResponseHeader, httpVer);
             
             // replace date field in old header by actual date, this is according to RFC
-            cachedResponseHeader.put(httpHeader.DATE, HttpClient.dateString(new Date()));
+            cachedResponseHeader.put(httpHeader.DATE, serverDate.formatRFC1123(new Date()));
             
 //          if (((String)requestHeader.get(httpHeader.ACCEPT_ENCODING,"")).indexOf("gzip") != -1) {
 //          chunked = new httpChunkedOutputStream(respond);

source/de/anomic/icap/icapd.java

@@ -36,7 +36,6 @@ import java.net.InetAddress;
 import java.util.Date;
 import java.util.Properties;
 
-import de.anomic.http.HttpClient;
 import de.anomic.http.httpChunkedInputStream;
 import de.anomic.http.httpRequestHeader;
 import de.anomic.http.httpResponseHeader;
@@ -46,6 +45,7 @@ import de.anomic.plasma.plasmaHTCache;
 import de.anomic.plasma.plasmaParser;
 import de.anomic.plasma.plasmaSwitchboard;
 import de.anomic.server.serverCore;
+import de.anomic.server.serverDate;
 import de.anomic.server.serverFileUtils;
 import de.anomic.server.serverHandler;
 import de.anomic.server.logging.serverLog;
@@ -125,7 +125,7 @@ public class icapd implements serverHandler, Cloneable {
         final icapHeader newHeaders = new icapHeader();
         
         newHeaders.put(icapHeader.SERVER,"YaCy/" + sb.getConfig("vString",""));
-        newHeaders.put(icapHeader.DATE, HttpClient.dateString(new Date()));
+        newHeaders.put(icapHeader.DATE, serverDate.formatRFC1123(new Date()));
         newHeaders.put(icapHeader.ISTAG, "\"" + sb.getConfig("vString","") + "\"");
         
         return newHeaders;

source/de/anomic/kelondro/kelondroAbstractIOChunks.java

@@ -40,7 +40,6 @@ public abstract class kelondroAbstractIOChunks {
     
     // pseudo-native methods:
     abstract public long length() throws IOException;
-    abstract public int read(long pos, byte[] b, int off, int len) throws IOException;
     abstract public void write(long pos, byte[] b, int off, int len) throws IOException;
     abstract public void close() throws IOException;
     
@@ -50,13 +49,8 @@ public abstract class kelondroAbstractIOChunks {
         final long handle = profile.startRead();
         if (len < 0) throw new IndexOutOfBoundsException("length is negative:" + len);
         if (b.length < off + len) throw new IndexOutOfBoundsException("bounds do not fit: b.length=" + b.length + ", off=" + off + ", len=" + len);
-        while (len > 0) {
+        if (len > 0) {
-            final int r = read(pos, b, off, len); // blocks until at least one byte is available
+            readFully(pos, b, off, len);
-            if (r < 0) throw new IOException("EOF"); // read exceeded EOF
-            if (r == 0) throw new IOException("readFully cannot read remaining " + len + " bytes"); // security exception to prevent endless loops
-            pos += r;
-            off += r;
-            len -= r;
         }
         profile.stopRead(handle);
     }

source/de/anomic/kelondro/kelondroAbstractRA.java

@@ -27,7 +27,6 @@ package de.anomic.kelondro;
 
 import java.io.BufferedReader;
 import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStreamReader;
@@ -56,36 +55,16 @@ abstract class kelondroAbstractRA implements kelondroRA {
     abstract public int read() throws IOException;
     abstract public void write(int b) throws IOException;
 
-    abstract public int read(byte[] b, int off, int len) throws IOException;
     abstract public void write(byte[] b, int off, int len) throws IOException;
 
     abstract public void seek(long pos) throws IOException;
     abstract public void close() throws IOException;
 
     // derived methods:
-    public void readFully(final byte[] b, int off, int len) throws IOException {
-        if (len < 0) throw new IndexOutOfBoundsException("length is negative:" + len);
-        if (b.length < off + len) throw new IndexOutOfBoundsException("bounds do not fit: b.length=" + b.length + ", off=" + off + ", len=" + len);
-        while (len > 0) {
-            final int r = read(b, off, len);
-            if (r < 0) throw new IOException("EOF"); // read exceeded EOF
-            off += r;
-            len -= r;
-        }
-    }
-    
     public byte[] readFully() throws IOException {
-        final ByteArrayOutputStream dest = new ByteArrayOutputStream(512);
+        final byte[] buffer = new byte[(int) this.available()];
-        final byte[] buffer = new byte[1024];
+        this.readFully(buffer, 0, buffer.length);
-        
+        return buffer;
-        int c, total = 0;
-        while ((c = read(buffer, 0, 1024)) > 0) {
-            dest.write(buffer, 0, c);
-            total += c;
-        }
-        dest.flush();
-        dest.close();
-        return dest.toByteArray();
     }
     
     public byte readByte() throws IOException {
@@ -234,7 +213,7 @@ abstract class kelondroAbstractRA implements kelondroRA {
         seek(0);
         final int l = readInt();
         final byte[] b = new byte[l];
-        read(b, 0, l);
+        readFully(b, 0, l);
         return b;
     }
     

source/de/anomic/kelondro/kelondroAbstractRecords.java

@@ -971,7 +971,12 @@ public abstract class kelondroAbstractRecords implements kelondroRecords {
                     serverLog.logSevere("kelondroCachedRecords", filename + " failed with " + e.getMessage(), e);
                     return null;
                 }
-                final byte[] key = nn.getKey();
+                byte[] key = null;
+                try {
+                    key = nn.getKey();
+                } catch (IOException e1) {
+                    return null;
+                }
                 if ((key == null) ||
                     ((key.length == 1) && (key[0] == (byte) 0x80)) || // the NUL pointer ('lost' chain terminator)
                     (key.length < 3) ||

source/de/anomic/kelondro/kelondroBLOBTree.java

@@ -399,14 +399,13 @@ public class kelondroBLOBTree implements kelondroBLOB {
             put(filekey, seekpos++, i);
         }
 
-        public int read(final byte[] b, final int off, final int len) throws IOException {
+        public void readFully(final byte[] b, final int off, final int len) throws IOException {
             int l = Math.min(b.length - off, len);
             final byte[] buf = get(filekey, seekpos, l);
-            if (buf == null) return -1;
+            if (buf == null) throw new IOException("record at off " + off + ", len " + len + " not found");
             l = Math.min(buf.length, l);
             System.arraycopy(buf, 0, b, off, l);
             seekpos += l;
-            return l;
         }
 
         public void write(final byte[] b, final int off, final int len) throws IOException {

source/de/anomic/kelondro/kelondroBufferedIOChunks.java

@@ -60,7 +60,7 @@ public final class kelondroBufferedIOChunks extends kelondroAbstractIOChunks imp
         return ra.length();
     }
     
-    public synchronized int read(final long pos, final byte[] b, final int off, final int len) throws IOException {
+    public synchronized void readFully(final long pos, final byte[] b, final int off, final int len) throws IOException {
         assert (b.length >= off + len): "read pos=" + pos  + ", b.length=" + b.length + ", off=" + off + ", len=" + len;
         
         // check commit time
@@ -77,18 +77,19 @@ public final class kelondroBufferedIOChunks extends kelondroAbstractIOChunks imp
                 // entry not known, read directly from IO
                 synchronized (this.ra) {
                     this.ra.seek(pos + off);
-                    return ra.read(b, off, len);
+                    ra.readFully(b, off, len);
+                    return;
                 }
             }
             // use buffered entry
             if (bb.length >= off + len) {
                 // the buffered entry is long enough
                 System.arraycopy(bb, off, b, off, len);
-                return len;
+                return;
             }
             // the entry is not long enough. transmit only a part
             System.arraycopy(bb, off, b, off, bb.length - off);
-            return bb.length - off;
+            return;
         }
     }
 
@@ -172,4 +173,5 @@ public final class kelondroBufferedIOChunks extends kelondroAbstractIOChunks imp
     public void deleteOnExit() {
         this.ra.deleteOnExit();
     }
+
 }

source/de/anomic/kelondro/kelondroBufferedRA.java

@@ -61,11 +61,10 @@ public class kelondroBufferedRA extends kelondroAbstractRA implements kelondroRA
         return 0xff & sbb.byteAt((int) pos++);
     }
 
-    public int read(final byte[] b, final int off, final int len) throws IOException {
+    public void readFully(final byte[] b, final int off, final int len) throws IOException {
         final byte[] g = sbb.getBytes((int) pos, len);
         pos += g.length;
         System.arraycopy(g, 0, b, off, g.length);
-        return g.length;
     }
 
     public void seek(final long pos) throws IOException {

source/de/anomic/kelondro/kelondroByteArray.java

@@ -168,11 +168,6 @@ public class kelondroByteArray {
     public void write(final int to_position, final kelondroByteArray from_array, final int from_offset, final int from_length) {
         System.arraycopy(from_array.buffer, from_array.offset + from_offset, this.buffer, this.offset + to_position, from_length);
     }
-
-    public int write(final int to_position, final kelondroRA from_file, final int len) throws IOException {
-        if (len == 0) return 0;
-        return from_file.read(this.buffer, to_position, len);
-    }
     
     public static boolean equals(final byte[] buffer, final byte[] pattern) {
         return equals(buffer, 0, pattern);

source/de/anomic/kelondro/kelondroCachedRA.java

@@ -82,7 +82,7 @@ public class kelondroCachedRA extends kelondroAbstractRA implements kelondroRA {
             cache = new byte[cacheElementSize];
             //System.out.println("buffernr=" + bufferNr + ", elSize=" + bufferElementSize);
             ra.seek(cacheNr * (long) cacheElementSize);
-            ra.read(cache, 0, cacheElementSize);
+            ra.readFully(cache, 0, cacheElementSize);
             cacheMemory.put(cacheNrI, cache);
         }
         cacheScore.setScore(cacheNrI, (int) (0xFFFFFFFFL & System.currentTimeMillis()));
@@ -115,7 +115,7 @@ public class kelondroCachedRA extends kelondroAbstractRA implements kelondroRA {
         //writeBuffer(buffer, bn);
     }
 
-    public int read(final byte[] b, final int off, final int len) throws IOException {
+    public void readFully(byte[] b, int off, int len) throws IOException {
         final int bn1 = cacheElementNumber(seekpos);
         final int bn2 = cacheElementNumber(seekpos + len - 1);
         final int offset = cacheElementOffset(seekpos);
@@ -125,7 +125,7 @@ public class kelondroCachedRA extends kelondroAbstractRA implements kelondroRA {
             //System.out.println("C1: bn1=" + bn1 + ", offset=" + offset + ", off=" + off + ", len=" + len);
             System.arraycopy(buffer, offset, b, off, len);
             seekpos += len;
-            return len;
+            return;
         }
         
         // do recursively
@@ -133,7 +133,8 @@ public class kelondroCachedRA extends kelondroAbstractRA implements kelondroRA {
         //System.out.println("C2: bn1=" + bn1 + ", bn2=" + bn2 +", offset=" + offset + ", off=" + off + ", len=" + len + ", thislen=" + thislen);
         System.arraycopy(buffer, offset, b, off, thislen);
         seekpos += thislen;
-        return thislen + read(b, off + thislen, len - thislen);
+        readFully(b, off + thislen, len - thislen);
+        return;
     }
 
     public void write(final byte[] b, final int off, final int len) throws IOException {
@@ -179,4 +180,5 @@ public class kelondroCachedRA extends kelondroAbstractRA implements kelondroRA {
             close();
         } catch (final IOException e) {}
     }
+
 }

source/de/anomic/kelondro/kelondroEcoFS.java

@@ -198,8 +198,7 @@ public class kelondroEcoFS {
         
         // copy records from file to cache
         raf.seek(this.recordsize * index);
-        final int bytesRead = raf.read(this.cache, 0, this.recordsize * this.cachecount);
+        raf.readFully(this.cache, 0, this.recordsize * this.cachecount);
-        assert bytesRead == this.recordsize * this.cachecount;
     }
 
     /**

source/de/anomic/kelondro/kelondroFileRA.java

@@ -60,8 +60,8 @@ public final class kelondroFileRA extends kelondroAbstractRA implements kelondro
         RAFile.write(b);
     }
 
-    public int read(final byte[] b, final int off, final int len) throws IOException {
+    public final void readFully(final byte[] b, final int off, final int len) throws IOException {
-        return RAFile.read(b, off, len);
+        RAFile.readFully(b, off, len);
     }
 
     public void write(final byte[] b, final int off, final int len) throws IOException {

source/de/anomic/kelondro/kelondroFlexTable.java

@@ -166,7 +166,12 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr
         while (content.hasNext()) {
             node = content.next();
             i = node.handle().hashCode();
-            key = node.getKey();
+            try {
+                key = node.getKey();
+            } catch (IOException e1) {
+                e1.printStackTrace();
+                break;
+            }
             assert (key != null) : "DEBUG: empty key in initializeRamIndex"; // should not happen; if it does, it is an error of the condentNodes iterator
             //System.out.println("ENTRY: " + serverLog.arrayList(indexentry.bytes(), 0, indexentry.objectsize()));
             try { ri.addi(key, i); } catch (final IOException e) {} // no IOException can happen here

source/de/anomic/kelondro/kelondroFullRecords.java

@@ -161,10 +161,10 @@ public class kelondroFullRecords extends kelondroAbstractRecords {
 
             // read record
             this.ohChunk = new byte[overhead];
-            this.bodyChunk = new byte[ROW.objectsize];
             if (overhead > 0) entryFile.readFully(seekpos(this.handle), this.ohChunk, 0, overhead);
+            this.bodyChunk = null; /*new byte[ROW.objectsize];
             entryFile.readFully(seekpos(this.handle) + overhead, this.bodyChunk, 0, this.bodyChunk.length);
-            
+            */
             // mark chunks as not changed
             this.ohChanged = false;
             this.bodyChanged = false;
@@ -222,11 +222,17 @@ public class kelondroFullRecords extends kelondroAbstractRecords {
         public synchronized boolean valid() {
             // returns true if the key starts with non-zero byte
             // this may help to detect deleted entries
-            return (this.bodyChunk[0] != 0) && ((this.bodyChunk[0] != -128) || (this.bodyChunk[1] != 0));
+            return this.bodyChunk == null || (this.bodyChunk[0] != 0) && ((this.bodyChunk[0] != -128) || (this.bodyChunk[1] != 0));
         }
 
-        public synchronized byte[] getKey() {
+        public synchronized byte[] getKey() throws IOException {
             // read key
+            if (this.bodyChunk == null) {
+                // load all values from the database file
+                this.bodyChunk = new byte[ROW.objectsize];
+                // read values
+                entryFile.readFully(seekpos(this.handle) + overhead, this.bodyChunk, 0, this.bodyChunk.length);
+            }
             return trimCopy(this.bodyChunk, 0, ROW.width(0));
         }
 

source/de/anomic/kelondro/kelondroIOChunks.java

@@ -34,13 +34,12 @@ public interface kelondroIOChunks {
     
     // pseudo-native methods:
     public long length() throws IOException;
-    public int read(long pos, byte[] b, int off, int len) throws IOException;
+    public void readFully(long pos, byte[] b, int off, int len) throws IOException;
     public void write(long pos, byte[] b, int off, int len) throws IOException;
     public void commit() throws IOException;
     public void close() throws IOException;
 
     // derived methods:
-    public void readFully(long pos, byte[] b, int off, int len) throws IOException;
     public byte readByte(long pos) throws IOException;
     public void writeByte(long pos, int v) throws IOException;
 

source/de/anomic/kelondro/kelondroNIOFileRA.java

@@ -164,11 +164,10 @@ public class kelondroNIOFileRA extends kelondroAbstractRA implements kelondroRA
         seekPos++;
     }
 
-    public int read(final byte[] b, final int off, final int len) throws IOException {
+    public void readFully(final byte[] b, final int off, final int len) throws IOException {
-	for (int i = 0; i < len; i++) {
+        for (int i = 0; i < len; i++) {
             b[off + i] = (byte) read();
         }
-        return len;
     }
 
     public void write(final byte[] b, final int off, final int len) throws IOException {

source/de/anomic/kelondro/kelondroNode.java

@@ -38,7 +38,7 @@ public interface kelondroNode {
     public void commit() throws IOException;
     public void setValueRow(byte[] row) throws IOException;
     public boolean valid();
-    public byte[] getKey();
+    public byte[] getKey() throws IOException;
     public byte[] getValueRow() throws IOException;
     public String toString();
 

source/de/anomic/kelondro/kelondroRA.java

@@ -46,16 +46,15 @@ public interface kelondroRA {
     public long available() throws IOException;
     
     public int read() throws IOException;
+    public void readFully(byte[] b, int off, int len) throws IOException;
     public void write(int b) throws IOException;
 
-    public int read(byte[] b, int off, int len) throws IOException;
     public void write(byte[] b, int off, int len) throws IOException;
 
     public void seek(long pos) throws IOException;
     public void close() throws IOException;
 
     // derived methods:
-    public void readFully(byte[] b, int off, int len) throws IOException;
     public byte[] readFully() throws IOException;
     public byte readByte() throws IOException;
     public void writeByte(int v) throws IOException;

source/de/anomic/kelondro/kelondroRAIOChunks.java

@@ -40,19 +40,6 @@ public final class kelondroRAIOChunks extends kelondroAbstractIOChunks implement
     public synchronized long length() throws IOException {
         return ra.length();
     }
-    
-    public synchronized int read(final long pos, final byte[] b, final int off, final int len) throws IOException {
-        if (len == 0) return 0;
-        this.ra.seek(pos);
-        final long available = ra.available();
-        if (available >= len) {
-            return ra.read(b, off, len);
-        } else if (available == 0) {
-            return -1;
-        } else {
-            return ra.read(b, off, (int) available);
-        }
-    }
 
     public synchronized void write(final long pos, final byte[] b, final int off, final int len) throws IOException {
         this.ra.seek(pos);

source/de/anomic/server/serverDate.java

@@ -139,10 +139,26 @@ public final class serverDate {
      * @param date The Date instance to transform.
      * @return A fixed width (20 chars) ISO8601 date String.
      */
-    public static String formatISO8601(final Date date){
+    public static final String formatISO8601(final Date date) {
-            return format(FORMAT_ISO8601, date);
+        if (date == null) return "";
+        return format(FORMAT_ISO8601, date);
     }
 
+    private static long lastRFC1123long = 0;
+    private static String lastRFC1123string = "";
+    
+    public static final String formatRFC1123(final Date date) {
+        if (date == null) return "";
+        if (date.getTime() - lastRFC1123long < 1000) {
+            //System.out.println("date cache hit - " + lastRFC1123string);
+            return lastRFC1123string;
+        }
+        String s = format(FORMAT_RFC1123, date);
+        lastRFC1123long = date.getTime();
+        lastRFC1123string = s;
+        return s;
+    }
+    
     /**
      * Parse dates as defined in {@linkplain http://www.w3.org/TR/NOTE-datetime}.
      * This format (also specified in ISO8601) allows different "precisions".

source/de/anomic/yacy/yacyURL.java

@@ -229,6 +229,8 @@ public class yacyURL implements Serializable {
         escape();
     }
 
+    private final Pattern backPathPattern = Pattern.compile("(/[^/]+(?<!/\\.{1,2})/)[.]{2}(?=/|$)|/\\.(?=/)|/(?=/)");
+    
     //  resolve '..'
     String resolveBackpath(String path) /* throws MalformedURLException */ {
         /* original version by [MC]
@@ -243,8 +245,7 @@ public class yacyURL implements Serializable {
         /* by [MT] */
         if (path.length() == 0 || path.charAt(0) != '/') { path = "/" + path; }
 
-        final Pattern pathPattern = Pattern.compile("(/[^/]+(?<!/\\.{1,2})/)[.]{2}(?=/|$)|/\\.(?=/)|/(?=/)");
+        final Matcher matcher = backPathPattern.matcher(path);
-        final Matcher matcher = pathPattern.matcher(path);
         while (matcher.find()) {
             path = matcher.replaceAll("");
             matcher.reset(path);