performance hacks

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5376 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 16 years ago
parent 5280ad638d
commit d39d420b39

@ -4,7 +4,7 @@ S=`date "+%s"`
C=0
for N in `cat searchtest.words`; do
echo search for $N:
./localsearch.sh $N
./localsearch.sh $N > /dev/null
C=$(($C+1))
done
T=`date "+%s"`

@ -3,7 +3,7 @@ javacSource=1.5
javacTarget=1.5
# Release Configuration
releaseVersion=0.615
releaseVersion=0.616
stdReleaseFile=yacy_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz
embReleaseFile=yacy_emb_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz
proReleaseFile=yacy_pro_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz

@ -36,9 +36,9 @@ import java.util.Iterator;
import de.anomic.data.blogBoard;
import de.anomic.data.userDB;
import de.anomic.http.HttpClient;
import de.anomic.http.httpRequestHeader;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverDate;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.yacy.yacyNewsPool;
@ -344,7 +344,7 @@ public class Blog {
}
prop.put("mode_entries_" + number + "_date", dateString(entry.getDate()));
prop.put("mode_entries_" + number + "_rfc822date", HttpClient.dateString(entry.getDate()));
prop.put("mode_entries_" + number + "_rfc822date", serverDate.formatRFC1123(entry.getDate()));
prop.put("mode_entries_" + number + "_pageid", entry.getKey());
prop.put("mode_entries_" + number + "_address", address);
prop.put("mode_entries_" + number + "_ip", entry.getIp());

@ -40,7 +40,6 @@ import de.anomic.data.bookmarksDB;
import de.anomic.data.listManager;
import de.anomic.data.userDB;
import de.anomic.data.bookmarksDB.Tag;
import de.anomic.http.HttpClient;
import de.anomic.http.httpRequestHeader;
import de.anomic.index.indexURLReference;
import de.anomic.plasma.plasmaParserDocument;
@ -294,7 +293,7 @@ public class Bookmarks {
prop.putHTML("bookmarks_"+count+"_title", bookmark.getTitle());
prop.putHTML("bookmarks_"+count+"_description", bookmark.getDescription());
prop.put("bookmarks_"+count+"_date", serverDate.formatISO8601(new Date(bookmark.getTimeStamp())));
prop.put("bookmarks_"+count+"_rfc822date", HttpClient.dateString(new Date(bookmark.getTimeStamp())));
prop.put("bookmarks_"+count+"_rfc822date", serverDate.formatRFC1123(new Date(bookmark.getTimeStamp())));
prop.put("bookmarks_"+count+"_public", (bookmark.getPublic() ? "1" : "0"));
//List Tags.

@ -28,9 +28,9 @@ import java.util.Date;
import java.util.Iterator;
import java.util.Map;
import de.anomic.http.HttpClient;
import de.anomic.http.httpRequestHeader;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverDate;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
@ -78,7 +78,7 @@ public class CookieMonitorIncoming_p {
prop.put("list_" + entCount + "_dark", dark ? "1" : "0");
dark =! dark;
prop.put("list_" + entCount + "_host", host);
prop.put("list_" + entCount + "_date", HttpClient.dateString(date));
prop.put("list_" + entCount + "_date", serverDate.formatRFC1123(date));
prop.put("list_" + entCount + "_client", client);
while (tmpCount < cookies.length){
prop.putHTML("list_" + entCount + "_cookies_" + tmpCount + "_item", ((String) cookies[tmpCount]));

@ -28,9 +28,9 @@ import java.util.Date;
import java.util.Iterator;
import java.util.Map;
import de.anomic.http.HttpClient;
import de.anomic.http.httpRequestHeader;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverDate;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
@ -78,7 +78,7 @@ public class CookieMonitorOutgoing_p {
prop.put("list_" + entCount + "_dark", dark ? "1" : "0" );
dark =! dark;
prop.put("list_" + entCount + "_host", host);
prop.put("list_" + entCount + "_date", HttpClient.dateString(date));
prop.put("list_" + entCount + "_date", serverDate.formatRFC1123(date));
prop.put("list_" + entCount + "_client", client);
while (tmpCount < cookies.length){
prop.putHTML("list_" + entCount + "_cookies_" + tmpCount + "_item", ((String) cookies[tmpCount]));

@ -33,9 +33,9 @@ import java.util.Iterator;
import java.util.TreeMap;
import de.anomic.data.messageBoard;
import de.anomic.http.HttpClient;
import de.anomic.http.httpRequestHeader;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverDate;
import de.anomic.server.serverFileUtils;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
@ -131,7 +131,7 @@ public class Messages_p {
prop.put("mode_messages_"+count+"_peerAddress", peerAddress);
// set the rfc822 date
prop.put("mode_messages_"+count+"_rfc822Date",HttpClient.dateString(message.date()));
prop.put("mode_messages_"+count+"_rfc822Date", serverDate.formatRFC1123(message.date()));
// also write out the message body (needed for the RSS feed)
try {

@ -29,7 +29,6 @@ import java.util.Iterator;
import java.util.Map;
import java.util.Map.Entry;
import de.anomic.http.HttpClient;
import de.anomic.http.httpRequestHeader;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverDate;
@ -117,7 +116,7 @@ public class News {
prop.put("table_list_" + i + "_id", record.id());
prop.putHTML("table_list_" + i + "_ori", (seed == null) ? record.originator() : seed.getName());
prop.put("table_list_" + i + "_cre", serverDate.formatShortSecond(record.created()));
prop.put("table_list_" + i + "_crerfcdate", HttpClient.dateString(record.created()));
prop.put("table_list_" + i + "_crerfcdate", serverDate.formatRFC1123(record.created()));
prop.putHTML("table_list_" + i + "_cat", category);
prop.put("table_list_" + i + "_rec", (record.received() == null) ? "-" : serverDate.formatShortSecond(record.received()));
prop.put("table_list_" + i + "_dis", record.distributed());

@ -34,6 +34,8 @@ import java.io.IOException;
import java.util.Date;
import java.util.Map;
import java.util.Map.Entry;
import java.util.ArrayList;
import java.util.HashMap;
import de.anomic.htmlFilter.htmlFilterCharacterCoding;
import de.anomic.http.httpRequestHeader;
@ -97,6 +99,8 @@ public class Threaddump_p {
final File classPath = new File(rootPath, "source");
Thread thread;
// collect single dumps
HashMap<String, ArrayList<String>> dumps = new HashMap<String, ArrayList<String>>();
for (final Entry<Thread, StackTraceElement[]> entry: stackTraces.entrySet()) {
thread = entry.getKey();
final StackTraceElement[] stackTraceElements = entry.getValue();
@ -104,7 +108,8 @@ public class Threaddump_p {
String line;
String tracename = "";
File classFile;
if ((stateIn.equals(thread.getState())) && (stackTraceElements.length > 0)) {
if ((stateIn.equals(thread.getState())) && (stackTraceElements.length > 0)) {
StringBuffer sb = new StringBuffer();
if (plain) {
classFile = getClassFile(classPath, stackTraceElements[stackTraceElements.length - 1].getClassName());
tracename = classFile.getName();
@ -113,7 +118,7 @@ public class Threaddump_p {
while (tracename.length() < 20) tracename = tracename + "_";
tracename = "[" + tracename + "] ";
}
bufferappend(buffer, plain, tracename + "Thread= " + thread.getName() + " " + (thread.isDaemon()?"daemon":"") + " id=" + thread.getId() + " " + thread.getState().toString());
String threadtitle = tracename + "Thread= " + thread.getName() + " " + (thread.isDaemon()?"daemon":"") + " id=" + thread.getId() + " " + thread.getState().toString();
for (int i = 0; i < stackTraceElements.length; i++) {
ste = stackTraceElements[i];
if (i == 0) {
@ -122,14 +127,26 @@ public class Threaddump_p {
line = null;
}
if ((line != null) && (line.length() > 0)) {
bufferappend(buffer, plain, tracename + "at " + htmlFilterCharacterCoding.unicode2html(ste.toString(), true) + " [" + line.trim() + "]");
bufferappend(sb, plain, tracename + "at " + htmlFilterCharacterCoding.unicode2html(ste.toString(), true) + " [" + line.trim() + "]");
} else {
bufferappend(buffer, plain, tracename + "at " + htmlFilterCharacterCoding.unicode2html(ste.toString(), true));
bufferappend(sb, plain, tracename + "at " + htmlFilterCharacterCoding.unicode2html(ste.toString(), true));
}
}
bufferappend(buffer, plain, "");
String threaddump = sb.toString();
ArrayList<String> threads = dumps.get(threaddump);
if (threads == null) threads = new ArrayList<String>();
threads.add(threadtitle);
dumps.put(threaddump, threads);
}
}
// write dumps
for (final Entry<String, ArrayList<String>> entry: dumps.entrySet()) {
ArrayList<String> threads = entry.getValue();
for (int i = 0; i < threads.size(); i++) bufferappend(buffer, plain, threads.get(i));
bufferappend(buffer, plain, entry.getKey());
bufferappend(buffer, plain, "");
}
bufferappend(buffer, plain, "");
}

@ -385,6 +385,7 @@ public class yacysearch {
prop.put("results_" + i + "_item", offset + i);
prop.put("results_" + i + "_eventID", theQuery.id(false));
prop.put("results_" + i + "_display", display);
prop.put("results_" + i + "_nl", (i < theQuery.displayResults() - 1) ? 1 : 0);
}
prop.put("results", theQuery.displayResults());
prop.put("resultTable", (contentdomCode <= 1) ? "0" : "1");

@ -18,8 +18,9 @@
"items": [
#{results}#
<!--#include virtual="yacysearchitem.json?item=#[item]#&eventID=#[eventID]#" -->
#(nl)#:: ,#(/nl)#
#{/results}#
{}],
],
<!--#include virtual="yacysearchtrailer.json?eventID=#[eventID]#" -->
}]
}

@ -20,9 +20,9 @@
<atom:link rel="related" href="opensearchdescription.xml" type="application/opensearchdescription+xml"/>
<opensearch:Query role="request" searchTerms="#[rss_queryenc]#" />
#{results}#
#{results}#
<!--#include virtual="yacysearchitem.xml?item=#[item]#&eventID=#[eventID]#" -->
#{/results}#
#{/results}#
<!--#include virtual="yacysearchtrailer.xml?eventID=#[eventID]#" -->
</channel>

@ -4,4 +4,4 @@
"description": "#[description]#",
"pubDate": "#[date822]#",
"guid": "#[urlhash]#"
},#(/content)#
}#(/content)#

@ -101,12 +101,13 @@ public class yacysearchtrailer {
prop.put("words_" + hintcount + "_offset", "0");
prop.put("words_" + hintcount + "_contentdom", theQuery.contentdom());
prop.put("words_" + hintcount + "_resource", ((theQuery.isLocal()) ? "local" : "global"));
prop.put("words_" + hintcount + "_nl", (iter.hasNext() && hintcount < MAX_TOPWORDS) ? 1 : 0);
}
prop.put("words", hintcount);
if (hintcount++ > MAX_TOPWORDS) {
break;
}
}
prop.put("words", hintcount);
}
serverProfiling.update("SEARCH", new plasmaProfiling.searchEvent(theQuery.id(true), plasmaSearchEvent.FINALIZATION + "-" + "bottomline", 0, 0));

@ -1,5 +1,5 @@
"yacy$topwords": [
#{words}#
{ "word": "#[word]#" },
{ "word": "#[word]#" }#(nl)#::,#(/nl)#
#{/words}#
{}]
]

@ -32,7 +32,6 @@ import java.io.IOException;
import java.io.PrintStream;
import java.util.Date;
import de.anomic.http.JakartaCommonsHttpClient;
import de.anomic.http.httpRequestHeader;
import de.anomic.http.httpResponseHeader;
import de.anomic.http.httpdProxyCacheEntry;
@ -41,6 +40,7 @@ import de.anomic.net.ftpc;
import de.anomic.plasma.plasmaHTCache;
import de.anomic.plasma.plasmaParser;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverDate;
import de.anomic.server.logging.serverLog;
import de.anomic.yacy.yacyURL;
@ -60,7 +60,7 @@ public class FTPLoader {
httpRequestHeader requestHeader = new httpRequestHeader();
requestHeader.put(httpRequestHeader.REFERER, sb.getURL(entry.referrerhash()).toNormalform(true, false));
httpResponseHeader responseHeader = new httpResponseHeader();
responseHeader.put(httpResponseHeader.LAST_MODIFIED, JakartaCommonsHttpClient.date2String(fileDate));
responseHeader.put(httpResponseHeader.LAST_MODIFIED, serverDate.formatRFC1123(fileDate));
responseHeader.put(httpResponseHeader.CONTENT_TYPE, mimeType);
indexDocumentMetadata metadata = new httpdProxyCacheEntry(entry.depth(), entry.url(), entry.name(), "OK",
requestHeader, responseHeader,

@ -222,7 +222,7 @@ public class NoticedURL {
if (s > balancer.size()) continue;
final int aftersize = balancer.size();
balancer.clear(); // the balancer is broken and cannot shrink
throw new IOException("entry is null, balancer cannot shrink (bevore pop = " + s + ", after pop = " + aftersize + "); reset of balancer");
serverLog.logWarning("BALANCER", "entry is null, balancer cannot shrink (bevore pop = " + s + ", after pop = " + aftersize + "); reset of balancer");
}
return entry;
}

@ -40,7 +40,6 @@ import java.util.LinkedList;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import de.anomic.http.HttpClient;
import de.anomic.http.JakartaCommonsHttpClient;
import de.anomic.http.JakartaCommonsHttpResponse;
import de.anomic.http.httpRequestHeader;
@ -51,6 +50,7 @@ import de.anomic.kelondro.kelondroException;
import de.anomic.kelondro.kelondroMap;
import de.anomic.kelondro.kelondroNaturalOrder;
import de.anomic.server.serverByteBuffer;
import de.anomic.server.serverDate;
import de.anomic.server.serverFileUtils;
import de.anomic.server.logging.serverLog;
import de.anomic.yacy.yacyURL;
@ -516,7 +516,7 @@ public class RobotsTxt {
oldEtag = entry.getETag();
reqHeaders = new httpRequestHeader();
final Date modDate = entry.getModDate();
if (modDate != null) reqHeaders.put(httpRequestHeader.IF_MODIFIED_SINCE,HttpClient.dateString(entry.getModDate()));
if (modDate != null) reqHeaders.put(httpRequestHeader.IF_MODIFIED_SINCE, serverDate.formatRFC1123(entry.getModDate()));
}

@ -26,8 +26,9 @@
package de.anomic.data;
import java.util.HashMap;
import java.util.Random;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentLinkedQueue;
import de.anomic.yacy.yacyURL;
@ -35,13 +36,19 @@ public class URLLicense {
// this class defines a license-generation for URLs
// it is used in case of snippet- and preview-Image-fetching to grant also non-authorized users the usage of a image-fetcher servlet
private static final int maxQueue = 500;
private static final long minCheck = 5000;
private final Random random;
private final HashMap<String, yacyURL> permissions;
private final int keylen;
private final ConcurrentHashMap<String, yacyURL> permissions;
private final ConcurrentLinkedQueue<String> aging;
private long lastCheck;
private int keylen;
public URLLicense(final int keylen) {
this.permissions = new HashMap<String, yacyURL>();
this.permissions = new ConcurrentHashMap<String, yacyURL>();
this.aging = new ConcurrentLinkedQueue<String>();
this.lastCheck = System.currentTimeMillis();
this.random = new Random(System.currentTimeMillis());
this.keylen = keylen;
}
@ -52,8 +59,16 @@ public class URLLicense {
while (license.length() < keylen) license += Integer.toHexString(random.nextInt());
license = license.substring(0, keylen);
// store reference to url with license key
synchronized (permissions) {
permissions.put(license, url);
permissions.put(license, url);
aging.add(license);
if (System.currentTimeMillis() - this.lastCheck > minCheck) {
// check aging
this.lastCheck = System.currentTimeMillis();
String s;
while (aging.size() > maxQueue) {
s = aging.poll();
if (s != null) permissions.remove(s);
}
}
// return the license key
return license;
@ -61,9 +76,7 @@ public class URLLicense {
public yacyURL releaseLicense(final String license) {
yacyURL url = null;
synchronized (permissions) {
url = permissions.remove(license);
}
url = permissions.remove(license);
/*
if (url == null) {
System.out.println("DEBUG-URLLICENSE: no URL license present for code=" + license);

@ -27,7 +27,6 @@
package de.anomic.http;
import java.io.IOException;
import java.util.Date;
import de.anomic.server.logging.serverLog;
@ -67,18 +66,6 @@ public abstract class HttpClient {
return systemOST;
}
/**
* for easy access
*
* @see date2String(Date)
* @param date
* @return
*/
public static String dateString(final Date date) {
return JakartaCommonsHttpClient.date2String(date);
}
/**
* Gets a page (as raw bytes) addressing vhost at host in uri with specified header and timeout
*

@ -30,7 +30,6 @@ import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.List;
import java.util.Map.Entry;
import java.util.zip.GZIPOutputStream;
@ -58,7 +57,6 @@ import org.apache.commons.httpclient.params.DefaultHttpParams;
import org.apache.commons.httpclient.params.HttpMethodParams;
import org.apache.commons.httpclient.protocol.Protocol;
import org.apache.commons.httpclient.protocol.ProtocolSocketFactory;
import org.apache.commons.httpclient.util.DateUtil;
import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.server.logging.serverLog;
@ -585,20 +583,6 @@ public class JakartaCommonsHttpClient {
return hostConfig;
}
/**
* Returns the given date in an HTTP-usable format. (according to RFC1123/RFC822)
*
* @param date The Date-Object to be converted.
* @return String with the date.
*/
public static String date2String(final Date date) { // TODO: merge this method with serverDate
if (date == null) {
return "";
}
return DateUtil.formatDate(date);
}
/**
* close all connections
*/

@ -39,23 +39,24 @@ public class httpSSI {
writeSSI(in, 0, out, authorization, requesthost);
}
public static void writeSSI(final serverByteBuffer in, final int off, final OutputStream out, final String authorization, final String requesthost) throws IOException {
final int p = in.indexOf("<!--#".getBytes(), off);
if (p >= 0) {
final int q = in.indexOf("-->".getBytes(), p + 10);
public static void writeSSI(final serverByteBuffer in, int off, final OutputStream out, final String authorization, final String requesthost) throws IOException {
int p = in.indexOf("<!--#".getBytes(), off);
int q;
while (p >= 0) {
q = in.indexOf("-->".getBytes(), p + 10);
if (out instanceof httpChunkedOutputStream) {
((httpChunkedOutputStream) out).write(in, off, p - off);
} else {
out.write(in.getBytes(off, p - off));
}
parseSSI(in, p, q + 3 - p, out, authorization, requesthost);
writeSSI(in, q + 3, out, authorization, requesthost);
} else /* p < 0 */ {
if (out instanceof httpChunkedOutputStream) {
((httpChunkedOutputStream) out).write(in, off, in.length() - off);
} else {
out.write(in.getBytes(off, in.length() - off));
}
off = q + 3;
p = in.indexOf("<!--#".getBytes(), off);
}
if (out instanceof httpChunkedOutputStream) {
((httpChunkedOutputStream) out).write(in, off, in.length() - off);
} else {
out.write(in.getBytes(off, in.length() - off));
}
}

@ -402,11 +402,11 @@ public final class httpTemplate {
text.append(":".getBytes("UTF-8"));
}
if(!found){
text.append((byte)bb);
text.append((byte)bb);/*
if(pis.available()==0){
serverLog.logSevere("TEMPLATE", "No Close Key found for #("+new String(key)+")# (by Index)");
found=true;
}
}*/
}
}//while
}//if(byName) (else branch)

@ -67,6 +67,7 @@ import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverByteBuffer;
import de.anomic.server.serverCodings;
import de.anomic.server.serverCore;
import de.anomic.server.serverDate;
import de.anomic.server.serverDomains;
import de.anomic.server.serverFileUtils;
import de.anomic.server.serverHandler;
@ -1199,7 +1200,7 @@ public final class httpd implements serverHandler, Cloneable {
// Generated Tue, 23 Aug 2005 11:19:14 GMT by brain.wg (squid/2.5.STABLE3)
// adding some system information
final String systemDate = HttpClient.dateString(new Date());
final String systemDate = serverDate.formatRFC1123(new Date());
tp.put("date", systemDate);
// rewrite the file
@ -1299,9 +1300,9 @@ public final class httpd implements serverHandler, Cloneable {
final Date now = new Date(System.currentTimeMillis());
headers.put(httpResponseHeader.SERVER, "AnomicHTTPD (www.anomic.de)");
headers.put(httpResponseHeader.DATE, HttpClient.dateString(now));
headers.put(httpResponseHeader.DATE, serverDate.formatRFC1123(now));
if (moddate.after(now)) moddate = now;
headers.put(httpResponseHeader.LAST_MODIFIED, HttpClient.dateString(moddate));
headers.put(httpResponseHeader.LAST_MODIFIED, serverDate.formatRFC1123(moddate));
if (nocache) {
if (httpVersion.toUpperCase().equals(httpHeader.HTTP_VERSION_1_1)) headers.put(httpResponseHeader.CACHE_CONTROL, "no-cache");
@ -1315,7 +1316,7 @@ public final class httpd implements serverHandler, Cloneable {
headers.put(httpHeader.CONTENT_TYPE, contentType);
if (contentLength > 0) headers.put(httpResponseHeader.CONTENT_LENGTH, Long.toString(contentLength));
//if (cookie != null) headers.put(httpResponseHeader.SET_COOKIE, cookie);
if (expires != null) headers.put(httpResponseHeader.EXPIRES, HttpClient.dateString(expires));
if (expires != null) headers.put(httpResponseHeader.EXPIRES, serverDate.formatRFC1123(expires));
if (contentEnc != null) headers.put(httpResponseHeader.CONTENT_ENCODING, contentEnc);
if (transferEnc != null) headers.put(httpResponseHeader.TRANSFER_ENCODING, transferEnc);
@ -1366,7 +1367,7 @@ public final class httpd implements serverHandler, Cloneable {
// prepare header
if (!responseHeader.containsKey(httpHeader.DATE))
responseHeader.put(httpHeader.DATE, HttpClient.dateString(new Date()));
responseHeader.put(httpHeader.DATE, serverDate.formatRFC1123(new Date()));
if (!responseHeader.containsKey(httpHeader.CONTENT_TYPE))
responseHeader.put(httpHeader.CONTENT_TYPE, "text/html; charset=UTF-8"); // fix this
if (!responseHeader.containsKey(httpRequestHeader.CONNECTION) && conProp.containsKey(httpHeader.CONNECTION_PROP_PERSISTENT))

@ -232,7 +232,7 @@ public final class httpdFileHandler {
ext = path.substring(pos + 1).toLowerCase();
}
headers.put(httpResponseHeader.SERVER, "AnomicHTTPD (www.anomic.de)");
headers.put(httpResponseHeader.DATE, HttpClient.dateString(new Date()));
headers.put(httpResponseHeader.DATE, serverDate.formatRFC1123(new Date()));
if(!(plasmaParser.mediaExtContains(ext))){
headers.put(httpResponseHeader.PRAGMA, "no-cache");
}

@ -87,6 +87,7 @@ import de.anomic.plasma.plasmaParser;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.plasma.plasmaSwitchboardConstants;
import de.anomic.server.serverCore;
import de.anomic.server.serverDate;
import de.anomic.server.serverDomains;
import de.anomic.server.serverFileUtils;
import de.anomic.server.serverObjects;
@ -714,7 +715,7 @@ public final class httpdProxyHandler {
modifyProxyHeaders(cachedResponseHeader, httpVer);
// replace date field in old header by actual date, this is according to RFC
cachedResponseHeader.put(httpHeader.DATE, HttpClient.dateString(new Date()));
cachedResponseHeader.put(httpHeader.DATE, serverDate.formatRFC1123(new Date()));
// if (((String)requestHeader.get(httpHeader.ACCEPT_ENCODING,"")).indexOf("gzip") != -1) {
// chunked = new httpChunkedOutputStream(respond);

@ -36,7 +36,6 @@ import java.net.InetAddress;
import java.util.Date;
import java.util.Properties;
import de.anomic.http.HttpClient;
import de.anomic.http.httpChunkedInputStream;
import de.anomic.http.httpRequestHeader;
import de.anomic.http.httpResponseHeader;
@ -46,6 +45,7 @@ import de.anomic.plasma.plasmaHTCache;
import de.anomic.plasma.plasmaParser;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverCore;
import de.anomic.server.serverDate;
import de.anomic.server.serverFileUtils;
import de.anomic.server.serverHandler;
import de.anomic.server.logging.serverLog;
@ -125,7 +125,7 @@ public class icapd implements serverHandler, Cloneable {
final icapHeader newHeaders = new icapHeader();
newHeaders.put(icapHeader.SERVER,"YaCy/" + sb.getConfig("vString",""));
newHeaders.put(icapHeader.DATE, HttpClient.dateString(new Date()));
newHeaders.put(icapHeader.DATE, serverDate.formatRFC1123(new Date()));
newHeaders.put(icapHeader.ISTAG, "\"" + sb.getConfig("vString","") + "\"");
return newHeaders;

@ -40,7 +40,6 @@ public abstract class kelondroAbstractIOChunks {
// pseudo-native methods:
abstract public long length() throws IOException;
abstract public int read(long pos, byte[] b, int off, int len) throws IOException;
abstract public void write(long pos, byte[] b, int off, int len) throws IOException;
abstract public void close() throws IOException;
@ -50,13 +49,8 @@ public abstract class kelondroAbstractIOChunks {
final long handle = profile.startRead();
if (len < 0) throw new IndexOutOfBoundsException("length is negative:" + len);
if (b.length < off + len) throw new IndexOutOfBoundsException("bounds do not fit: b.length=" + b.length + ", off=" + off + ", len=" + len);
while (len > 0) {
final int r = read(pos, b, off, len); // blocks until at least one byte is available
if (r < 0) throw new IOException("EOF"); // read exceeded EOF
if (r == 0) throw new IOException("readFully cannot read remaining " + len + " bytes"); // security exception to prevent endless loops
pos += r;
off += r;
len -= r;
if (len > 0) {
readFully(pos, b, off, len);
}
profile.stopRead(handle);
}

@ -27,7 +27,6 @@ package de.anomic.kelondro;
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStreamReader;
@ -56,36 +55,16 @@ abstract class kelondroAbstractRA implements kelondroRA {
abstract public int read() throws IOException;
abstract public void write(int b) throws IOException;
abstract public int read(byte[] b, int off, int len) throws IOException;
abstract public void write(byte[] b, int off, int len) throws IOException;
abstract public void seek(long pos) throws IOException;
abstract public void close() throws IOException;
// derived methods:
public void readFully(final byte[] b, int off, int len) throws IOException {
if (len < 0) throw new IndexOutOfBoundsException("length is negative:" + len);
if (b.length < off + len) throw new IndexOutOfBoundsException("bounds do not fit: b.length=" + b.length + ", off=" + off + ", len=" + len);
while (len > 0) {
final int r = read(b, off, len);
if (r < 0) throw new IOException("EOF"); // read exceeded EOF
off += r;
len -= r;
}
}
public byte[] readFully() throws IOException {
final ByteArrayOutputStream dest = new ByteArrayOutputStream(512);
final byte[] buffer = new byte[1024];
int c, total = 0;
while ((c = read(buffer, 0, 1024)) > 0) {
dest.write(buffer, 0, c);
total += c;
}
dest.flush();
dest.close();
return dest.toByteArray();
final byte[] buffer = new byte[(int) this.available()];
this.readFully(buffer, 0, buffer.length);
return buffer;
}
public byte readByte() throws IOException {
@ -234,7 +213,7 @@ abstract class kelondroAbstractRA implements kelondroRA {
seek(0);
final int l = readInt();
final byte[] b = new byte[l];
read(b, 0, l);
readFully(b, 0, l);
return b;
}

@ -971,7 +971,12 @@ public abstract class kelondroAbstractRecords implements kelondroRecords {
serverLog.logSevere("kelondroCachedRecords", filename + " failed with " + e.getMessage(), e);
return null;
}
final byte[] key = nn.getKey();
byte[] key = null;
try {
key = nn.getKey();
} catch (IOException e1) {
return null;
}
if ((key == null) ||
((key.length == 1) && (key[0] == (byte) 0x80)) || // the NUL pointer ('lost' chain terminator)
(key.length < 3) ||

@ -399,14 +399,13 @@ public class kelondroBLOBTree implements kelondroBLOB {
put(filekey, seekpos++, i);
}
public int read(final byte[] b, final int off, final int len) throws IOException {
public void readFully(final byte[] b, final int off, final int len) throws IOException {
int l = Math.min(b.length - off, len);
final byte[] buf = get(filekey, seekpos, l);
if (buf == null) return -1;
if (buf == null) throw new IOException("record at off " + off + ", len " + len + " not found");
l = Math.min(buf.length, l);
System.arraycopy(buf, 0, b, off, l);
seekpos += l;
return l;
}
public void write(final byte[] b, final int off, final int len) throws IOException {

@ -60,7 +60,7 @@ public final class kelondroBufferedIOChunks extends kelondroAbstractIOChunks imp
return ra.length();
}
public synchronized int read(final long pos, final byte[] b, final int off, final int len) throws IOException {
public synchronized void readFully(final long pos, final byte[] b, final int off, final int len) throws IOException {
assert (b.length >= off + len): "read pos=" + pos + ", b.length=" + b.length + ", off=" + off + ", len=" + len;
// check commit time
@ -77,18 +77,19 @@ public final class kelondroBufferedIOChunks extends kelondroAbstractIOChunks imp
// entry not known, read directly from IO
synchronized (this.ra) {
this.ra.seek(pos + off);
return ra.read(b, off, len);
ra.readFully(b, off, len);
return;
}
}
// use buffered entry
if (bb.length >= off + len) {
// the buffered entry is long enough
System.arraycopy(bb, off, b, off, len);
return len;
return;
}
// the entry is not long enough. transmit only a part
System.arraycopy(bb, off, b, off, bb.length - off);
return bb.length - off;
return;
}
}
@ -172,4 +173,5 @@ public final class kelondroBufferedIOChunks extends kelondroAbstractIOChunks imp
public void deleteOnExit() {
this.ra.deleteOnExit();
}
}

@ -61,11 +61,10 @@ public class kelondroBufferedRA extends kelondroAbstractRA implements kelondroRA
return 0xff & sbb.byteAt((int) pos++);
}
public int read(final byte[] b, final int off, final int len) throws IOException {
public void readFully(final byte[] b, final int off, final int len) throws IOException {
final byte[] g = sbb.getBytes((int) pos, len);
pos += g.length;
System.arraycopy(g, 0, b, off, g.length);
return g.length;
}
public void seek(final long pos) throws IOException {

@ -169,11 +169,6 @@ public class kelondroByteArray {
System.arraycopy(from_array.buffer, from_array.offset + from_offset, this.buffer, this.offset + to_position, from_length);
}
public int write(final int to_position, final kelondroRA from_file, final int len) throws IOException {
if (len == 0) return 0;
return from_file.read(this.buffer, to_position, len);
}
public static boolean equals(final byte[] buffer, final byte[] pattern) {
return equals(buffer, 0, pattern);
}

@ -82,7 +82,7 @@ public class kelondroCachedRA extends kelondroAbstractRA implements kelondroRA {
cache = new byte[cacheElementSize];
//System.out.println("buffernr=" + bufferNr + ", elSize=" + bufferElementSize);
ra.seek(cacheNr * (long) cacheElementSize);
ra.read(cache, 0, cacheElementSize);
ra.readFully(cache, 0, cacheElementSize);
cacheMemory.put(cacheNrI, cache);
}
cacheScore.setScore(cacheNrI, (int) (0xFFFFFFFFL & System.currentTimeMillis()));
@ -115,7 +115,7 @@ public class kelondroCachedRA extends kelondroAbstractRA implements kelondroRA {
//writeBuffer(buffer, bn);
}
public int read(final byte[] b, final int off, final int len) throws IOException {
public void readFully(byte[] b, int off, int len) throws IOException {
final int bn1 = cacheElementNumber(seekpos);
final int bn2 = cacheElementNumber(seekpos + len - 1);
final int offset = cacheElementOffset(seekpos);
@ -125,7 +125,7 @@ public class kelondroCachedRA extends kelondroAbstractRA implements kelondroRA {
//System.out.println("C1: bn1=" + bn1 + ", offset=" + offset + ", off=" + off + ", len=" + len);
System.arraycopy(buffer, offset, b, off, len);
seekpos += len;
return len;
return;
}
// do recursively
@ -133,7 +133,8 @@ public class kelondroCachedRA extends kelondroAbstractRA implements kelondroRA {
//System.out.println("C2: bn1=" + bn1 + ", bn2=" + bn2 +", offset=" + offset + ", off=" + off + ", len=" + len + ", thislen=" + thislen);
System.arraycopy(buffer, offset, b, off, thislen);
seekpos += thislen;
return thislen + read(b, off + thislen, len - thislen);
readFully(b, off + thislen, len - thislen);
return;
}
public void write(final byte[] b, final int off, final int len) throws IOException {
@ -179,4 +180,5 @@ public class kelondroCachedRA extends kelondroAbstractRA implements kelondroRA {
close();
} catch (final IOException e) {}
}
}

@ -198,8 +198,7 @@ public class kelondroEcoFS {
// copy records from file to cache
raf.seek(this.recordsize * index);
final int bytesRead = raf.read(this.cache, 0, this.recordsize * this.cachecount);
assert bytesRead == this.recordsize * this.cachecount;
raf.readFully(this.cache, 0, this.recordsize * this.cachecount);
}
/**

@ -60,8 +60,8 @@ public final class kelondroFileRA extends kelondroAbstractRA implements kelondro
RAFile.write(b);
}
public int read(final byte[] b, final int off, final int len) throws IOException {
return RAFile.read(b, off, len);
public final void readFully(final byte[] b, final int off, final int len) throws IOException {
RAFile.readFully(b, off, len);
}
public void write(final byte[] b, final int off, final int len) throws IOException {

@ -166,7 +166,12 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr
while (content.hasNext()) {
node = content.next();
i = node.handle().hashCode();
key = node.getKey();
try {
key = node.getKey();
} catch (IOException e1) {
e1.printStackTrace();
break;
}
assert (key != null) : "DEBUG: empty key in initializeRamIndex"; // should not happen; if it does, it is an error of the condentNodes iterator
//System.out.println("ENTRY: " + serverLog.arrayList(indexentry.bytes(), 0, indexentry.objectsize()));
try { ri.addi(key, i); } catch (final IOException e) {} // no IOException can happen here

@ -161,10 +161,10 @@ public class kelondroFullRecords extends kelondroAbstractRecords {
// read record
this.ohChunk = new byte[overhead];
this.bodyChunk = new byte[ROW.objectsize];
if (overhead > 0) entryFile.readFully(seekpos(this.handle), this.ohChunk, 0, overhead);
this.bodyChunk = null; /*new byte[ROW.objectsize];
entryFile.readFully(seekpos(this.handle) + overhead, this.bodyChunk, 0, this.bodyChunk.length);
*/
// mark chunks as not changed
this.ohChanged = false;
this.bodyChanged = false;
@ -222,11 +222,17 @@ public class kelondroFullRecords extends kelondroAbstractRecords {
public synchronized boolean valid() {
// returns true if the key starts with non-zero byte
// this may help to detect deleted entries
return (this.bodyChunk[0] != 0) && ((this.bodyChunk[0] != -128) || (this.bodyChunk[1] != 0));
return this.bodyChunk == null || (this.bodyChunk[0] != 0) && ((this.bodyChunk[0] != -128) || (this.bodyChunk[1] != 0));
}
public synchronized byte[] getKey() {
public synchronized byte[] getKey() throws IOException {
// read key
if (this.bodyChunk == null) {
// load all values from the database file
this.bodyChunk = new byte[ROW.objectsize];
// read values
entryFile.readFully(seekpos(this.handle) + overhead, this.bodyChunk, 0, this.bodyChunk.length);
}
return trimCopy(this.bodyChunk, 0, ROW.width(0));
}

@ -34,13 +34,12 @@ public interface kelondroIOChunks {
// pseudo-native methods:
public long length() throws IOException;
public int read(long pos, byte[] b, int off, int len) throws IOException;
public void readFully(long pos, byte[] b, int off, int len) throws IOException;
public void write(long pos, byte[] b, int off, int len) throws IOException;
public void commit() throws IOException;
public void close() throws IOException;
// derived methods:
public void readFully(long pos, byte[] b, int off, int len) throws IOException;
public byte readByte(long pos) throws IOException;
public void writeByte(long pos, int v) throws IOException;

@ -164,11 +164,10 @@ public class kelondroNIOFileRA extends kelondroAbstractRA implements kelondroRA
seekPos++;
}
public int read(final byte[] b, final int off, final int len) throws IOException {
for (int i = 0; i < len; i++) {
public void readFully(final byte[] b, final int off, final int len) throws IOException {
for (int i = 0; i < len; i++) {
b[off + i] = (byte) read();
}
return len;
}
public void write(final byte[] b, final int off, final int len) throws IOException {

@ -38,7 +38,7 @@ public interface kelondroNode {
public void commit() throws IOException;
public void setValueRow(byte[] row) throws IOException;
public boolean valid();
public byte[] getKey();
public byte[] getKey() throws IOException;
public byte[] getValueRow() throws IOException;
public String toString();

@ -46,16 +46,15 @@ public interface kelondroRA {
public long available() throws IOException;
public int read() throws IOException;
public void readFully(byte[] b, int off, int len) throws IOException;
public void write(int b) throws IOException;
public int read(byte[] b, int off, int len) throws IOException;
public void write(byte[] b, int off, int len) throws IOException;
public void seek(long pos) throws IOException;
public void close() throws IOException;
// derived methods:
public void readFully(byte[] b, int off, int len) throws IOException;
public byte[] readFully() throws IOException;
public byte readByte() throws IOException;
public void writeByte(int v) throws IOException;

@ -41,19 +41,6 @@ public final class kelondroRAIOChunks extends kelondroAbstractIOChunks implement
return ra.length();
}
public synchronized int read(final long pos, final byte[] b, final int off, final int len) throws IOException {
if (len == 0) return 0;
this.ra.seek(pos);
final long available = ra.available();
if (available >= len) {
return ra.read(b, off, len);
} else if (available == 0) {
return -1;
} else {
return ra.read(b, off, (int) available);
}
}
public synchronized void write(final long pos, final byte[] b, final int off, final int len) throws IOException {
this.ra.seek(pos);
this.ra.write(b, off, len);

@ -139,8 +139,24 @@ public final class serverDate {
* @param date The Date instance to transform.
* @return A fixed width (20 chars) ISO8601 date String.
*/
public static String formatISO8601(final Date date){
return format(FORMAT_ISO8601, date);
public static final String formatISO8601(final Date date) {
if (date == null) return "";
return format(FORMAT_ISO8601, date);
}
private static long lastRFC1123long = 0;
private static String lastRFC1123string = "";
public static final String formatRFC1123(final Date date) {
if (date == null) return "";
if (date.getTime() - lastRFC1123long < 1000) {
//System.out.println("date cache hit - " + lastRFC1123string);
return lastRFC1123string;
}
String s = format(FORMAT_RFC1123, date);
lastRFC1123long = date.getTime();
lastRFC1123string = s;
return s;
}
/**

@ -229,6 +229,8 @@ public class yacyURL implements Serializable {
escape();
}
private final Pattern backPathPattern = Pattern.compile("(/[^/]+(?<!/\\.{1,2})/)[.]{2}(?=/|$)|/\\.(?=/)|/(?=/)");
// resolve '..'
String resolveBackpath(String path) /* throws MalformedURLException */ {
/* original version by [MC]
@ -243,8 +245,7 @@ public class yacyURL implements Serializable {
/* by [MT] */
if (path.length() == 0 || path.charAt(0) != '/') { path = "/" + path; }
final Pattern pathPattern = Pattern.compile("(/[^/]+(?<!/\\.{1,2})/)[.]{2}(?=/|$)|/\\.(?=/)|/(?=/)");
final Matcher matcher = pathPattern.matcher(path);
final Matcher matcher = backPathPattern.matcher(path);
while (matcher.find()) {
path = matcher.replaceAll("");
matcher.reset(path);

Loading…
Cancel
Save