From bdae051d9a256de20c4d786bed61715f896c4465 Mon Sep 17 00:00:00 2001 From: orbiter Date: Wed, 13 Aug 2008 10:37:53 +0000 Subject: [PATCH] - extended new performance graph (better timing) - added paths for new libraries in classpath for eclipse - refactoring to remove compiler warnings (static access to finals variables) - removed some unused import git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5055 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- .classpath | 89 ++++++++++--------- .../htmlFilter/htmlFilterInputStream.java | 6 +- .../anomic/htmlFilter/htmlFilterWriter.java | 1 - source/de/anomic/http/httpd.java | 26 +++--- .../anomic/kelondro/kelondroIntBytesMap.java | 4 +- .../de/anomic/kelondro/kelondroSQLTable.java | 2 +- .../de/anomic/plasma/plasmaSwitchboard.java | 13 ++- source/de/anomic/plasma/plasmaWordIndex.java | 29 +++--- source/de/anomic/tools/cryptbig.java | 12 ++- 9 files changed, 99 insertions(+), 83 deletions(-) diff --git a/.classpath b/.classpath index 818d438c4..375acf68c 100644 --- a/.classpath +++ b/.classpath @@ -1,43 +1,46 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/source/de/anomic/htmlFilter/htmlFilterInputStream.java b/source/de/anomic/htmlFilter/htmlFilterInputStream.java index e832be6f4..41109bf51 100644 --- a/source/de/anomic/htmlFilter/htmlFilterInputStream.java +++ b/source/de/anomic/htmlFilter/htmlFilterInputStream.java @@ -63,8 +63,8 @@ public class htmlFilterInputStream extends InputStream implements htmlFilterEven final boolean passbyIfBinarySuspect ) throws UnsupportedEncodingException { // create a input stream for buffereing - this.bufferedIn = new BufferedInputStream(inStream,(int)this.preBufferSize); - this.bufferedIn.mark((int)this.preBufferSize); + this.bufferedIn = new BufferedInputStream(inStream,(int) preBufferSize); + this.bufferedIn.mark((int) preBufferSize); final htmlFilterContentScraper scraper = new htmlFilterContentScraper(rooturl); scraper.registerHtmlFilterEventListener(this); @@ -128,7 +128,7 @@ public class htmlFilterInputStream extends InputStream implements htmlFilterEven public int read() throws IOException { // mode 0 is called from within the detectCharset function if (this.mode == MODE_PRESCAN) { - if (this.endOfHead || this.charsetChanged || this.preRead >= this.preBufferSize-1) { + if (this.endOfHead || this.charsetChanged || this.preRead >= preBufferSize - 1) { return -1; } this.preRead++; diff --git a/source/de/anomic/htmlFilter/htmlFilterWriter.java b/source/de/anomic/htmlFilter/htmlFilterWriter.java index 049873aad..1d09356d9 100644 --- a/source/de/anomic/htmlFilter/htmlFilterWriter.java +++ b/source/de/anomic/htmlFilter/htmlFilterWriter.java @@ -38,7 +38,6 @@ import java.io.IOException; import java.io.OutputStream; import java.io.OutputStreamWriter; import java.io.Reader; -import java.io.UnsupportedEncodingException; import java.io.Writer; import java.net.MalformedURLException; import java.nio.charset.Charset; diff --git a/source/de/anomic/http/httpd.java b/source/de/anomic/http/httpd.java index e65532713..6e5a26258 100644 --- a/source/de/anomic/http/httpd.java +++ b/source/de/anomic/http/httpd.java @@ -198,7 +198,7 @@ public final class httpd implements serverHandler, Cloneable { // check if we want to allow this socket to connect us if (!(this.allowProxy || this.allowServer || this.allowYaCyHop)) { final String errorMsg = "CONNECTION FROM " + this.userAddress.getHostName() + " [" + this.clientIP + "] FORBIDDEN"; - this.log.logWarning(errorMsg); + log.logWarning(errorMsg); throw new IOException(errorMsg); } @@ -238,7 +238,7 @@ public final class httpd implements serverHandler, Cloneable { public String error(final Throwable e) { // OBLIGATORIC FUNCTION // return string in case of any error that occurs during communication // is always (but not only) called if an IO-dependent exception occurrs. - this.log.logSevere("Unexpected Error. " + e.getClass().getName(),e); + log.logSevere("Unexpected Error. " + e.getClass().getName(),e); final String message = e.getMessage(); if (message.indexOf("heap space") > 0) e.printStackTrace(); return "501 Exception occurred: " + message; @@ -509,22 +509,22 @@ public final class httpd implements serverHandler, Cloneable { private void logUnexpectedError(final Exception e) { if (e instanceof InterruptedException) { - this.log.logInfo("Interruption detected"); + log.logInfo("Interruption detected"); } else { final String errorMsg = e.getMessage(); if (errorMsg != null) { if (errorMsg.startsWith("Socket closed")) { - this.log.logInfo("httpd shutdown detected ..."); + log.logInfo("httpd shutdown detected ..."); } else if ((errorMsg.startsWith("Broken pipe") || errorMsg.startsWith("Connection reset"))) { // client closed the connection, so we just end silently - this.log.logInfo("Client unexpectedly closed connection"); + log.logInfo("Client unexpectedly closed connection"); } else if (errorMsg.equals("400 Bad request")) { - this.log.logInfo("Bad client request."); + log.logInfo("Bad client request."); } else { - this.log.logSevere("Unexpected Error. " + e.getClass().getName() + ": " + e.getMessage(),e); + log.logSevere("Unexpected Error. " + e.getClass().getName() + ": " + e.getMessage(),e); } } else { - this.log.logSevere("Unexpected Error. " + e.getClass().getName(),e); + log.logSevere("Unexpected Error. " + e.getClass().getName(),e); } } } @@ -594,7 +594,7 @@ public final class httpd implements serverHandler, Cloneable { final String transferEncoding = header.get(httpHeader.TRANSFER_ENCODING); if (transferEncoding != null) { if (!httpHeader.HTTP_VERSION_1_1.equals(httpVersion)) { - this.log.logWarning("client "+ session.getName() +" uses transfer-coding with HTTP version "+ httpVersion +"!"); + log.logWarning("client "+ session.getName() +" uses transfer-coding with HTTP version "+ httpVersion +"!"); } if("chunked".equalsIgnoreCase(header.get(httpHeader.TRANSFER_ENCODING))) { sessionIn = new ChunkedInputStream(this.session.in); @@ -1112,6 +1112,7 @@ public final class httpd implements serverHandler, Cloneable { * @see * org.apache.commons.fileupload.RequestContext#getCharacterEncoding() */ + //@Override public String getCharacterEncoding() { return header.getCharacterEncoding(); } @@ -1121,6 +1122,7 @@ public final class httpd implements serverHandler, Cloneable { * * @see org.apache.commons.fileupload.RequestContext#getContentLength() */ + //@Override public int getContentLength() { return (int) header.contentLength(); } @@ -1130,6 +1132,7 @@ public final class httpd implements serverHandler, Cloneable { * * @see org.apache.commons.fileupload.RequestContext#getContentType() */ + //@Override public String getContentType() { return header.get(httpHeader.CONTENT_TYPE); } @@ -1139,12 +1142,14 @@ public final class httpd implements serverHandler, Cloneable { * * @see org.apache.commons.fileupload.RequestContext#getInputStream() */ + //@Override public InputStream getInputStream() throws IOException { return inStream; } } + /* static int nextPos = -1; private static byte[] readLine(final int start, final byte[] array) { // read a string from an array; line ending is always CRLF @@ -1159,7 +1164,8 @@ public final class httpd implements serverHandler, Cloneable { java.lang.System.arraycopy(array, start, result, 0, pos - start); return result; } - + */ + public static int indexOf(final int start, final byte[] array, final byte[] pattern) { // return a position of a pattern in an array if (start > array.length - pattern.length) return -1; diff --git a/source/de/anomic/kelondro/kelondroIntBytesMap.java b/source/de/anomic/kelondro/kelondroIntBytesMap.java index a89132de8..6f9ca71c7 100644 --- a/source/de/anomic/kelondro/kelondroIntBytesMap.java +++ b/source/de/anomic/kelondro/kelondroIntBytesMap.java @@ -41,13 +41,13 @@ public class kelondroIntBytesMap { private final kelondroRow rowdef; private final kelondroRowSet index0; private kelondroRowSet index1; - private final kelondroOrder entryOrder; + //private final kelondroOrder entryOrder; public kelondroIntBytesMap(final int payloadSize, final int initSize) { this.rowdef = new kelondroRow("Cardinal key-4 {b256}, byte[] payload-" + payloadSize, kelondroNaturalOrder.naturalOrder, 0); this.index0 = new kelondroRowSet(rowdef, initSize); this.index1 = null; - this.entryOrder = new kelondroRow.EntryComparator(rowdef.objectOrder); + //this.entryOrder = new kelondroRow.EntryComparator(rowdef.objectOrder); } public long memoryNeededForGrow() { diff --git a/source/de/anomic/kelondro/kelondroSQLTable.java b/source/de/anomic/kelondro/kelondroSQLTable.java index b42ee4595..f922ff78f 100644 --- a/source/de/anomic/kelondro/kelondroSQLTable.java +++ b/source/de/anomic/kelondro/kelondroSQLTable.java @@ -86,7 +86,7 @@ public class kelondroSQLTable implements kelondroIndex { throw new Exception ("Unable to load the jdbc driver: " + e.getMessage(),e); } try { - this.theDBConnection = DriverManager.getConnection (dbConnStr,this.db_usr_str,this.db_pwd_str); + this.theDBConnection = DriverManager.getConnection (dbConnStr, db_usr_str, db_pwd_str); } catch (final Exception e) { throw new Exception ("Unable to establish a database connection: " + e.getMessage(),e); } diff --git a/source/de/anomic/plasma/plasmaSwitchboard.java b/source/de/anomic/plasma/plasmaSwitchboard.java index 452b12bef..5e1768d6a 100644 --- a/source/de/anomic/plasma/plasmaSwitchboard.java +++ b/source/de/anomic/plasma/plasmaSwitchboard.java @@ -722,10 +722,8 @@ public final class plasmaSwitchboard extends serverAbstractSwitch 30000) { // we don't want to do this too often updateMySeed(); - serverProfiling.update("ppm", Long.valueOf(currentPPM())); - serverProfiling.update("wordcache", Long.valueOf(webIndex.cacheSize())); + serverProfiling.update("ppm", new Long(currentPPM())); lastPPMUpdate = System.currentTimeMillis(); } serverProfiling.update("indexed", queueEntry.url().toNormalform(true, false)); @@ -1872,7 +1869,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch accessSet = tracker.get(host); if (accessSet == null) return 0; - return accessSet.tailSet(Long.valueOf(System.currentTimeMillis() - timeInterval)).size(); + return accessSet.tailSet(new Long(System.currentTimeMillis() - timeInterval)).size(); } public void startTransferWholeIndex(final yacySeed seed, final boolean delete) { @@ -2199,7 +2196,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch wCacheMaxChunk)) { - flushCache(theCache, Math.min(10, theCache.size())); - } - // next flush more entries if the size exceeds the maximum size of the cache - if ((theCache.size() > theCache.getMaxWordCount()) || - (serverMemory.available() < collections.minMem())) { - flushCache(theCache, Math.min(theCache.size() - theCache.getMaxWordCount() + 1, theCache.size())); + int cs = cacheSize(); + if (cs > 0) { + // flush elements that are too big. This flushing depends on the fact that the flush rule + // selects the biggest elements first for flushing. If it does not for any reason, the following + // loop would not terminate. + serverProfiling.update("wordcache", new Long(cs)); + // To ensure termination an additional counter is used + int l = 0; + while ((l++ < 100) && (theCache.maxURLinCache() > wCacheMaxChunk)) { + flushCache(theCache, Math.min(10, theCache.size())); + } + // next flush more entries if the size exceeds the maximum size of the cache + if ((theCache.size() > theCache.getMaxWordCount()) || + (serverMemory.available() < collections.minMem())) { + flushCache(theCache, Math.min(theCache.size() - theCache.getMaxWordCount() + 1, theCache.size())); + } + if (cacheSize() != cs) serverProfiling.update("wordcache", new Long(cacheSize())); } } diff --git a/source/de/anomic/tools/cryptbig.java b/source/de/anomic/tools/cryptbig.java index c1cfeabb0..46ea635b8 100644 --- a/source/de/anomic/tools/cryptbig.java +++ b/source/de/anomic/tools/cryptbig.java @@ -287,16 +287,20 @@ public class cryptbig { fin = new BufferedInputStream(new FileInputStream(inFileName), 4096); // read the file properties - final byte[] thisMagic = new byte[magicString.length()]; int bytesRead = fin.read(thisMagic); + final byte[] thisMagic = new byte[magicString.length()]; + fin.read(thisMagic); if (!((new String(thisMagic)).equals(magicString))) { // this is not an crypt file, so dont do anything fin.close(); return; } - final byte[] C = new byte[1]; bytesRead = fin.read(C); // the length of the following String, encoded as b64 - final byte[] B = new byte[(int) kelondroBase64Order.standardCoder.decodeLong(new String(C))]; bytesRead = fin.read(B); // this is again the length of the following string, as encrypted b64-ed integer - final byte[] A = new byte[(int) kelondroBase64Order.standardCoder.decodeLong(new String(dcipher.doFinal(B), "UTF8"))]; bytesRead = fin.read(A); + final byte[] C = new byte[1]; + fin.read(C); // the length of the following String, encoded as b64 + final byte[] B = new byte[(int) kelondroBase64Order.standardCoder.decodeLong(new String(C))]; + fin.read(B); // this is again the length of the following string, as encrypted b64-ed integer + final byte[] A = new byte[(int) kelondroBase64Order.standardCoder.decodeLong(new String(dcipher.doFinal(B), "UTF8"))]; + fin.read(A); final String X = new String(dcipher.doFinal(A), "UTF8"); System.out.println("TEST: detecting X-String : " + X);