boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

- extended new performance graph (better timing)

Browse Source 
- added paths for new libraries in classpath for eclipse
- refactoring to remove compiler warnings (static access to finals variables)
- removed some unused import

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5055 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 16 years ago
parent d9cea5ff23
commit bdae051d9a
9 changed files with 99 additions and 83 deletions
Show Stats Download Patch File Download Diff File

89
.classpath
Unescape View File

@ -1,43 +1,46 @@
<?xml version="1.0" encoding="UTF-8"?>
<classpath>
<classpathentry excluding="env/|htdocsdefault/|proxymsg/|yacy/|env/|yacy/user/|yacy/user/|yacy/ui/|processing/domaingraph/applet/|processing/domaingraph/" kind="src" path="htroot"/>
<classpathentry excluding="user/|user/|ui/" kind="src" path="htroot/yacy"/>
<classpathentry kind="src" path="htroot/env"/>
<classpathentry kind="src" path="source"/>
<classpathentry kind="src" path="htroot/yacy/user"/>
<classpathentry kind="src" path="htroot/yacy/ui"/>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
<classpathentry kind="lib" path="lib/tar.jar"/>
<classpathentry kind="lib" path="libx/bzip2.jar"/>
<classpathentry kind="lib" path="libx/commons-discovery.jar"/>
<classpathentry kind="lib" path="libx/commons-jxpath-1.1.jar"/>
<classpathentry kind="lib" path="libx/jakarta-oro-2.0.7.jar"/>
<classpathentry kind="lib" path="libx/jaxrpc.jar"/>
<classpathentry kind="lib" path="libx/jmimemagic-0.1.0.jar"/>
<classpathentry kind="lib" path="libx/jrpm-head.jar"/>
<classpathentry kind="lib" path="libx/jrpm-SNAPSHOT.jar"/>
<classpathentry kind="lib" path="libx/jsch-0.1.21.jar"/>
<classpathentry kind="lib" path="libx/log4j-1.2.9.jar"/>
<classpathentry kind="lib" path="libx/odf_utils_05_11_29.jar"/>
<classpathentry kind="lib" path="libx/saaj.jar"/>
<classpathentry kind="lib" path="libx/sbbi-upnplib-1.0.3.jar"/>
<classpathentry kind="lib" path="libx/tm-extractors-0.4.jar"/>
<classpathentry kind="lib" path="libx/wsdl4j.jar"/>
<classpathentry kind="lib" path="libx/xerces.jar"/>
<classpathentry kind="lib" path="libx/poi-3.0-alpha2-20060616.jar"/>
<classpathentry kind="lib" path="libx/poi-scratchpad-3.0-alpha2-20060616.jar"/>
<classpathentry kind="lib" path="libx/inetlib.jar"/>
<classpathentry kind="lib" path="libx/gnumail.jar"/>
<classpathentry kind="lib" path="libx/activation.jar"/>
<classpathentry kind="lib" path="libx/webcat-0.1-swf.jar"/>
<classpathentry kind="lib" path="libx/J7Zip-modified.jar"/>
<classpathentry kind="lib" path="libx/PDFBox-0.7.3.jar"/>
<classpathentry kind="lib" path="libx/FontBox-0.1.0-dev.jar"/>
<classpathentry kind="lib" path="libx/bcprov-jdk14-139.jar"/>
<classpathentry kind="lib" path="libx/bcmail-jdk14-139.jar"/>
<classpathentry kind="lib" path="libx/commons-codec-1.3.jar"/>
<classpathentry kind="lib" path="lib/commons-httpclient-3.1.jar"/>
<classpathentry kind="lib" path="lib/commons-logging-1.1.1.jar"/>
<classpathentry kind="lib" path="lib/jdic.jar"/>
<classpathentry kind="output" path="gen"/>
</classpath>
<?xml version="1.0" encoding="UTF-8"?>
<classpath>
<classpathentry excluding="env/|htdocsdefault/|proxymsg/|yacy/|env/|yacy/user/|yacy/user/|yacy/ui/|processing/domaingraph/applet/|processing/domaingraph/" kind="src" path="htroot"/>
<classpathentry excluding="user/|user/|ui/" kind="src" path="htroot/yacy"/>
<classpathentry kind="src" path="htroot/env"/>
<classpathentry kind="src" path="source"/>
<classpathentry kind="src" path="htroot/yacy/user"/>
<classpathentry kind="src" path="htroot/yacy/ui"/>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
<classpathentry kind="lib" path="lib/tar.jar"/>
<classpathentry kind="lib" path="libx/bzip2.jar"/>
<classpathentry kind="lib" path="libx/commons-discovery.jar"/>
<classpathentry kind="lib" path="libx/commons-jxpath-1.1.jar"/>
<classpathentry kind="lib" path="libx/jakarta-oro-2.0.7.jar"/>
<classpathentry kind="lib" path="libx/jaxrpc.jar"/>
<classpathentry kind="lib" path="libx/jmimemagic-0.1.0.jar"/>
<classpathentry kind="lib" path="libx/jrpm-head.jar"/>
<classpathentry kind="lib" path="libx/jrpm-SNAPSHOT.jar"/>
<classpathentry kind="lib" path="libx/jsch-0.1.21.jar"/>
<classpathentry kind="lib" path="libx/log4j-1.2.9.jar"/>
<classpathentry kind="lib" path="libx/odf_utils_05_11_29.jar"/>
<classpathentry kind="lib" path="libx/saaj.jar"/>
<classpathentry kind="lib" path="libx/sbbi-upnplib-1.0.3.jar"/>
<classpathentry kind="lib" path="libx/tm-extractors-0.4.jar"/>
<classpathentry kind="lib" path="libx/wsdl4j.jar"/>
<classpathentry kind="lib" path="libx/xerces.jar"/>
<classpathentry kind="lib" path="libx/poi-3.0-alpha2-20060616.jar"/>
<classpathentry kind="lib" path="libx/poi-scratchpad-3.0-alpha2-20060616.jar"/>
<classpathentry kind="lib" path="libx/inetlib.jar"/>
<classpathentry kind="lib" path="libx/gnumail.jar"/>
<classpathentry kind="lib" path="libx/activation.jar"/>
<classpathentry kind="lib" path="libx/webcat-0.1-swf.jar"/>
<classpathentry kind="lib" path="libx/J7Zip-modified.jar"/>
<classpathentry kind="lib" path="libx/PDFBox-0.7.3.jar"/>
<classpathentry kind="lib" path="libx/FontBox-0.1.0-dev.jar"/>
<classpathentry kind="lib" path="libx/bcprov-jdk14-139.jar"/>
<classpathentry kind="lib" path="libx/bcmail-jdk14-139.jar"/>
<classpathentry kind="lib" path="libx/commons-codec-1.3.jar"/>
<classpathentry kind="lib" path="lib/commons-httpclient-3.1.jar"/>
<classpathentry kind="lib" path="lib/commons-logging-1.1.1.jar"/>
<classpathentry kind="lib" path="lib/jdic.jar"/>
<classpathentry kind="lib" path="lib/commons-io-1.4.jar"/>
<classpathentry kind="lib" path="lib/commons-fileupload-1.2.1.jar"/>
<classpathentry kind="lib" path="lib/servlet-api.jar"/>
<classpathentry kind="output" path="gen"/>
</classpath>

6
source/de/anomic/htmlFilter/htmlFilterInputStream.java
Unescape View File

@ -63,8 +63,8 @@ public class htmlFilterInputStream extends InputStream implements htmlFilterEven
final boolean passbyIfBinarySuspect
) throws UnsupportedEncodingException {
// create a input stream for buffereing
this.bufferedIn = new BufferedInputStream(inStream,(int)this.preBufferSize);
this.bufferedIn.mark((int)this.preBufferSize);
this.bufferedIn = new BufferedInputStream(inStream,(int) preBufferSize);
this.bufferedIn.mark((int) preBufferSize);
final htmlFilterContentScraper scraper = new htmlFilterContentScraper(rooturl);
scraper.registerHtmlFilterEventListener(this);
@ -128,7 +128,7 @@ public class htmlFilterInputStream extends InputStream implements htmlFilterEven
public int read() throws IOException {
// mode 0 is called from within the detectCharset function
if (this.mode == MODE_PRESCAN) {
if (this.endOfHead || this.charsetChanged || this.preRead >= this.preBufferSize-1) {
if (this.endOfHead || this.charsetChanged || this.preRead >= preBufferSize - 1) {
return -1;
}
this.preRead++;

1
source/de/anomic/htmlFilter/htmlFilterWriter.java
Unescape View File

@ -38,7 +38,6 @@ import java.io.IOException;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.Reader;
import java.io.UnsupportedEncodingException;
import java.io.Writer;
import java.net.MalformedURLException;
import java.nio.charset.Charset;

26
source/de/anomic/http/httpd.java
Unescape View File

@ -198,7 +198,7 @@ public final class httpd implements serverHandler, Cloneable {
// check if we want to allow this socket to connect us
if (!(this.allowProxy || this.allowServer || this.allowYaCyHop)) {
final String errorMsg = "CONNECTION FROM " + this.userAddress.getHostName() + " [" + this.clientIP + "] FORBIDDEN";
this.log.logWarning(errorMsg);
log.logWarning(errorMsg);
throw new IOException(errorMsg);
}
@ -238,7 +238,7 @@ public final class httpd implements serverHandler, Cloneable {
public String error(final Throwable e) { // OBLIGATORIC FUNCTION
// return string in case of any error that occurs during communication
// is always (but not only) called if an IO-dependent exception occurrs.
this.log.logSevere("Unexpected Error. " + e.getClass().getName(),e);
log.logSevere("Unexpected Error. " + e.getClass().getName(),e);
final String message = e.getMessage();
if (message.indexOf("heap space") > 0) e.printStackTrace();
return "501 Exception occurred: " + message;
@ -509,22 +509,22 @@ public final class httpd implements serverHandler, Cloneable {
private void logUnexpectedError(final Exception e) {
if (e instanceof InterruptedException) {
this.log.logInfo("Interruption detected");
log.logInfo("Interruption detected");
} else {
final String errorMsg = e.getMessage();
if (errorMsg != null) {
if (errorMsg.startsWith("Socket closed")) {
this.log.logInfo("httpd shutdown detected ...");
log.logInfo("httpd shutdown detected ...");
} else if ((errorMsg.startsWith("Broken pipe") || errorMsg.startsWith("Connection reset"))) {
// client closed the connection, so we just end silently
this.log.logInfo("Client unexpectedly closed connection");
log.logInfo("Client unexpectedly closed connection");
} else if (errorMsg.equals("400 Bad request")) {
this.log.logInfo("Bad client request.");
log.logInfo("Bad client request.");
} else {
this.log.logSevere("Unexpected Error. " + e.getClass().getName() + ": " + e.getMessage(),e);
log.logSevere("Unexpected Error. " + e.getClass().getName() + ": " + e.getMessage(),e);
}
} else {
this.log.logSevere("Unexpected Error. " + e.getClass().getName(),e);
log.logSevere("Unexpected Error. " + e.getClass().getName(),e);
}
}
}
@ -594,7 +594,7 @@ public final class httpd implements serverHandler, Cloneable {
final String transferEncoding = header.get(httpHeader.TRANSFER_ENCODING);
if (transferEncoding != null) {
if (!httpHeader.HTTP_VERSION_1_1.equals(httpVersion)) {
this.log.logWarning("client "+ session.getName() +" uses transfer-coding with HTTP version "+ httpVersion +"!");
log.logWarning("client "+ session.getName() +" uses transfer-coding with HTTP version "+ httpVersion +"!");
}
if("chunked".equalsIgnoreCase(header.get(httpHeader.TRANSFER_ENCODING))) {
sessionIn = new ChunkedInputStream(this.session.in);
@ -1112,6 +1112,7 @@ public final class httpd implements serverHandler, Cloneable {
* @see
* org.apache.commons.fileupload.RequestContext#getCharacterEncoding()
*/
//@Override
public String getCharacterEncoding() {
return header.getCharacterEncoding();
}
@ -1121,6 +1122,7 @@ public final class httpd implements serverHandler, Cloneable {
*
* @see org.apache.commons.fileupload.RequestContext#getContentLength()
*/
//@Override
public int getContentLength() {
return (int) header.contentLength();
}
@ -1130,6 +1132,7 @@ public final class httpd implements serverHandler, Cloneable {
*
* @see org.apache.commons.fileupload.RequestContext#getContentType()
*/
//@Override
public String getContentType() {
return header.get(httpHeader.CONTENT_TYPE);
}
@ -1139,12 +1142,14 @@ public final class httpd implements serverHandler, Cloneable {
*
* @see org.apache.commons.fileupload.RequestContext#getInputStream()
*/
//@Override
public InputStream getInputStream() throws IOException {
return inStream;
}
}
/*
static int nextPos = -1;
private static byte[] readLine(final int start, final byte[] array) {
// read a string from an array; line ending is always CRLF
@ -1159,7 +1164,8 @@ public final class httpd implements serverHandler, Cloneable {
java.lang.System.arraycopy(array, start, result, 0, pos - start);
return result;
}
*/
public static int indexOf(final int start, final byte[] array, final byte[] pattern) {
// return a position of a pattern in an array
if (start > array.length - pattern.length) return -1;

4
source/de/anomic/kelondro/kelondroIntBytesMap.java
Unescape View File

@ -41,13 +41,13 @@ public class kelondroIntBytesMap {
private final kelondroRow rowdef;
private final kelondroRowSet index0;
private kelondroRowSet index1;
private final kelondroOrder<kelondroRow.Entry> entryOrder;
//private final kelondroOrder<kelondroRow.Entry> entryOrder;
public kelondroIntBytesMap(final int payloadSize, final int initSize) {
this.rowdef = new kelondroRow("Cardinal key-4 {b256}, byte[] payload-" + payloadSize, kelondroNaturalOrder.naturalOrder, 0);
this.index0 = new kelondroRowSet(rowdef, initSize);
this.index1 = null;
this.entryOrder = new kelondroRow.EntryComparator(rowdef.objectOrder);
//this.entryOrder = new kelondroRow.EntryComparator(rowdef.objectOrder);
}
public long memoryNeededForGrow() {

2
source/de/anomic/kelondro/kelondroSQLTable.java
Unescape View File

@ -86,7 +86,7 @@ public class kelondroSQLTable implements kelondroIndex {
throw new Exception ("Unable to load the jdbc driver: " + e.getMessage(),e);
}
try {
this.theDBConnection = DriverManager.getConnection (dbConnStr,this.db_usr_str,this.db_pwd_str);
this.theDBConnection = DriverManager.getConnection (dbConnStr, db_usr_str, db_pwd_str);
} catch (final Exception e) {
throw new Exception ("Unable to establish a database connection: " + e.getMessage(),e);
}

13
source/de/anomic/plasma/plasmaSwitchboard.java
Unescape View File

@ -722,10 +722,8 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<IndexingStack.
// clean search events which have cached relations to the old index
plasmaSearchEvent.cleanupEvents(true);
// switch the networks
synchronized (this.webIndex) {
this.webIndex.close();
}
synchronized (this) {
synchronized (this.webIndex) {
this.webIndex.close();
setConfig("network.unit.definition", networkDefinition);
overwriteNetworkDefinition(this);
final File indexPrimaryPath = getConfigPath(plasmaSwitchboardConstants.INDEX_PRIMARY_PATH, plasmaSwitchboardConstants.INDEX_PATH_DEFAULT);
@ -1686,8 +1684,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<IndexingStack.
if (System.currentTimeMillis() - lastPPMUpdate > 30000) {
// we don't want to do this too often
updateMySeed();
serverProfiling.update("ppm", Long.valueOf(currentPPM()));
serverProfiling.update("wordcache", Long.valueOf(webIndex.cacheSize()));
serverProfiling.update("ppm", new Long(currentPPM()));
lastPPMUpdate = System.currentTimeMillis();
}
serverProfiling.update("indexed", queueEntry.url().toNormalform(true, false));
@ -1872,7 +1869,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<IndexingStack.
final long timeInterval = 1000 * 60 * 60;
final TreeSet<Long> accessSet = tracker.get(host);
if (accessSet == null) return 0;
return accessSet.tailSet(Long.valueOf(System.currentTimeMillis() - timeInterval)).size();
return accessSet.tailSet(new Long(System.currentTimeMillis() - timeInterval)).size();
}
public void startTransferWholeIndex(final yacySeed seed, final boolean delete) {
@ -2199,7 +2196,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<IndexingStack.
while (enu.hasNext()) {
ys = yacySeed.genRemoteSeed(enu.next(), null, false);
if ((ys != null) &&
((!webIndex.seedDB.mySeedIsDefined()) || (!webIndex.seedDB.mySeed().hash.equals(ys.hash)))) {
((!webIndex.seedDB.mySeedIsDefined()) || (webIndex.seedDB.mySeed().hash != ys.hash))) {
if (webIndex.peerActions.connectPeer(ys, false)) lc++;
//seedDB.writeMap(ys.hash, ys.getMap(), "init");
//System.out.println("BOOTSTRAP: received peer " + ys.get(yacySeed.NAME, "anonymous") + "/" + ys.getAddress());

29
source/de/anomic/plasma/plasmaWordIndex.java
Unescape View File

@ -61,6 +61,7 @@ import de.anomic.kelondro.kelondroOrder;
import de.anomic.kelondro.kelondroRotateIterator;
import de.anomic.kelondro.kelondroRowCollection;
import de.anomic.server.serverMemory;
import de.anomic.server.serverProfiling;
import de.anomic.server.logging.serverLog;
import de.anomic.xml.RSSFeed;
import de.anomic.xml.RSSMessage;
@ -417,17 +418,23 @@ public final class plasmaWordIndex implements indexRI {
public void dhtFlushControl(final indexRAMRI theCache) {
// check for forced flush
int l = 0;
// flush elements that are too big. This flushing depends on the fact that the flush rule
// selects the biggest elements first for flushing. If it does not for any reason, the following
// loop would not terminate. To ensure termination an additional counter is used
while ((l++ < 100) && (theCache.maxURLinCache() > wCacheMaxChunk)) {
flushCache(theCache, Math.min(10, theCache.size()));
}
// next flush more entries if the size exceeds the maximum size of the cache
if ((theCache.size() > theCache.getMaxWordCount()) ||
(serverMemory.available() < collections.minMem())) {
flushCache(theCache, Math.min(theCache.size() - theCache.getMaxWordCount() + 1, theCache.size()));
int cs = cacheSize();
if (cs > 0) {
// flush elements that are too big. This flushing depends on the fact that the flush rule
// selects the biggest elements first for flushing. If it does not for any reason, the following
// loop would not terminate.
serverProfiling.update("wordcache", new Long(cs));
// To ensure termination an additional counter is used
int l = 0;
while ((l++ < 100) && (theCache.maxURLinCache() > wCacheMaxChunk)) {
flushCache(theCache, Math.min(10, theCache.size()));
}
// next flush more entries if the size exceeds the maximum size of the cache
if ((theCache.size() > theCache.getMaxWordCount()) ||
(serverMemory.available() < collections.minMem())) {
flushCache(theCache, Math.min(theCache.size() - theCache.getMaxWordCount() + 1, theCache.size()));
}
if (cacheSize() != cs) serverProfiling.update("wordcache", new Long(cacheSize()));
}
}

12
source/de/anomic/tools/cryptbig.java
Unescape View File

@ -287,16 +287,20 @@ public class cryptbig {
fin = new BufferedInputStream(new FileInputStream(inFileName), 4096);
// read the file properties
final byte[] thisMagic = new byte[magicString.length()]; int bytesRead = fin.read(thisMagic);
final byte[] thisMagic = new byte[magicString.length()];
fin.read(thisMagic);
if (!((new String(thisMagic)).equals(magicString))) {
// this is not an crypt file, so dont do anything
fin.close();
return;
}
final byte[] C = new byte[1]; bytesRead = fin.read(C); // the length of the following String, encoded as b64
final byte[] B = new byte[(int) kelondroBase64Order.standardCoder.decodeLong(new String(C))]; bytesRead = fin.read(B); // this is again the length of the following string, as encrypted b64-ed integer
final byte[] A = new byte[(int) kelondroBase64Order.standardCoder.decodeLong(new String(dcipher.doFinal(B), "UTF8"))]; bytesRead = fin.read(A);
final byte[] C = new byte[1];
fin.read(C); // the length of the following String, encoded as b64
final byte[] B = new byte[(int) kelondroBase64Order.standardCoder.decodeLong(new String(C))];
fin.read(B); // this is again the length of the following string, as encrypted b64-ed integer
final byte[] A = new byte[(int) kelondroBase64Order.standardCoder.decodeLong(new String(dcipher.doFinal(B), "UTF8"))];
fin.read(A);
final String X = new String(dcipher.doFinal(A), "UTF8");
System.out.println("TEST: detecting X-String : " + X);

Write Preview
Loading…
Cancel
Save