boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

- fixed a bug in remote search that prevented that any results had been generated (!)

Browse Source 
- added a great number of printStackTrace and new exceptions that shall be used to find the cause
  for a bug in yacy client-server communication which causes the interruption of data transfer
  which then causes the parser bug for the seed strings.
- tried to fix the communication bug on server-side (copy functions)
Be aware that the log may be full of errors and bugs - there should not be more bugs but there is more to see


git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4519 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 17 years ago
parent 0ddbed9451
commit 4fdf695064
10 changed files with 85 additions and 46 deletions
Show Stats Download Patch File Download Diff File

38
source/de/anomic/http/httpc.java
Unescape View File

@ -400,6 +400,7 @@ public final class httpc {
this.socket.setSoTimeout(timeout); // waiting time for read
// get the connection
this.socket.connect(address, timeout);
this.socket.setSoTimeout(timeout); // waiting time for read
if (incomingByteCountAccounting != null) {
this.clientInputByteCount = new httpdByteCountInputStream(this.socket.getInputStream(),incomingByteCountAccounting);
@ -411,7 +412,7 @@ public final class httpc {
// getting input and output streams
this.clientInput = new PushbackInputStream((this.clientInputByteCount!=null)?
this.clientInputByteCount:
this.socket.getInputStream());
this.socket.getInputStream());
this.clientOutput = this.socket.getOutputStream();
// if we reached this point, we should have a connection
@ -422,7 +423,7 @@ public final class httpc {
} catch (IOException e) {
// There was an error while connecting the socket, probably a SocketTimeoutException
// we have to close the httpc, otherwise it would stay in activeConnections forever
serverLog.logFine("HTTPC", "Couldn't open socket to: " + this.adressed_host + ":" + this.adressed_port);
serverLog.logFine("HTTPC", "Couldn't open socket to " + this.adressed_host + ":" + this.adressed_port + ": " + e.getMessage());
close();
// TODO do we need to hand it over to the caller?
@ -979,6 +980,8 @@ public final class httpc {
// read connection body and return body
serverByteBuffer sbb = new serverByteBuffer();
res.writeContent(sbb, null);
if ((res.responseHeader.contentLength() > 0) && (res.responseHeader.contentLength() != sbb.length()))
throw new IOException("content length and loaded resource length from http://" + realhost + ":" + port + path + " does not match. HEADER.Content-Length = " + res.responseHeader.contentLength() + ", resource.length =" + sbb.length());
con.close();
return sbb.getBytes();
}
@ -1244,7 +1247,7 @@ public final class httpc {
}
// reads in the http header, right now, right here
byte[] b = serverCore.receive(httpc.this.clientInput, terminalMaxLength, false);
byte[] b = serverCore.receive(httpc.this.clientInput, terminalMaxLength, true);
if (b == null) {
// the server has meanwhile disconnected
this.statusCode = 503;
@ -1263,7 +1266,7 @@ public final class httpc {
if ((this.statusCode==500)&&(this.statusText.equals("status line parse error"))) {
// flush in anything that comes without parsing
while ((b != null) && (b.length != 0)) b = serverCore.receive(httpc.this.clientInput, terminalMaxLength, false);
while ((b != null) && (b.length != 0)) b = serverCore.receive(httpc.this.clientInput, terminalMaxLength, true);
return; // in bad mood
}
@ -1271,13 +1274,13 @@ public final class httpc {
if (this.statusCode == 400) {
// bad request
// flush in anything that comes without parsing
while ((b = serverCore.receive(httpc.this.clientInput, terminalMaxLength, false)).length != 0) {}
while ((b = serverCore.receive(httpc.this.clientInput, terminalMaxLength, true)).length != 0) {}
return; // in bad mood
}
// at this point we should have a valid response. read in the header properties
String key = "";
while ((b = serverCore.receive(httpc.this.clientInput, terminalMaxLength, false)) != null) {
while ((b = serverCore.receive(httpc.this.clientInput, terminalMaxLength, true)) != null) {
if (b.length == 0) break;
buffer = new String(b);
buffer=buffer.trim();
@ -1362,7 +1365,7 @@ public final class httpc {
return new GZIPInputStream(httpc.this.clientInput);
} else if (this.responseHeader.contentLength() != -1) {
// use a httpContentLengthInputStream to read until the end of the response body is reached
return new httpContentLengthInputStream(httpc.this.clientInput,this.responseHeader.contentLength());
return new httpContentLengthInputStream(httpc.this.clientInput, this.responseHeader.contentLength());
}
// no Content-Lenght was set. In this case we can read until EOF
return httpc.this.clientInput;
@ -1429,13 +1432,17 @@ public final class httpc {
} else {
throw new IllegalArgumentException("Invalid procOS object type '" + procOS.getClass().getName() + "'");
}
} catch (IOException e) {}
} catch (IOException e) {
e.printStackTrace();
}
if (bufferOS != null) {
try {
bufferOS.flush();
bufferOS.close();
} catch (IOException e) {}
} catch (IOException e) {
e.printStackTrace();
}
if (file.length() == 0) file.delete();
}
}
@ -1453,13 +1460,16 @@ public final class httpc {
if (System.currentTimeMillis() - lastIO > 30000) break;
this.wait(300);
continue io;
} catch (InterruptedException e) {} // may happen without EOF
} catch (InterruptedException e) {
e.printStackTrace();
} // may happen without EOF
lastIO = System.currentTimeMillis();
c += l;
if (procOS != null) procOS.write(buffer, 0, l);
if (bufferOS != null) bufferOS.write(buffer, 0, l);
} catch (IOException e) {
//System.out.println("*** DEBUG: writeX/IOStream terminated with IOException, processed " + c + " bytes.");
System.out.println("*** DEBUG: writeX/IOStream terminated with IOException, processed " + c + " bytes. cause: " + e.getMessage());
e.printStackTrace();
break;
}
@ -1484,13 +1494,15 @@ public final class httpc {
if (System.currentTimeMillis() - lastIO > 30000) break;
this.wait(300);
continue io;
} catch (InterruptedException e) {} // may happen without EOF
} catch (InterruptedException e) {
e.printStackTrace();
} // may happen without EOF
lastIO = System.currentTimeMillis();
c += l;
if (procOS != null) procOS.write(buffer, 0, l);
if (bufferOSWriter != null) bufferOSWriter.write(buffer, 0, l);
} catch (IOException e) {
//System.out.println("*** DEBUG: writeX/ReaderWriter terminated with IOException, processed " + c + " bytes.");
System.out.println("*** DEBUG: writeX/ReaderWriter terminated with IOException, processed " + c + " bytes. cause: " + e.getMessage());
break;
}

5
source/de/anomic/http/httpdByteCountInputStream.java
Unescape View File

@ -91,9 +91,14 @@ public class httpdByteCountInputStream extends FilterInputStream {
}
public int read(byte[] b, int off, int len) throws IOException {
try {
int readCount = super.read(b, off, len);
if (readCount > 0) this.byteCount += readCount;
return readCount;
} catch (IOException e) {
e.printStackTrace();
throw new IOException(e.getMessage() + "; b.length = " + b.length + ", off = " + off + ", len = " + len);
}
}
public int read() throws IOException {

2
source/de/anomic/kelondro/kelondroSortStore.java
Unescape View File

@ -100,7 +100,7 @@ public class kelondroSortStore<E> extends kelondroSortStack<E> {
return this.offstack;
}
if (size() < count) throw new RuntimeException("list(" + count + ") exceeded avaiable number of elements (" + size() + ")");
while (this.onstack.size() < count) {
while (this.offstack.size() < count) {
Long w = this.onstack.firstKey();
E element = this.onstack.remove(w);
stackElement se = new stackElement(element, w);

2
source/de/anomic/plasma/crawler/plasmaCrawlQueues.java
Unescape View File

@ -178,8 +178,6 @@ public class plasmaCrawlQueues {
log.logFine("CoreCrawl: online caution, omitting processing");
return false;
}
// if the server is busy, we do crawling more slowly
//if (!(cacheManager.idle())) try {Thread.currentThread().sleep(2000);} catch (InterruptedException e) {}
// if crawling was paused we have to wait until we wer notified to continue
Object[] status = (Object[]) sb.crawlJobsStatus.get(plasmaSwitchboard.CRAWLJOB_LOCAL_CRAWL);

2
source/de/anomic/server/serverCore.java
Unescape View File

@ -601,7 +601,7 @@ public final class serverCore extends serverAbstractThread implements serverThre
}
public byte[] readLine() {
return receive(this.in, serverCore.this.commandMaxLength, false);
return receive(this.in, serverCore.this.commandMaxLength, true);
}
/**

23
source/de/anomic/server/serverFileUtils.java
Unescape View File

@ -72,7 +72,7 @@ import de.anomic.tools.nxTools;
public final class serverFileUtils {
private static final int DEFAULT_BUFFER_SIZE = 4096;
private static final int DEFAULT_BUFFER_SIZE = 512;
public static long copy(InputStream source, OutputStream dest) throws IOException {
return copy(source,dest, -1);
@ -95,13 +95,13 @@ public final class serverFileUtils {
int chunkSize = (int) ((count > 0) ? Math.min(count, DEFAULT_BUFFER_SIZE) : DEFAULT_BUFFER_SIZE);
int c; long total = 0;
while ((c = source.read(buffer,0,chunkSize)) > 0) {
while ((c = source.read(buffer, 0, chunkSize)) > 0) {
dest.write(buffer, 0, c);
dest.flush();
total += c;
if (count > 0) {
chunkSize = (int)Math.min(count-total,DEFAULT_BUFFER_SIZE);
chunkSize = (int) Math.min(count-total, DEFAULT_BUFFER_SIZE);
if (chunkSize == 0) break;
}
@ -111,7 +111,7 @@ public final class serverFileUtils {
return total;
}
public static int copy (File source, String inputCharset, Writer dest) throws IOException {
public static int copy(File source, String inputCharset, Writer dest) throws IOException {
InputStream fis = null;
try {
fis = new FileInputStream(source);
@ -121,18 +121,18 @@ public final class serverFileUtils {
}
}
public static int copy (InputStream source, Writer dest, String inputCharset) throws IOException {
public static int copy(InputStream source, Writer dest, String inputCharset) throws IOException {
InputStreamReader reader = new InputStreamReader(source,inputCharset);
return copy(reader,dest);
}
public static int copy (String source, Writer dest) throws IOException {
public static int copy(String source, Writer dest) throws IOException {
dest.write(source);
dest.flush();
return source.length();
}
public static int copy (Reader source, Writer dest) throws IOException {
public static int copy(Reader source, Writer dest) throws IOException {
char[] buffer = new char[4096];
int count = 0;
int n = 0;
@ -188,7 +188,7 @@ public final class serverFileUtils {
fis = new FileInputStream(source);
long skipped = fis.skip(start);
if (skipped != start) throw new IllegalStateException("Unable to skip '" + start + "' bytes. Only '" + skipped + "' bytes skipped.");
copy(fis, dest,-1);
copy(fis, dest, -1);
} finally {
if (fis != null) try { fis.close(); } catch (Exception e) {}
}
@ -305,11 +305,13 @@ public final class serverFileUtils {
}
public static void write(String source, Writer dest) throws IOException {
copy(source,dest);
dest.write(source);
dest.flush();
}
public static void write(byte[] source, OutputStream dest) throws IOException {
copy(new ByteArrayInputStream(source), dest, -1);
dest.write(source, 0, source.length);
dest.flush();
}
public static void write(byte[] source, File dest) throws IOException {
@ -327,6 +329,7 @@ public final class serverFileUtils {
// support of gzipped data (requested by roland)
if ((source.length > 1) && (((source[1] << 8) | source[0]) == GZIPInputStream.GZIP_MAGIC)) {
System.out.println("DEBUG: uncompressGZipArray - uncompressing source");
try {
ByteArrayInputStream byteInput = new ByteArrayInputStream(source);
ByteArrayOutputStream byteOutput = new ByteArrayOutputStream();

32
source/de/anomic/tools/crypt.java
Unescape View File

@ -119,14 +119,30 @@ public class crypt {
}
public static String simpleDecode(String encoded, String key) {
if (encoded == null || encoded.length() < 3) { return null; }
if (encoded.charAt(1) != '|') { return encoded; } // not encoded
switch (encoded.charAt(0)) {
case 'b' : return kelondroBase64Order.enhancedCoder.decodeString(encoded.substring(2), "de.anomic.tools.crypt.simpleDecode()");
case 'z' : return gzip.gunzipString(kelondroBase64Order.enhancedCoder.decode(encoded.substring(2), "de.anomic.tools.crypt.simpleDecode()"));
case 'p' : return encoded.substring(2);
default : return null;
}
if (encoded == null || encoded.length() < 3) {
return null;
}
if (encoded.charAt(1) != '|') {
return encoded;
} // not encoded
switch (encoded.charAt(0)) {
case 'b': {
return kelondroBase64Order.enhancedCoder.decodeString(encoded.substring(2), "de.anomic.tools.crypt.simpleDecode()");
}
case 'z':
try {
return gzip.gunzipString(kelondroBase64Order.enhancedCoder.decode(encoded.substring(2), "de.anomic.tools.crypt.simpleDecode()"));
} catch (Exception e) {
e.printStackTrace();
return null;
}
case 'p': {
return encoded.substring(2);
}
default: {
return null;
}
}
}
public static void main(String[] args) {

11
source/de/anomic/tools/cryptbig.java
Unescape View File

@ -395,7 +395,7 @@ public class cryptbig {
return salt + ((gzFlag) ? "1" : "0") + kelondroBase64Order.enhancedCoder.encode(enc);
}
public static String descrambleString(String key, String s) {
public static String descrambleString(String key, String s) throws IOException {
String salt = s.substring(0, 8);
boolean gzFlag = (s.charAt(8) == '1');
s = s.substring(9);
@ -434,7 +434,7 @@ public class cryptbig {
return null;
}
public static String simpleDecode(String encoded, String key) {
public static String simpleDecode(String encoded, String key) throws IOException {
if ((encoded == null) || (encoded.length() < 3)) return null;
if (encoded.charAt(1) != '|') return encoded; // not encoded
char method = encoded.charAt(0);
@ -620,7 +620,12 @@ public class cryptbig {
// 'descramble' string
if (s.length != 3) {help(); System.exit(-1);}
long t = System.currentTimeMillis();
System.out.println(descrambleString(s[1], s[2]));
try {
System.out.println(descrambleString(s[1], s[2]));
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
System.out.println("Calculation time: " + (System.currentTimeMillis() - t) + " milliseconds");
System.exit(0);
}

8
source/de/anomic/tools/gzip.java
Unescape View File

@ -107,19 +107,13 @@ public class gzip {
}
}
public static String gunzipString(byte[] in) {
try {
public static String gunzipString(byte[] in) throws IOException {
InputStream fin = new GZIPInputStream(new ByteArrayInputStream(in));
ByteArrayOutputStream fout = new ByteArrayOutputStream();
copy(fout, fin, 128);
fin.close();
fout.close();
return new String(fout.toByteArray(), "UTF-8");
} catch (IOException e) {
//System.err.println("ERROR: IO trouble ");
logger.logWarning("ERROR: IO trouble ",e);
return null;
}
}
private static void copy(OutputStream out, InputStream in, int bufferSize) throws IOException {

8
source/de/anomic/yacy/yacyClient.java
Unescape View File

@ -233,9 +233,15 @@ public final class yacyClient {
yacyCore.log.logInfo("hello/client: rejected contacting seed; too large (" + seedStr.length() + " > " + yacySeed.maxsize + ")");
} else {
//System.out.println("DEBUG yacyClient.publishMySeed seedStr = " + seedStr);
if (yacyCore.peerActions.peerArrival(yacySeed.genRemoteSeed(seedStr, post.get("key", ""), true), (i == 1))) count++;
yacySeed remoteSeed = yacySeed.genRemoteSeed(seedStr, post.get("key", ""), true);
if (remoteSeed == null) {
yacyCore.log.logWarning("hello/client: bad seed string from peer " + otherHash + ", address = " + address + ", count = " + (i-1) + " seedStr = " + seedStr);
} else {
if (yacyCore.peerActions.peerArrival(remoteSeed, (i == 1))) count++;
}
}
}
return count;
}

Write Preview
Loading…
Cancel
Save