diff --git a/source/de/anomic/http/httpc.java b/source/de/anomic/http/httpc.java index 0c17920d8..3659e7941 100644 --- a/source/de/anomic/http/httpc.java +++ b/source/de/anomic/http/httpc.java @@ -400,6 +400,7 @@ public final class httpc { this.socket.setSoTimeout(timeout); // waiting time for read // get the connection this.socket.connect(address, timeout); + this.socket.setSoTimeout(timeout); // waiting time for read if (incomingByteCountAccounting != null) { this.clientInputByteCount = new httpdByteCountInputStream(this.socket.getInputStream(),incomingByteCountAccounting); @@ -411,7 +412,7 @@ public final class httpc { // getting input and output streams this.clientInput = new PushbackInputStream((this.clientInputByteCount!=null)? this.clientInputByteCount: - this.socket.getInputStream()); + this.socket.getInputStream()); this.clientOutput = this.socket.getOutputStream(); // if we reached this point, we should have a connection @@ -422,7 +423,7 @@ public final class httpc { } catch (IOException e) { // There was an error while connecting the socket, probably a SocketTimeoutException // we have to close the httpc, otherwise it would stay in activeConnections forever - serverLog.logFine("HTTPC", "Couldn't open socket to: " + this.adressed_host + ":" + this.adressed_port); + serverLog.logFine("HTTPC", "Couldn't open socket to " + this.adressed_host + ":" + this.adressed_port + ": " + e.getMessage()); close(); // TODO do we need to hand it over to the caller? @@ -979,6 +980,8 @@ public final class httpc { // read connection body and return body serverByteBuffer sbb = new serverByteBuffer(); res.writeContent(sbb, null); + if ((res.responseHeader.contentLength() > 0) && (res.responseHeader.contentLength() != sbb.length())) + throw new IOException("content length and loaded resource length from http://" + realhost + ":" + port + path + " does not match. HEADER.Content-Length = " + res.responseHeader.contentLength() + ", resource.length =" + sbb.length()); con.close(); return sbb.getBytes(); } @@ -1244,7 +1247,7 @@ public final class httpc { } // reads in the http header, right now, right here - byte[] b = serverCore.receive(httpc.this.clientInput, terminalMaxLength, false); + byte[] b = serverCore.receive(httpc.this.clientInput, terminalMaxLength, true); if (b == null) { // the server has meanwhile disconnected this.statusCode = 503; @@ -1263,7 +1266,7 @@ public final class httpc { if ((this.statusCode==500)&&(this.statusText.equals("status line parse error"))) { // flush in anything that comes without parsing - while ((b != null) && (b.length != 0)) b = serverCore.receive(httpc.this.clientInput, terminalMaxLength, false); + while ((b != null) && (b.length != 0)) b = serverCore.receive(httpc.this.clientInput, terminalMaxLength, true); return; // in bad mood } @@ -1271,13 +1274,13 @@ public final class httpc { if (this.statusCode == 400) { // bad request // flush in anything that comes without parsing - while ((b = serverCore.receive(httpc.this.clientInput, terminalMaxLength, false)).length != 0) {} + while ((b = serverCore.receive(httpc.this.clientInput, terminalMaxLength, true)).length != 0) {} return; // in bad mood } // at this point we should have a valid response. read in the header properties String key = ""; - while ((b = serverCore.receive(httpc.this.clientInput, terminalMaxLength, false)) != null) { + while ((b = serverCore.receive(httpc.this.clientInput, terminalMaxLength, true)) != null) { if (b.length == 0) break; buffer = new String(b); buffer=buffer.trim(); @@ -1362,7 +1365,7 @@ public final class httpc { return new GZIPInputStream(httpc.this.clientInput); } else if (this.responseHeader.contentLength() != -1) { // use a httpContentLengthInputStream to read until the end of the response body is reached - return new httpContentLengthInputStream(httpc.this.clientInput,this.responseHeader.contentLength()); + return new httpContentLengthInputStream(httpc.this.clientInput, this.responseHeader.contentLength()); } // no Content-Lenght was set. In this case we can read until EOF return httpc.this.clientInput; @@ -1429,13 +1432,17 @@ public final class httpc { } else { throw new IllegalArgumentException("Invalid procOS object type '" + procOS.getClass().getName() + "'"); } - } catch (IOException e) {} + } catch (IOException e) { + e.printStackTrace(); + } if (bufferOS != null) { try { bufferOS.flush(); bufferOS.close(); - } catch (IOException e) {} + } catch (IOException e) { + e.printStackTrace(); + } if (file.length() == 0) file.delete(); } } @@ -1453,13 +1460,16 @@ public final class httpc { if (System.currentTimeMillis() - lastIO > 30000) break; this.wait(300); continue io; - } catch (InterruptedException e) {} // may happen without EOF + } catch (InterruptedException e) { + e.printStackTrace(); + } // may happen without EOF lastIO = System.currentTimeMillis(); c += l; if (procOS != null) procOS.write(buffer, 0, l); if (bufferOS != null) bufferOS.write(buffer, 0, l); } catch (IOException e) { - //System.out.println("*** DEBUG: writeX/IOStream terminated with IOException, processed " + c + " bytes."); + System.out.println("*** DEBUG: writeX/IOStream terminated with IOException, processed " + c + " bytes. cause: " + e.getMessage()); + e.printStackTrace(); break; } @@ -1484,13 +1494,15 @@ public final class httpc { if (System.currentTimeMillis() - lastIO > 30000) break; this.wait(300); continue io; - } catch (InterruptedException e) {} // may happen without EOF + } catch (InterruptedException e) { + e.printStackTrace(); + } // may happen without EOF lastIO = System.currentTimeMillis(); c += l; if (procOS != null) procOS.write(buffer, 0, l); if (bufferOSWriter != null) bufferOSWriter.write(buffer, 0, l); } catch (IOException e) { - //System.out.println("*** DEBUG: writeX/ReaderWriter terminated with IOException, processed " + c + " bytes."); + System.out.println("*** DEBUG: writeX/ReaderWriter terminated with IOException, processed " + c + " bytes. cause: " + e.getMessage()); break; } diff --git a/source/de/anomic/http/httpdByteCountInputStream.java b/source/de/anomic/http/httpdByteCountInputStream.java index f1e5a319c..54339bcb7 100644 --- a/source/de/anomic/http/httpdByteCountInputStream.java +++ b/source/de/anomic/http/httpdByteCountInputStream.java @@ -91,9 +91,14 @@ public class httpdByteCountInputStream extends FilterInputStream { } public int read(byte[] b, int off, int len) throws IOException { + try { int readCount = super.read(b, off, len); if (readCount > 0) this.byteCount += readCount; return readCount; + } catch (IOException e) { + e.printStackTrace(); + throw new IOException(e.getMessage() + "; b.length = " + b.length + ", off = " + off + ", len = " + len); + } } public int read() throws IOException { diff --git a/source/de/anomic/kelondro/kelondroSortStore.java b/source/de/anomic/kelondro/kelondroSortStore.java index 9fae754ca..d80461a58 100644 --- a/source/de/anomic/kelondro/kelondroSortStore.java +++ b/source/de/anomic/kelondro/kelondroSortStore.java @@ -100,7 +100,7 @@ public class kelondroSortStore extends kelondroSortStack { return this.offstack; } if (size() < count) throw new RuntimeException("list(" + count + ") exceeded avaiable number of elements (" + size() + ")"); - while (this.onstack.size() < count) { + while (this.offstack.size() < count) { Long w = this.onstack.firstKey(); E element = this.onstack.remove(w); stackElement se = new stackElement(element, w); diff --git a/source/de/anomic/plasma/crawler/plasmaCrawlQueues.java b/source/de/anomic/plasma/crawler/plasmaCrawlQueues.java index 057b093be..fe3805497 100644 --- a/source/de/anomic/plasma/crawler/plasmaCrawlQueues.java +++ b/source/de/anomic/plasma/crawler/plasmaCrawlQueues.java @@ -178,8 +178,6 @@ public class plasmaCrawlQueues { log.logFine("CoreCrawl: online caution, omitting processing"); return false; } - // if the server is busy, we do crawling more slowly - //if (!(cacheManager.idle())) try {Thread.currentThread().sleep(2000);} catch (InterruptedException e) {} // if crawling was paused we have to wait until we wer notified to continue Object[] status = (Object[]) sb.crawlJobsStatus.get(plasmaSwitchboard.CRAWLJOB_LOCAL_CRAWL); diff --git a/source/de/anomic/server/serverCore.java b/source/de/anomic/server/serverCore.java index a5cb41397..07a145770 100644 --- a/source/de/anomic/server/serverCore.java +++ b/source/de/anomic/server/serverCore.java @@ -601,7 +601,7 @@ public final class serverCore extends serverAbstractThread implements serverThre } public byte[] readLine() { - return receive(this.in, serverCore.this.commandMaxLength, false); + return receive(this.in, serverCore.this.commandMaxLength, true); } /** diff --git a/source/de/anomic/server/serverFileUtils.java b/source/de/anomic/server/serverFileUtils.java index bf1ff0fe6..4ca643b42 100644 --- a/source/de/anomic/server/serverFileUtils.java +++ b/source/de/anomic/server/serverFileUtils.java @@ -72,7 +72,7 @@ import de.anomic.tools.nxTools; public final class serverFileUtils { - private static final int DEFAULT_BUFFER_SIZE = 4096; + private static final int DEFAULT_BUFFER_SIZE = 512; public static long copy(InputStream source, OutputStream dest) throws IOException { return copy(source,dest, -1); @@ -95,13 +95,13 @@ public final class serverFileUtils { int chunkSize = (int) ((count > 0) ? Math.min(count, DEFAULT_BUFFER_SIZE) : DEFAULT_BUFFER_SIZE); int c; long total = 0; - while ((c = source.read(buffer,0,chunkSize)) > 0) { + while ((c = source.read(buffer, 0, chunkSize)) > 0) { dest.write(buffer, 0, c); dest.flush(); total += c; if (count > 0) { - chunkSize = (int)Math.min(count-total,DEFAULT_BUFFER_SIZE); + chunkSize = (int) Math.min(count-total, DEFAULT_BUFFER_SIZE); if (chunkSize == 0) break; } @@ -111,7 +111,7 @@ public final class serverFileUtils { return total; } - public static int copy (File source, String inputCharset, Writer dest) throws IOException { + public static int copy(File source, String inputCharset, Writer dest) throws IOException { InputStream fis = null; try { fis = new FileInputStream(source); @@ -121,18 +121,18 @@ public final class serverFileUtils { } } - public static int copy (InputStream source, Writer dest, String inputCharset) throws IOException { + public static int copy(InputStream source, Writer dest, String inputCharset) throws IOException { InputStreamReader reader = new InputStreamReader(source,inputCharset); return copy(reader,dest); } - public static int copy (String source, Writer dest) throws IOException { + public static int copy(String source, Writer dest) throws IOException { dest.write(source); dest.flush(); return source.length(); } - public static int copy (Reader source, Writer dest) throws IOException { + public static int copy(Reader source, Writer dest) throws IOException { char[] buffer = new char[4096]; int count = 0; int n = 0; @@ -188,7 +188,7 @@ public final class serverFileUtils { fis = new FileInputStream(source); long skipped = fis.skip(start); if (skipped != start) throw new IllegalStateException("Unable to skip '" + start + "' bytes. Only '" + skipped + "' bytes skipped."); - copy(fis, dest,-1); + copy(fis, dest, -1); } finally { if (fis != null) try { fis.close(); } catch (Exception e) {} } @@ -305,11 +305,13 @@ public final class serverFileUtils { } public static void write(String source, Writer dest) throws IOException { - copy(source,dest); + dest.write(source); + dest.flush(); } public static void write(byte[] source, OutputStream dest) throws IOException { - copy(new ByteArrayInputStream(source), dest, -1); + dest.write(source, 0, source.length); + dest.flush(); } public static void write(byte[] source, File dest) throws IOException { @@ -327,6 +329,7 @@ public final class serverFileUtils { // support of gzipped data (requested by roland) if ((source.length > 1) && (((source[1] << 8) | source[0]) == GZIPInputStream.GZIP_MAGIC)) { + System.out.println("DEBUG: uncompressGZipArray - uncompressing source"); try { ByteArrayInputStream byteInput = new ByteArrayInputStream(source); ByteArrayOutputStream byteOutput = new ByteArrayOutputStream(); diff --git a/source/de/anomic/tools/crypt.java b/source/de/anomic/tools/crypt.java index 2c201583c..4b5639954 100644 --- a/source/de/anomic/tools/crypt.java +++ b/source/de/anomic/tools/crypt.java @@ -119,14 +119,30 @@ public class crypt { } public static String simpleDecode(String encoded, String key) { - if (encoded == null || encoded.length() < 3) { return null; } - if (encoded.charAt(1) != '|') { return encoded; } // not encoded - switch (encoded.charAt(0)) { - case 'b' : return kelondroBase64Order.enhancedCoder.decodeString(encoded.substring(2), "de.anomic.tools.crypt.simpleDecode()"); - case 'z' : return gzip.gunzipString(kelondroBase64Order.enhancedCoder.decode(encoded.substring(2), "de.anomic.tools.crypt.simpleDecode()")); - case 'p' : return encoded.substring(2); - default : return null; - } + if (encoded == null || encoded.length() < 3) { + return null; + } + if (encoded.charAt(1) != '|') { + return encoded; + } // not encoded + switch (encoded.charAt(0)) { + case 'b': { + return kelondroBase64Order.enhancedCoder.decodeString(encoded.substring(2), "de.anomic.tools.crypt.simpleDecode()"); + } + case 'z': + try { + return gzip.gunzipString(kelondroBase64Order.enhancedCoder.decode(encoded.substring(2), "de.anomic.tools.crypt.simpleDecode()")); + } catch (Exception e) { + e.printStackTrace(); + return null; + } + case 'p': { + return encoded.substring(2); + } + default: { + return null; + } + } } public static void main(String[] args) { diff --git a/source/de/anomic/tools/cryptbig.java b/source/de/anomic/tools/cryptbig.java index b8ffb6999..dc1444329 100644 --- a/source/de/anomic/tools/cryptbig.java +++ b/source/de/anomic/tools/cryptbig.java @@ -395,7 +395,7 @@ public class cryptbig { return salt + ((gzFlag) ? "1" : "0") + kelondroBase64Order.enhancedCoder.encode(enc); } - public static String descrambleString(String key, String s) { + public static String descrambleString(String key, String s) throws IOException { String salt = s.substring(0, 8); boolean gzFlag = (s.charAt(8) == '1'); s = s.substring(9); @@ -434,7 +434,7 @@ public class cryptbig { return null; } - public static String simpleDecode(String encoded, String key) { + public static String simpleDecode(String encoded, String key) throws IOException { if ((encoded == null) || (encoded.length() < 3)) return null; if (encoded.charAt(1) != '|') return encoded; // not encoded char method = encoded.charAt(0); @@ -620,7 +620,12 @@ public class cryptbig { // 'descramble' string if (s.length != 3) {help(); System.exit(-1);} long t = System.currentTimeMillis(); - System.out.println(descrambleString(s[1], s[2])); + try { + System.out.println(descrambleString(s[1], s[2])); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } System.out.println("Calculation time: " + (System.currentTimeMillis() - t) + " milliseconds"); System.exit(0); } diff --git a/source/de/anomic/tools/gzip.java b/source/de/anomic/tools/gzip.java index d9eafeea9..52c97526a 100644 --- a/source/de/anomic/tools/gzip.java +++ b/source/de/anomic/tools/gzip.java @@ -107,19 +107,13 @@ public class gzip { } } - public static String gunzipString(byte[] in) { - try { + public static String gunzipString(byte[] in) throws IOException { InputStream fin = new GZIPInputStream(new ByteArrayInputStream(in)); ByteArrayOutputStream fout = new ByteArrayOutputStream(); copy(fout, fin, 128); fin.close(); fout.close(); return new String(fout.toByteArray(), "UTF-8"); - } catch (IOException e) { - //System.err.println("ERROR: IO trouble "); - logger.logWarning("ERROR: IO trouble ",e); - return null; - } } private static void copy(OutputStream out, InputStream in, int bufferSize) throws IOException { diff --git a/source/de/anomic/yacy/yacyClient.java b/source/de/anomic/yacy/yacyClient.java index d8dba2738..546b1fa30 100644 --- a/source/de/anomic/yacy/yacyClient.java +++ b/source/de/anomic/yacy/yacyClient.java @@ -233,9 +233,15 @@ public final class yacyClient { yacyCore.log.logInfo("hello/client: rejected contacting seed; too large (" + seedStr.length() + " > " + yacySeed.maxsize + ")"); } else { //System.out.println("DEBUG yacyClient.publishMySeed seedStr = " + seedStr); - if (yacyCore.peerActions.peerArrival(yacySeed.genRemoteSeed(seedStr, post.get("key", ""), true), (i == 1))) count++; + yacySeed remoteSeed = yacySeed.genRemoteSeed(seedStr, post.get("key", ""), true); + if (remoteSeed == null) { + yacyCore.log.logWarning("hello/client: bad seed string from peer " + otherHash + ", address = " + address + ", count = " + (i-1) + " seedStr = " + seedStr); + } else { + if (yacyCore.peerActions.peerArrival(remoteSeed, (i == 1))) count++; + } } } + return count; }