- fixed a bug in remote search that prevented that any results had been generated (!)

- added a great number of printStackTrace and new exceptions that shall be used to find the cause
  for a bug in yacy client-server communication which causes the interruption of data transfer
  which then causes the parser bug for the seed strings.
- tried to fix the communication bug on server-side (copy functions)
Be aware that the log may be full of errors and bugs - there should not be more bugs but there is more to see


git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4519 6c8d7289-2bf4-0310-a012-ef5d649a1542

source/de/anomic/http/httpc.java

@@ -400,6 +400,7 @@ public final class httpc {
             this.socket.setSoTimeout(timeout); // waiting time for read
             // get the connection
             this.socket.connect(address, timeout);
+            this.socket.setSoTimeout(timeout); // waiting time for read
             
             if (incomingByteCountAccounting != null) {
                 this.clientInputByteCount = new httpdByteCountInputStream(this.socket.getInputStream(),incomingByteCountAccounting);
@@ -422,7 +423,7 @@ public final class httpc {
         } catch (IOException e) {
             // There was an error while connecting the socket, probably a SocketTimeoutException
             // we have to close the httpc, otherwise it would stay in activeConnections forever
-            serverLog.logFine("HTTPC", "Couldn't open socket to: " + this.adressed_host + ":" + this.adressed_port);
+            serverLog.logFine("HTTPC", "Couldn't open socket to " + this.adressed_host + ":" + this.adressed_port + ": " + e.getMessage());
             close();
             
             // TODO do we need to hand it over to the caller?
@@ -979,6 +980,8 @@ public final class httpc {
         // read connection body and return body
         serverByteBuffer sbb = new serverByteBuffer();
         res.writeContent(sbb, null);
+        if ((res.responseHeader.contentLength() > 0) && (res.responseHeader.contentLength() != sbb.length()))
+            throw new IOException("content length and loaded resource length from http://" + realhost + ":" + port + path + " does not match. HEADER.Content-Length = " + res.responseHeader.contentLength() + ", resource.length =" + sbb.length());
         con.close();
         return sbb.getBytes();
     }
@@ -1244,7 +1247,7 @@ public final class httpc {
             }
 
             // reads in the http header, right now, right here
-            byte[] b = serverCore.receive(httpc.this.clientInput, terminalMaxLength, false);
+            byte[] b = serverCore.receive(httpc.this.clientInput, terminalMaxLength, true);
             if (b == null) {
                 // the server has meanwhile disconnected
                 this.statusCode = 503;
@@ -1263,7 +1266,7 @@ public final class httpc {
             
             if ((this.statusCode==500)&&(this.statusText.equals("status line parse error"))) {
                 // flush in anything that comes without parsing
-                while ((b != null) && (b.length != 0)) b = serverCore.receive(httpc.this.clientInput, terminalMaxLength, false);
+                while ((b != null) && (b.length != 0)) b = serverCore.receive(httpc.this.clientInput, terminalMaxLength, true);
                 return; // in bad mood                
             }
                         
@@ -1271,13 +1274,13 @@ public final class httpc {
             if (this.statusCode == 400) {
                 // bad request
                 // flush in anything that comes without parsing
-                while ((b = serverCore.receive(httpc.this.clientInput, terminalMaxLength, false)).length != 0) {}
+                while ((b = serverCore.receive(httpc.this.clientInput, terminalMaxLength, true)).length != 0) {}
                 return; // in bad mood
             }
 
             // at this point we should have a valid response. read in the header properties
             String key = "";
-            while ((b = serverCore.receive(httpc.this.clientInput, terminalMaxLength, false)) != null) {
+            while ((b = serverCore.receive(httpc.this.clientInput, terminalMaxLength, true)) != null) {
                 if (b.length == 0) break;
                 buffer = new String(b);
                 buffer=buffer.trim();
@@ -1429,13 +1432,17 @@ public final class httpc {
                 } else {
                     throw new IllegalArgumentException("Invalid procOS object type '" + procOS.getClass().getName() + "'");
                 }
-            } catch (IOException e) {}
+            } catch (IOException e) {
+                e.printStackTrace();
+            }
             
             if (bufferOS != null) {
                 try {
                     bufferOS.flush();
                     bufferOS.close();
-                } catch (IOException e) {}
+                } catch (IOException e) {
+                    e.printStackTrace();
+                }
                 if (file.length() == 0) file.delete();
             }
         }
@@ -1453,13 +1460,16 @@ public final class httpc {
                     if (System.currentTimeMillis() - lastIO > 30000) break;
                     this.wait(300);
                     continue io;
-                } catch (InterruptedException e) {} // may happen without EOF
+                } catch (InterruptedException e) {
+                    e.printStackTrace();
+                } // may happen without EOF
                 lastIO = System.currentTimeMillis();
                 c += l;
                 if (procOS != null) procOS.write(buffer, 0, l);
                 if (bufferOS != null) bufferOS.write(buffer, 0, l);
             } catch (IOException e) {
-                //System.out.println("*** DEBUG: writeX/IOStream terminated with IOException, processed " + c + " bytes.");
+                System.out.println("*** DEBUG: writeX/IOStream terminated with IOException, processed " + c + " bytes. cause: " + e.getMessage());
+                e.printStackTrace();
                 break;
             }
             
@@ -1484,13 +1494,15 @@ public final class httpc {
                         if (System.currentTimeMillis() - lastIO > 30000) break;
                         this.wait(300);
                         continue io;
-                    } catch (InterruptedException e) {} // may happen without EOF
+                    } catch (InterruptedException e) {
+                        e.printStackTrace();
+                    } // may happen without EOF
                     lastIO = System.currentTimeMillis();
                     c += l;
                     if (procOS != null) procOS.write(buffer, 0, l);
                     if (bufferOSWriter != null) bufferOSWriter.write(buffer, 0, l);
                 } catch (IOException e) {
-                    //System.out.println("*** DEBUG: writeX/ReaderWriter terminated with IOException, processed " + c + " bytes.");
+                    System.out.println("*** DEBUG: writeX/ReaderWriter terminated with IOException, processed " + c + " bytes. cause: " + e.getMessage());
                     break;
                 }
                 

source/de/anomic/http/httpdByteCountInputStream.java

@@ -91,9 +91,14 @@ public class httpdByteCountInputStream extends FilterInputStream {
     }
 
     public int read(byte[] b, int off, int len) throws IOException {
+        try {
         int readCount = super.read(b, off, len);
         if (readCount > 0) this.byteCount += readCount;
         return readCount;
+        } catch (IOException e) {
+            e.printStackTrace();
+            throw new IOException(e.getMessage() + "; b.length = " + b.length + ", off = " + off + ", len = " + len);
+        }
     }
 
     public int read() throws IOException {

source/de/anomic/kelondro/kelondroSortStore.java

@@ -100,7 +100,7 @@ public class kelondroSortStore<E> extends kelondroSortStack<E> {
             return this.offstack;
         }
         if (size() < count) throw new RuntimeException("list(" + count + ") exceeded avaiable number of elements (" + size() + ")"); 
-        while (this.onstack.size() < count) {
+        while (this.offstack.size() < count) {
             Long w = this.onstack.firstKey();
             E element = this.onstack.remove(w);
             stackElement se = new stackElement(element, w);

source/de/anomic/plasma/crawler/plasmaCrawlQueues.java

@@ -178,8 +178,6 @@ public class plasmaCrawlQueues {
             log.logFine("CoreCrawl: online caution, omitting processing");
             return false;
         }
-        // if the server is busy, we do crawling more slowly
-        //if (!(cacheManager.idle())) try {Thread.currentThread().sleep(2000);} catch (InterruptedException e) {}
         
         // if crawling was paused we have to wait until we wer notified to continue
         Object[] status = (Object[]) sb.crawlJobsStatus.get(plasmaSwitchboard.CRAWLJOB_LOCAL_CRAWL);

source/de/anomic/server/serverCore.java

@@ -601,7 +601,7 @@ public final class serverCore extends serverAbstractThread implements serverThre
     	}
     
     	public byte[] readLine() {
-    	    return receive(this.in, serverCore.this.commandMaxLength, false);
+    	    return receive(this.in, serverCore.this.commandMaxLength, true);
     	}
     
         /**

source/de/anomic/server/serverFileUtils.java

@@ -72,7 +72,7 @@ import de.anomic.tools.nxTools;
 
 public final class serverFileUtils {
 
-    private static final int DEFAULT_BUFFER_SIZE = 4096;
+    private static final int DEFAULT_BUFFER_SIZE = 512;
     
     public static long copy(InputStream source, OutputStream dest) throws IOException {
         return copy(source,dest, -1);
@@ -305,11 +305,13 @@ public final class serverFileUtils {
     }
     
     public static void write(String source, Writer dest) throws IOException {
-        copy(source,dest);
+        dest.write(source);
+        dest.flush();
     }
 
     public static void write(byte[] source, OutputStream dest) throws IOException {
-        copy(new ByteArrayInputStream(source), dest, -1);
+        dest.write(source, 0, source.length);
+        dest.flush();
     }
 
     public static void write(byte[] source, File dest) throws IOException {
@@ -327,6 +329,7 @@ public final class serverFileUtils {
     	
         // support of gzipped data (requested by roland)      
         if ((source.length > 1) && (((source[1] << 8) | source[0]) == GZIPInputStream.GZIP_MAGIC)) {
+            System.out.println("DEBUG: uncompressGZipArray - uncompressing source");
             try {
                 ByteArrayInputStream byteInput = new ByteArrayInputStream(source);
                 ByteArrayOutputStream byteOutput = new ByteArrayOutputStream();

source/de/anomic/tools/crypt.java

@@ -119,13 +119,29 @@ public class crypt {
     }
 
     public static String simpleDecode(String encoded, String key) {
-    if (encoded == null || encoded.length() < 3) { return null; }
+        if (encoded == null || encoded.length() < 3) {
-    if (encoded.charAt(1) != '|') { return encoded; } // not encoded
+            return null;
+        }
+        if (encoded.charAt(1) != '|') {
+            return encoded;
+        } // not encoded
         switch (encoded.charAt(0)) {
-    case 'b' : return kelondroBase64Order.enhancedCoder.decodeString(encoded.substring(2), "de.anomic.tools.crypt.simpleDecode()");
+        case 'b': {
-    case 'z' : return gzip.gunzipString(kelondroBase64Order.enhancedCoder.decode(encoded.substring(2), "de.anomic.tools.crypt.simpleDecode()"));
+            return kelondroBase64Order.enhancedCoder.decodeString(encoded.substring(2), "de.anomic.tools.crypt.simpleDecode()");
-    case 'p' : return encoded.substring(2);
+        }
-    default  : return null;
+        case 'z':
+            try {
+                return gzip.gunzipString(kelondroBase64Order.enhancedCoder.decode(encoded.substring(2), "de.anomic.tools.crypt.simpleDecode()"));
+            } catch (Exception e) {
+                e.printStackTrace();
+                return null;
+            }
+        case 'p': {
+            return encoded.substring(2);
+        }
+        default: {
+            return null;
+        }
         }
     }
 

source/de/anomic/tools/cryptbig.java

@@ -395,7 +395,7 @@ public class cryptbig {
 	return salt + ((gzFlag) ? "1" : "0") + kelondroBase64Order.enhancedCoder.encode(enc);
     }
 
-    public static String descrambleString(String key, String s) {
+    public static String descrambleString(String key, String s) throws IOException {
 	String salt = s.substring(0, 8);
 	boolean gzFlag = (s.charAt(8) == '1');
 	s = s.substring(9);
@@ -434,7 +434,7 @@ public class cryptbig {
 	return null;
     }
 
-    public static String simpleDecode(String encoded, String key) {
+    public static String simpleDecode(String encoded, String key) throws IOException {
 	if ((encoded == null) || (encoded.length() < 3)) return null;
 	if (encoded.charAt(1) != '|') return encoded; // not encoded
 	char method = encoded.charAt(0);
@@ -620,7 +620,12 @@ public class cryptbig {
 	    // 'descramble' string
 	    if (s.length != 3) {help(); System.exit(-1);}
 	    long t = System.currentTimeMillis();
+	    try {
             System.out.println(descrambleString(s[1], s[2]));
+        } catch (IOException e) {
+            // TODO Auto-generated catch block
+            e.printStackTrace();
+        }
 	    System.out.println("Calculation time: " + (System.currentTimeMillis() - t) + " milliseconds");
 	    System.exit(0);
 	}

source/de/anomic/tools/gzip.java

@@ -107,19 +107,13 @@ public class gzip {
 	}
     }
 	
-    public static String gunzipString(byte[] in) {
+    public static String gunzipString(byte[] in) throws IOException {
-	try {
 	    InputStream  fin  = new GZIPInputStream(new ByteArrayInputStream(in));
 	    ByteArrayOutputStream fout = new ByteArrayOutputStream();
 	    copy(fout, fin, 128);
 	    fin.close();
 	    fout.close();
 	    return new String(fout.toByteArray(), "UTF-8");
-	} catch (IOException e) {
-            //System.err.println("ERROR: IO trouble ");
-	    logger.logWarning("ERROR: IO trouble ",e);
-	    return null;
-	}
     }
 
     private static void copy(OutputStream out, InputStream in, int bufferSize) throws IOException {

source/de/anomic/yacy/yacyClient.java

@@ -233,9 +233,15 @@ public final class yacyClient {
                 yacyCore.log.logInfo("hello/client: rejected contacting seed; too large (" + seedStr.length() + " > " + yacySeed.maxsize + ")");
             } else {
                 //System.out.println("DEBUG yacyClient.publishMySeed seedStr = " + seedStr);
-                if (yacyCore.peerActions.peerArrival(yacySeed.genRemoteSeed(seedStr, post.get("key", ""), true), (i == 1))) count++;
+                yacySeed remoteSeed = yacySeed.genRemoteSeed(seedStr, post.get("key", ""), true);
+                if (remoteSeed == null) {
+                    yacyCore.log.logWarning("hello/client: bad seed string from peer " + otherHash + ", address = " + address + ", count = " + (i-1) + " seedStr = " + seedStr);
+                } else {
+                    if (yacyCore.peerActions.peerArrival(remoteSeed, (i == 1))) count++;
+                }
             }
         }
+        
         return count;
     }