*) adding toString method to

- httpc
   - response
*) simplifying gzip encoding
*) remembering http version of contacted server
   (neede for later support of keep alive by httpc)
*) moving function shallTransportZipped to httpd.java
   because this function is used multiple times

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@242 6c8d7289-2bf4-0310-a012-ef5d649a1542

source/de/anomic/http/httpc.java

@@ -49,7 +49,6 @@
 
 package de.anomic.http;
 
-import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.FileOutputStream;
@@ -82,10 +81,11 @@ import org.apache.commons.pool.impl.GenericObjectPool;
 import de.anomic.server.serverByteBuffer;
 import de.anomic.server.serverCodings;
 import de.anomic.server.serverCore;
-import de.anomic.server.serverLog;
 import de.anomic.server.serverObjects;
+import de.anomic.server.logging.serverLog;
 import de.anomic.server.serverCore.Session;
 
+
 public final class httpc {
 
     // statics
@@ -126,6 +126,8 @@ public final class httpc {
         java.security.Security.setProperty("networkaddress.cache.negative.ttl" , "0");
     }
     
+
+    
     /**
      * A Object Pool containing all pooled httpc-objects.
      * @see httpcPool
@@ -164,6 +166,11 @@ public final class httpc {
      */
     final serverByteBuffer readLineBuffer = new serverByteBuffer();
     
+    public String toString() {
+        return (this.savedRemoteHost == null) ? "Disconnected" : "Connected to " + this.savedRemoteHost +  
+               ((this.remoteProxyUse) ? " via " + this.host : "");
+    }
+
     public static httpc getInstance(
             String server, 
             int port, 
@@ -371,9 +378,10 @@ public final class httpc {
 
 	// header information
 	public httpHeader responseHeader = null;
+    public String httpVer = "HTTP/0.9";
 	public String status; // the success/failure response string starting with status-code
 	private boolean gzip; // for gunzipping on-the-fly
-        private long gzipLength; // zipped-length of the response
+    private String encoding;
 
 	public response(boolean zipped) throws IOException {
 
@@ -390,7 +398,7 @@ public final class httpc {
 	    }
 
 	    // reads in the http header, right now, right here
-	    byte[] b = serverCore.receive(clientInput, readLineBuffer, timeout, terminalMaxLength, false);
+	    byte[] b = serverCore.receive(clientInput, readLineBuffer, terminalMaxLength, false);
 	    if (b == null) {
 		// the server has meanwhile disconnected
 		status = "503 server has closed connection";
@@ -402,9 +410,12 @@ public final class httpc {
 	    if (p < 0) {
 		status = "500 status line parse error";
 		// flush in anything that comes without parsing
-		while ((b = serverCore.receive(clientInput, readLineBuffer, timeout, terminalMaxLength, false)).length != 0) {}
+		while ((b != null) && (b.length != 0)) b = serverCore.receive(clientInput, readLineBuffer, terminalMaxLength, false); 
 		return; // in bad mood
 	    }
+        // the http version reported by the server
+        this.httpVer = buffer.substring(0,p); 
+        
 	    // we have a status
 	    status = buffer.substring(p + 1).trim(); // the status code plus reason-phrase
 
@@ -412,13 +423,13 @@ public final class httpc {
 	    if (status.startsWith("400")) {
 		// bad request
 		// flush in anything that comes without parsing
-		while ((b = serverCore.receive(clientInput, readLineBuffer, timeout, terminalMaxLength, false)).length != 0) {}
+		while ((b = serverCore.receive(clientInput, readLineBuffer, terminalMaxLength, false)).length != 0) {}
 		return; // in bad mood
 	    }
 
 	    // at this point we should have a valid response. read in the header properties
 	    String key = "";
-	    while ((b = serverCore.receive(clientInput, readLineBuffer, timeout, terminalMaxLength, false)) != null) {
+	    while ((b = serverCore.receive(clientInput, readLineBuffer, terminalMaxLength, false)) != null) {
 		if (b.length == 0) break;
 		buffer = new String(b);
 		//System.out.println("#H#" + buffer); // debug
@@ -446,18 +457,17 @@ public final class httpc {
 	    gzip = ((zipped) && (responseHeader.gzip()));
                 
 	    if (gzip) {
-		// change attributes in case of gzip decoding
-                gzipLength = responseHeader.contentLength();
             responseHeader.remove("CONTENT-ENCODING"); // we fake that we don't have encoding, since what comes out does not have gzip and we also don't know what was encoded
             responseHeader.remove("CONTENT-LENGTH"); // we cannot use the length during gunzippig yet; still we can hope that it works
-	    } else {
+	    }
-                gzipLength = -1;
     }
     
-            //System.out.println("###incoming header: " + responseHeader.toString());
+    public String toString() {
-            
+        StringBuffer toStringBuffer = new StringBuffer();
-	    // the body must be read separately by the get/writeContent methods
+        toStringBuffer.append((this.status == null) ? "Status: Unknown" : "Status: " + this.status)
-	    //System.out.println("## connection is " + ((socket.isClosed()) ? "closed" : "open") + ".");
+                      .append(" | Headers: ")
+                      .append((this.responseHeader == null) ? "none" : this.responseHeader.toString());
+        return toStringBuffer.toString();
     }
 
 	public boolean success() {
@@ -498,69 +508,42 @@ public final class httpc {
         long len = 0;
         
         // find out length
-	    long length = responseHeader.contentLength();
+        long length = this.responseHeader.contentLength();
+        
+        // using the proper intput stream
+        InputStream dis = (this.gzip) ? (InputStream) new GZIPInputStream(clientInput) : (InputStream) clientInput;
         
         // we have three methods of reading: length-based, length-based gzip and connection-close-based
+        try {
             if (length > 0) {
                 // we read exactly 'length' bytes
-		try {
+                while ((len < length) && ((l = dis.read(buffer)) >= 0)) {
-                    while ((len < length) && ((l = clientInput.read(buffer)) >= 0)) {
                     if (procOS != null) procOS.write(buffer, 0, l);
                     if (bufferOS != null) bufferOS.write(buffer, 0, l);
                     len += l;
                 }
-                } catch (java.net.SocketException e) {
-                    // this is an error:
-                    throw new IOException("Socket exception: " + e.getMessage());
-                } catch (java.net.SocketTimeoutException e) {
-                    // this is an error:
-                    throw new IOException("Socket time-out: " + e.getMessage());
-                }
-            } else if ((gzip) && (gzipLength > 0) && (gzipLength < 100000)) {
-                //System.out.println("PERFORMING NEW GZIP-LENGTH-BASED HTTPC: gzipLength=" + gzipLength); // DEBUG
-                // we read exactly 'gzipLength' bytes; first copy into buffer:
-                ByteArrayOutputStream baos = new ByteArrayOutputStream();
-                while ((len < gzipLength) && ((l = clientInput.read(buffer)) >= 0)) {
-                    baos.write(buffer, 0, l);
-                    len += l;
-                }
-                baos.flush();
-                // now uncompress
-                InputStream dis = new GZIPInputStream(new ByteArrayInputStream(baos.toByteArray()));
-                try {
-                    while ((l = dis.read(buffer)) > 0) {
-                        if (procOS != null) procOS.write(buffer, 0, l);
-                        if (bufferOS != null) bufferOS.write(buffer, 0, l);
-                        len += l;
-                    }
-                } catch (java.net.SocketException e) {
-                    // this is an error:
-                    throw new IOException("Socket exception: " + e.getMessage());
-                } catch (java.net.SocketTimeoutException e) {
-                    // this is an error:
-                    throw new IOException("Socket time-out: " + e.getMessage());
-                }
-                baos.close(); baos = null;
             } else {
                 // no content-length was given, thus we read until the connection closes
-    		InputStream dis = (gzip) ? (InputStream) new GZIPInputStream(clientInput) : (InputStream) clientInput;
-    		try {
                 while ((l = dis.read(buffer, 0, buffer.length)) >= 0) {
                     if (procOS != null) procOS.write(buffer, 0, l);
                     if (bufferOS != null) bufferOS.write(buffer, 0, l);
                 }
+            }
         } catch (java.net.SocketException e) {
-    		    // this is not an error: it's ok, we waited for that
+            throw new IOException("Socket exception: " + e.getMessage());
         } catch (java.net.SocketTimeoutException e) {
-                    // the same here; should be ok.
+            throw new IOException("Socket time-out: " + e.getMessage());
-		}
+        } finally {
-	    }
-
             // close the streams
-	    if (procOS != null) procOS.flush();
+            if (procOS != null) {
+                if (procOS instanceof httpChunkedOutputStream) 
+                    ((httpChunkedOutputStream)procOS).finish();
+                procOS.flush();
+            }
             if (bufferOS != null) bufferOS.flush();
             buffer = null;
         }
+    }
 
 	public void print() {
 	    serverLog.logInfo("HTTPC", "RESPONSE: status=" + status + ", header=" + responseHeader.toString());
@@ -659,7 +642,6 @@ public final class httpc {
         //System.out.println("***HEADER for path " + path + ": PROXY TO SERVER = " + header.toString()); // DEBUG
         Iterator i = header.keySet().iterator();
         String key;
-	String value;
         int count;
         char tag;
         while (i.hasNext()) {
@@ -668,35 +650,28 @@ public final class httpc {
             if ((tag != '*') && (tag != '#')) {
                 count = header.keyCount(key);
                 for (int j = 0; j < count; j++) {
-                    serverCore.send(clientOutput, key + ": " + ((String) header.getSingle(key, j)).trim());
+                    serverCore.send(this.clientOutput, key + ": " + ((String) header.getSingle(key, j)).trim());
                 }
                 //System.out.println("#" + key + ": " + value);
             }
         }
         
         // send terminating line
-	serverCore.send(clientOutput, "");
+        serverCore.send(this.clientOutput, "");
-	clientOutput.flush();
+        this.clientOutput.flush();
         
         // this is the place where www.stern.de refuses to answer ..???
     }
 
-	
-    private boolean shallTransportZipped(String path) {
-	return (!((path.endsWith(".gz")) || (path.endsWith(".tgz")) ||
-		  (path.endsWith(".jpg")) || (path.endsWith(".jpeg")) ||
-		  (path.endsWith(".gif")) | (path.endsWith(".zip"))));
-    }
-    
     public response GET(String path, httpHeader requestHeader) throws IOException {
         //serverLog.logDebug("HTTPC", handle + " requested GET '" + path + "', time = " + (System.currentTimeMillis() - handle));
         try {
-	    boolean zipped = shallTransportZipped(path);
+            boolean zipped = httpd.shallTransportZipped(path);
             send(httpHeader.METHOD_GET, path, requestHeader, zipped);
             response r = new response(zipped);
             //serverLog.logDebug("HTTPC", handle + " returned GET '" + path + "', time = " + (System.currentTimeMillis() - handle));
             return r;
-        } catch (SocketException e) {
+        } catch (Exception e) {
             throw new IOException(e.getMessage());
         }
     }