more fixes (character encoding, parser exceptions, http client failure, blob writing)

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5914 6c8d7289-2bf4-0310-a012-ef5d649a1542

source/de/anomic/http/httpClient.java

@@ -459,19 +459,19 @@ public class httpClient {
         } catch (final IllegalThreadStateException e) {
         	// cleanUp statistics
             yacyURL url = new yacyURL(method.getURI().toString(), null);
-            Latency.slowdown(url.hash().substring(6), url.getHost());
+            if (url.hash() != null) Latency.slowdown(url.hash().substring(6), url.getHost());
             HttpConnectionInfo.removeConnection(generateConInfo(method));
             throw e;
         } catch (final IOException e) {
             // cleanUp statistics
             yacyURL url = new yacyURL(method.getURI().toString(), null);
-            Latency.slowdown(url.hash().substring(6), url.getHost());
+            if (url.hash() != null) Latency.slowdown(url.hash().substring(6), url.getHost());
             HttpConnectionInfo.removeConnection(generateConInfo(method));
             throw e;
         } catch (final IllegalStateException e) {
             // cleanUp statistics
             yacyURL url = new yacyURL(method.getURI().toString(), null);
-            Latency.slowdown(url.hash().substring(6), url.getHost());
+            if (url.hash() != null) Latency.slowdown(url.hash().substring(6), url.getHost());
             HttpConnectionInfo.removeConnection(generateConInfo(method));
             throw new IOException(e.getMessage());
         }

source/de/anomic/http/httpResponseHeader.java

@@ -25,6 +25,7 @@ package de.anomic.http;
 
 import java.nio.charset.Charset;
 import java.nio.charset.IllegalCharsetNameException;
+import java.nio.charset.UnsupportedCharsetException;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.Map;
@@ -127,7 +128,11 @@ public class httpResponseHeader extends httpHeader {
                 return Charset.defaultCharset();
             }
         } catch(IllegalCharsetNameException e) {
-            Log.logSevere("httpHeader", "Charset in header is illegal: '"+ charSetName +"'\n    "+ toString());
+            Log.logSevere("httpHeader", "Charset in header is illegal: '"+ charSetName +"'\n    "+ toString() + "\n" + e.getMessage());
+            // use system default
+            return Charset.defaultCharset();
+        } catch (UnsupportedCharsetException e) {
+        	Log.logSevere("httpHeader", "Charset in header is unsupported: '"+ charSetName +"'\n    "+ toString() + "\n" + e.getMessage());
             // use system default
             return Charset.defaultCharset();
         }

source/de/anomic/kelondro/blob/BLOBHeap.java

@@ -156,8 +156,8 @@ public final class BLOBHeap extends BLOBHeapModifier implements BLOB {
         
         // append all contents of the buffer into one byte[]
         i = this.buffer.entrySet().iterator();
-        final int pos = (int) file.length();
-        int posFile = pos;
+        final long pos = file.length();
+        long posFile = pos;
         int posBuffer = 0;
         
         byte[] ba = new byte[l + (4 + this.index.row().primaryKeyLength) * this.buffer.size()];
@@ -175,6 +175,8 @@ public final class BLOBHeap extends BLOBHeapModifier implements BLOB {
             assert posBuffer + 4 + key.length <= ba.length : "posBuffer = " + posBuffer + ", key.length = " + key.length + ", ba.length = " + ba.length;
             System.arraycopy(key, 0, ba, posBuffer + 4, key.length);
             assert posBuffer + 4 + key.length + blob.length <= ba.length : "posBuffer = " + posBuffer + ", key.length = " + key.length + ", blob.length = " + blob.length + ", ba.length = " + ba.length;
+            //System.out.println("*** DEBUG blob.length=" + blob.length + ",ba.length=" + ba.length + ",posBuffer=" + posBuffer + ",key.length=" + key.length + ",blob.length=" + blob.length);
+            //System.err.println("*** DEBUG blob.length=" + blob.length + ",ba.length=" + ba.length + ",posBuffer=" + posBuffer + ",key.length=" + key.length + ",blob.length=" + blob.length);
             System.arraycopy(blob, 0, ba, posBuffer + 4 + key.length, blob.length); //java.lang.ArrayIndexOutOfBoundsException here
             posFile += 4 + key.length + blob.length;
             posBuffer += 4 + key.length + blob.length;

source/de/anomic/plasma/plasmaParser.java

@@ -37,6 +37,7 @@ import java.io.UnsupportedEncodingException;
 import java.net.MalformedURLException;
 import java.nio.charset.Charset;
 import java.nio.charset.IllegalCharsetNameException;
+import java.nio.charset.UnsupportedCharsetException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -720,7 +721,10 @@ public final class plasmaParser {
         	c = Charset.forName(charset);
         } catch (IllegalCharsetNameException e) {
         	c = Charset.defaultCharset();
+        } catch (UnsupportedCharsetException e) {
+        	c = Charset.defaultCharset();
         }
+        
         // parsing the content
         final htmlFilterContentScraper scraper = new htmlFilterContentScraper(location);        
         final htmlFilterWriter writer = new htmlFilterWriter(null,null,scraper,null,false);