added some UTF-8 handling.

hope this will help somehow.. for shure not THE solution to our UTF-8 problem


git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@1308 6c8d7289-2bf4-0310-a012-ef5d649a1542

htroot/MessageSend_p.java

@@ -44,6 +44,7 @@
 // javac -classpath .:../Classes MessageSend_p.java
 // if the shell's current path is HTROOT
 
+import java.io.UnsupportedEncodingException;
 import java.text.SimpleDateFormat;
 import java.util.Date;
 import java.util.HashMap;
@@ -128,7 +129,13 @@ public class MessageSend_p {
                 if (messagesize < 1000) messagesize = 1000; // debug
                 if (subject.length() > 100) subject = subject.substring(0, 100);
                 if (message.length() > messagesize) message = message.substring(0, messagesize);
-                HashMap result = yacyClient.postMessage(hash, subject, message.getBytes());
+                byte[] mb;
+                try {
+                    mb = message.getBytes("UTF-8");
+                } catch (UnsupportedEncodingException e) {
+                    mb = message.getBytes();
+                }
+                HashMap result = yacyClient.postMessage(hash, subject, mb);
                 body += "<p>Your message has been sent. The target peer responded:</p>";
                 body += "<p><i>" + result.get("response") + "</i></p>";
             } catch (NumberFormatException e) {

htroot/Wiki.java

@@ -73,7 +73,7 @@ public class Wiki {
     }
 
 
-    public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) {
+    public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) throws IOException {
 	plasmaSwitchboard switchboard = (plasmaSwitchboard) env;
 	serverObjects prop = new serverObjects();
 	if (post == null) {
@@ -94,11 +94,15 @@ public class Wiki {
             }
         }
         
-	if (post.containsKey("submit")) {
+	    if (post.containsKey("submit")) {
-	    // store a new page
+	        // store a new page
-            switchboard.wikiDB.write(switchboard.wikiDB.newEntry(pagename, author, ip,
+	        byte[] content;
-								 post.get("reason", "edit"),
+	        try {
-								 post.get("content", "").getBytes()));
+	            content = post.get("content", "").getBytes("UTF-8");
+	        } catch (UnsupportedEncodingException e) {
+	            content = post.get("content", "").getBytes();
+	        }
+            switchboard.wikiDB.write(switchboard.wikiDB.newEntry(pagename, author, ip, post.get("reason", "edit"), content));
             // create a news message
             HashMap map = new HashMap();
             map.put("page", pagename);

htroot/htdocsdefault/dir.java

@@ -173,7 +173,7 @@ public class dir {
             try {
                 serverFileUtils.write(binary, newfile);
                 String md5s = serverCodings.encodeMD5Hex(newfile);
-                serverFileUtils.write((md5s + "\n" + description).getBytes(), newfilemd5); // generate md5
+                serverFileUtils.write((md5s + "\n" + description).getBytes("UTF-8"), newfilemd5); // generate md5
 
                 // index file info
                 if (post.get("indexing", "").equals("on")) {
@@ -262,7 +262,7 @@ public class dir {
                                 // generate md5 on-the-fly
                                 md5s = serverCodings.encodeMD5Hex(f);
                                 description = "";
-                                serverFileUtils.write((md5s + "\n" + description).getBytes(), fmd5);
+                                serverFileUtils.write((md5s + "\n" + description).getBytes("UTF-8"), fmd5);
                             }
                         } catch (IOException e) {
                             md5s = "";
@@ -478,7 +478,7 @@ public class dir {
     public static void deletePhrase(plasmaSwitchboard switchboard, String urlstring, String phrase, String descr) {
         try {
             final String urlhash = plasmaURL.urlHash(new URL(urlstring));
-            final Set words = plasmaCondenser.getWords(("yacyshare " + phrase + " " + descr).getBytes());
+            final Set words = plasmaCondenser.getWords(("yacyshare " + phrase + " " + descr).getBytes("UTF-8"));
             switchboard.removeReferences(urlhash, words);
             switchboard.urlPool.loadedURL.remove(urlhash);
         } catch (Exception e) {

htroot/yacy/message.java

@@ -49,6 +49,7 @@
 import java.io.File;
 import java.io.IOException;
 import java.io.PrintWriter;
+import java.io.UnsupportedEncodingException;
 import java.text.SimpleDateFormat;
 import java.util.Date;
 import de.anomic.data.messageBoard;
@@ -125,11 +126,17 @@ public final class message {
 
             // save message
             messageBoard.entry msgEntry = null;
+            byte[] mb;
+            try {
+                mb = message.getBytes("UTF-8");
+            } catch (UnsupportedEncodingException e) {
+                mb = message.getBytes();
+            }
             sb.messageDB.write(msgEntry = sb.messageDB.newEntry(
                     "remote",
                     otherSeed.get(yacySeed.NAME, "anonymous"), otherSeed.hash,
                     yacyCore.seedDB.mySeed.getName(), yacyCore.seedDB.mySeed.hash,
-                    subject, message.getBytes()));
+                    subject, mb));
 
             messageForwardingViaEmail(ss, msgEntry);
 

source/dbtest.java

@@ -24,6 +24,7 @@ public class dbtest {
 
     public final static int keylength = 12;
     public final static int valuelength = 223; // sum of all data length as defined in plasmaURL
+    //public final static long buffer = 0;
     public final static long buffer = 8192 * 1024; // 8 MB buffer
     public static byte[] dummyvalue1 = new byte[valuelength];
     public static byte[] dummyvalue2 = new byte[valuelength];

source/de/anomic/data/wikiBoard.java

@@ -149,7 +149,7 @@ public class wikiBoard {
         return author;
     }
 
-    public entry newEntry(String subject, String author, String ip, String reason, byte[] page) {
+    public entry newEntry(String subject, String author, String ip, String reason, byte[] page) throws IOException {
 	return new entry(normalize(subject), author, ip, reason, page);
     }
 
@@ -158,17 +158,17 @@ public class wikiBoard {
 	String key;
         Map record;
 
-	public entry(String subject, String author, String ip, String reason, byte[] page) {
+	public entry(String subject, String author, String ip, String reason, byte[] page) throws IOException {
 	    record = new HashMap();
 	    key = subject;
 	    if (key.length() > keyLength) key = key.substring(0, keyLength);
 	    record.put("date", dateString());
 	    if ((author == null) || (author.length() == 0)) author = "anonymous";
-	    record.put("author", kelondroBase64Order.enhancedCoder.encode(author.getBytes()));
+	    record.put("author", kelondroBase64Order.enhancedCoder.encode(author.getBytes("UTF-8")));
 	    if ((ip == null) || (ip.length() == 0)) ip = "";
 	    record.put("ip", ip);
 	    if ((reason == null) || (reason.length() == 0)) reason = "";
-	    record.put("reason", kelondroBase64Order.enhancedCoder.encode(reason.getBytes()));
+	    record.put("reason", kelondroBase64Order.enhancedCoder.encode(reason.getBytes("UTF-8")));
 	    if (page == null)
 		record.put("page", "");
 	    else

source/de/anomic/data/wikiCode.java

@@ -50,6 +50,7 @@ import java.io.BufferedReader;
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.io.InputStreamReader;
+import java.io.UnsupportedEncodingException;
 
 import de.anomic.plasma.plasmaSwitchboard;
 import de.anomic.server.serverCore;
@@ -73,7 +74,11 @@ public class wikiCode {
     }
 
     public String transform(String content){
-        return transform(content.getBytes(), sb);
+        try {
+            return transform(content.getBytes("UTF-8"), sb);
+        } catch (UnsupportedEncodingException e) {
+            return transform(content.getBytes(), sb);
+        }
     }
     public String transform(byte[] content){
         return transform(content, sb);

source/de/anomic/htmlFilter/htmlFilterContentScraper.java

@@ -230,7 +230,7 @@ public class htmlFilterContentScraper extends htmlFilterAbstractScraper implemen
     }
 
     public byte[] getText() {
-        return content.getBytes();
+       return content.getBytes();
     }
 
     public Map getAnchors() {

source/de/anomic/htmlFilter/htmlFilterContentTransformer.java

@@ -46,6 +46,7 @@ package de.anomic.htmlFilter;
 import java.io.BufferedReader;
 import java.io.File;
 import java.io.FileReader;
+import java.io.UnsupportedEncodingException;
 import java.text.Collator;
 import java.util.ArrayList;
 import java.util.Locale;
@@ -117,7 +118,12 @@ public class htmlFilterContentTransformer extends htmlFilterAbstractTransformer
 
     private boolean hit(byte[] text) {
         if (text == null || bluelist == null) return false;
-        String lc = new String(text).toLowerCase();
+        String lc;
+        try {
+            lc = new String(text, "UTF-8").toLowerCase();
+        } catch (UnsupportedEncodingException e) {
+            lc = new String(text).toLowerCase();
+        }
         for (int i = 0; i < bluelist.size(); i++) {
             if (lc.indexOf((String) bluelist.get(i)) >= 0) return true;
         }

source/de/anomic/http/httpd.java

@@ -710,7 +710,7 @@ public final class httpd implements serverHandler {
             bout.close(); bout = null;
         }
         
-        int argc = parseArgs(args, new String(buffer));
+        int argc = parseArgs(args, new String(buffer, "UTF-8"));
         buffer = null;
         return argc;
     }

source/de/anomic/kelondro/kelondroAbstractRA.java

@@ -152,7 +152,7 @@ abstract class kelondroAbstractRA implements kelondroRA {
                 return new String(bb, 0, bbsize);
             }
             if (c == cr) continue;
-            if (c == lf) return new String(bb, 0, bbsize);
+            if (c == lf) return new String(bb, 0, bbsize, "UTF-8");
 
             // append to bb
             if (bbsize == bb.length) {

source/de/anomic/kelondro/kelondroArray.java

@@ -98,7 +98,6 @@ public class kelondroArray extends kelondroRecords {
         return getNode(new Handle(index)).getValues();
     }
 
-    
     public synchronized int seti(int index, int value) throws IOException {
         int before = getHandle(index).hashCode();
         setHandle(index, new Handle(value));
@@ -109,13 +108,28 @@ public class kelondroArray extends kelondroRecords {
         return getHandle(index).hashCode();
     }
     
+    public synchronized int add(byte[][] row) throws IOException {
+        if (row.length != columns())
+            throw new IllegalArgumentException("add: wrong row length " + row.length + "; must be " + columns());
+
+        Node n = newNode();
+        n.commit(CP_LOW);
+        int index = n.handle().hashCode();
+        set(index, row);
+        return index;
+    }
+
+    public synchronized void remove(int index) throws IOException {
+        deleteNode(new Handle(index));
+    }
+    
     public void print() throws IOException {
         System.out.println("PRINTOUT of table, length=" + size());
         byte[][] row;
         for (int i = 0; i < size(); i++) {
             System.out.print("row " + i + ": ");
             row = get(i);
-            for (int j = 0; j < columns(); j++) System.out.print(((row[j] == null) ? "NULL" : new String(row[j])) + ", ");
+            for (int j = 0; j < columns(); j++) System.out.print(((row[j] == null) ? "NULL" : new String(row[j], "UTF-8")) + ", ");
             System.out.println();
         }
         System.out.println("EndOfTable");
@@ -160,6 +174,20 @@ public class kelondroArray extends kelondroRecords {
                 fm.set(Integer.parseInt(args[2]), row);
                 fm.close();
             } else
+            if ((args.length == 3) && (args[0].equals("-a"))) {
+                // add <filename> <value>
+                kelondroArray fm = new kelondroArray(new File(args[1]));
+                byte[][] row = new byte[][] { args[2].getBytes() };
+                int index = fm.add(row);
+                System.out.println("Added to row " + index);
+                fm.close();
+            } else
+            if ((args.length == 3) && (args[0].equals("-d"))) {
+                // delete <filename> <index>
+                kelondroArray fm = new kelondroArray(new File(args[1]));
+                fm.remove(Integer.parseInt(args[2]));
+                fm.close();
+            } else
             if ((args.length == 1) && (args[0].equals("-test"))) {
                 File testfile = new File("test.array");
                 if (testfile.exists()) testfile.delete();

source/de/anomic/kelondro/kelondroBase64Order.java

@@ -52,6 +52,19 @@ public class kelondroBase64Order extends kelondroAbstractOrder implements kelond
 
     private static final char[] alpha_standard = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/".toCharArray();
     private static final char[] alpha_enhanced = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_".toCharArray();
+    private static final byte[] ahpla_standard = new byte[256];
+    private static final byte[] ahpla_enhanced = new byte[256];
+    
+    static {
+        for (int i = 0; i < 256; i++) {
+            ahpla_standard[i] = -1;
+            ahpla_enhanced[i] = -1;
+        }
+        for (int i = 0; i < alpha_standard.length; i++) {
+            ahpla_standard[alpha_standard[i]] = (byte) i;
+            ahpla_enhanced[alpha_enhanced[i]] = (byte) i;
+        }
+    }
     
     public static final kelondroBase64Order standardCoder = new kelondroBase64Order(true);
     public static final kelondroBase64Order enhancedCoder = new kelondroBase64Order(false);
@@ -59,15 +72,14 @@ public class kelondroBase64Order extends kelondroAbstractOrder implements kelond
     final boolean rfc1113compliant;
 
     private final char[] alpha;
-    private final byte[] ahpla = new byte[256];
+    private final byte[] ahpla;
 
     public kelondroBase64Order(boolean rfc1113compliant) {
         // if we choose not to be rfc1113compliant,
         // then we get shorter base64 results which are also filename-compatible
         this.rfc1113compliant = rfc1113compliant;
         alpha = (rfc1113compliant) ? alpha_standard : alpha_enhanced;
-        for (int i = 0; i < 256; i++) ahpla[i] = -1;
+        ahpla = (rfc1113compliant) ? ahpla_standard : ahpla_enhanced;
-        for (int i = 0; i < alpha.length; i++) ahpla[alpha[i]] = (byte) i;
     }
 
     public char encodeByte(byte b) {
@@ -143,7 +155,8 @@ public class kelondroBase64Order extends kelondroAbstractOrder implements kelond
 
     public String decodeString(String in) {
         try {
-            return new String(decode(in), "ISO-8859-1");
+            //return new String(decode(in), "ISO-8859-1");
+            return new String(decode(in), "UTF-8");
         } catch (java.io.UnsupportedEncodingException e) {
             System.out.println("internal error in base64: " + e.getMessage());
             return null;

source/de/anomic/kelondro/kelondroHashtable.java

@@ -230,7 +230,7 @@ public class kelondroHashtable {
             rowNumber = hash.node();
             if (rowNumber >= hashArray.size()) return new Object[]{new Integer(rowNumber), null};
             row = hashArray.get(rowNumber);
-            rowKey = (int) kelondroBase64Order.enhancedCoder.decodeLong(new String(row[0]));
+            rowKey = (int) kelondroBase64Order.enhancedCoder.decodeLong(new String(row[0], "UTF-8"));
             if (rowKey == 0) return new Object[]{new Integer(rowNumber), null};
             hash.rehash();
         } while (rowKey != hash.key());

source/de/anomic/kelondro/kelondroNaturalOrder.java

@@ -68,6 +68,10 @@ public class kelondroNaturalOrder extends kelondroAbstractOrder implements kelon
     // two arrays are also equal if one array is a subset of the other's array
     // with filled-up char(0)-values
     public int compare(byte[] a, byte[] b) {
+        return compares(a, b);
+    }
+
+    public static final int compares(byte[] a, byte[] b) {
         int i = 0;
         final int al = a.length;
         final int bl = b.length;

source/de/anomic/kelondro/kelondroRecords.java

@@ -790,7 +790,7 @@ public class kelondroRecords {
                     if (h == null) s = s + ":hNULL"; else s = s + ":h" + h.toString();
                 }
                 byte[][] content = getValues();
-                for (int i = 0; i < content.length; i++) s = s + ":" + ((content[i] == null) ? "NULL" : (new String(content[i])).trim());
+                for (int i = 0; i < content.length; i++) s = s + ":" + ((content[i] == null) ? "NULL" : (new String(content[i], "UTF-8")).trim());
             } catch (IOException e) {
                 s = s + ":***LOAD ERROR***:" + e.getMessage();
             }

source/de/anomic/plasma/parser/odt/odtParser.java

@@ -130,11 +130,11 @@ public class odtParser extends AbstractParser implements Parser {
                         if (docShortTitle != null) {
                             docLongTitle = docShortTitle;
                         } else if (docContent.length <= 80) {
-                            docLongTitle = new String(docContent);
+                            docLongTitle = new String(docContent, "UTF-8");
                         } else {
                             byte[] title = new byte[80];
                             System.arraycopy(docContent, 0, title, 0, 80);
-                            docLongTitle = new String(title);
+                            docLongTitle = new String(title, "UTF-8");
                         }
                         docLongTitle.
                         replaceAll("\r\n"," ").

source/de/anomic/plasma/parser/pdf/pdfParser.java

@@ -127,7 +127,7 @@ public class pdfParser extends AbstractParser implements Parser {
             out = null;
 			
             if ((docTitle == null) || (docTitle.length() == 0)) {
-                docTitle = ((contents.length > 80)? new String(contents, 0, 80):new String(contents)).
+                docTitle = ((contents.length > 80)? new String(contents, 0, 80, "UTF-8"):new String(contents, "UTF-8")).
                 replaceAll("\r\n"," ").
                 replaceAll("\n"," ").
                 replaceAll("\r"," ").

source/de/anomic/plasma/plasmaCrawlEURL.java

@@ -171,15 +171,15 @@ public class plasmaCrawlEURL extends plasmaURL {
             this.hash = hash;
             byte[][] entry = urlHashCache.get(hash.getBytes());
             if (entry != null) {
-                this.referrer = new String(entry[1]);
+                this.referrer = new String(entry[1], "UTF-8");
-                this.initiator = new String(entry[2]);
+                this.initiator = new String(entry[2], "UTF-8");
-                this.executor = new String(entry[3]);
+                this.executor = new String(entry[3], "UTF-8");
-                this.url = new URL(new String(entry[4]).trim());
+                this.url = new URL(new String(entry[4], "UTF-8").trim());
-                this.name = new String(entry[5]).trim();
+                this.name = new String(entry[5], "UTF-8").trim();
-                this.initdate = new Date(86400000 * kelondroBase64Order.enhancedCoder.decodeLong(new String(entry[6])));
+                this.initdate = new Date(86400000 * kelondroBase64Order.enhancedCoder.decodeLong(new String(entry[6], "UTF-8")));
-                this.trydate = new Date(86400000 * kelondroBase64Order.enhancedCoder.decodeLong(new String(entry[7])));
+                this.trydate = new Date(86400000 * kelondroBase64Order.enhancedCoder.decodeLong(new String(entry[7], "UTF-8")));
-                this.trycount = (int) kelondroBase64Order.enhancedCoder.decodeLong(new String(entry[8]));
+                this.trycount = (int) kelondroBase64Order.enhancedCoder.decodeLong(new String(entry[8], "UTF-8"));
-                this.failreason = new String(entry[9]);
+                this.failreason = new String(entry[9], "UTF-8");
                 this.flags = new bitfield(entry[10]);
                 return;
             }

source/de/anomic/plasma/plasmaCrawlLURL.java

@@ -475,18 +475,18 @@ public final class plasmaCrawlLURL extends plasmaURL {
             if (entry == null) throw new IOException("url hash " + urlHash + " not found in LURL");
             try {
                 if (entry != null) {
-                    this.url = new URL(new String(entry[1]).trim());
+                    this.url = new URL(new String(entry[1], "UTF-8").trim());
-                    this.descr = (entry[2] == null) ? this.url.toString() : new String(entry[2]).trim();
+                    this.descr = (entry[2] == null) ? this.url.toString() : new String(entry[2], "UTF-8").trim();
-                    this.moddate = new Date(86400000 * kelondroBase64Order.enhancedCoder.decodeLong(new String(entry[3])));
+                    this.moddate = new Date(86400000 * kelondroBase64Order.enhancedCoder.decodeLong(new String(entry[3], "UTF-8")));
-                    this.loaddate = new Date(86400000 * kelondroBase64Order.enhancedCoder.decodeLong(new String(entry[4])));
+                    this.loaddate = new Date(86400000 * kelondroBase64Order.enhancedCoder.decodeLong(new String(entry[4], "UTF-8")));
-                    this.referrerHash = (entry[5] == null) ? dummyHash : new String(entry[5]);
+                    this.referrerHash = (entry[5] == null) ? dummyHash : new String(entry[5], "UTF-8");
-                    this.copyCount = (int) kelondroBase64Order.enhancedCoder.decodeLong(new String(entry[6]));
+                    this.copyCount = (int) kelondroBase64Order.enhancedCoder.decodeLong(new String(entry[6], "UTF-8"));
-                    this.flags = new String(entry[7]);
+                    this.flags = new String(entry[7], "UTF-8");
-                    this.quality = (int) kelondroBase64Order.enhancedCoder.decodeLong(new String(entry[8]));
+                    this.quality = (int) kelondroBase64Order.enhancedCoder.decodeLong(new String(entry[8], "UTF-8"));
-                    this.language = new String(entry[9]);
+                    this.language = new String(entry[9], "UTF-8");
                     this.doctype = (char) entry[10][0];
-                    this.size = kelondroBase64Order.enhancedCoder.decodeLong(new String(entry[11]));
+                    this.size = kelondroBase64Order.enhancedCoder.decodeLong(new String(entry[11], "UTF-8"));
-                    this.wordCount = (int) kelondroBase64Order.enhancedCoder.decodeLong(new String(entry[12]));
+                    this.wordCount = (int) kelondroBase64Order.enhancedCoder.decodeLong(new String(entry[12], "UTF-8"));
                     this.snippet = null;
                     return;
                 }

source/de/anomic/plasma/plasmaCrawlNURL.java

@@ -189,43 +189,43 @@ public class plasmaCrawlNURL extends plasmaURL {
             Iterator i;
             try {
                 i = coreStack.iterator();
-                while (i.hasNext()) stackIndex.add(new String((byte[]) i.next()));
+                while (i.hasNext()) stackIndex.add(new String((byte[]) i.next(), "UTF-8"));
             } catch (Exception e) {
                 coreStack.reset();
             }
             try {
                 i = limitStack.iterator();
-                while (i.hasNext()) stackIndex.add(new String((byte[]) i.next()));
+                while (i.hasNext()) stackIndex.add(new String((byte[]) i.next(), "UTF-8"));
             } catch (Exception e) {
                 limitStack.reset();
             }
             try {
                 i = overhangStack.iterator();
-                while (i.hasNext()) stackIndex.add(new String((byte[]) i.next()));
+                while (i.hasNext()) stackIndex.add(new String((byte[]) i.next(), "UTF-8"));
             } catch (Exception e) {
                 overhangStack.reset();
             }
             try {
                 i = remoteStack.iterator();
-                while (i.hasNext()) stackIndex.add(new String((byte[]) i.next()));
+                while (i.hasNext()) stackIndex.add(new String((byte[]) i.next(), "UTF-8"));
             } catch (Exception e) {
                 remoteStack.reset();
             }
             try {
                 i = imageStack.iterator();
-                while (i.hasNext()) stackIndex.add(new String(((kelondroRecords.Node) i.next()).getKey()));
+                while (i.hasNext()) stackIndex.add(new String(((kelondroRecords.Node) i.next()).getKey(), "UTF-8"));
             } catch (Exception e) {
                 imageStack = kelondroStack.reset(imageStack);
             }
             try {
                 i = movieStack.iterator();
-                while (i.hasNext()) stackIndex.add(new String(((kelondroRecords.Node) i.next()).getKey()));
+                while (i.hasNext()) stackIndex.add(new String(((kelondroRecords.Node) i.next()).getKey(), "UTF-8"));
             } catch (Exception e) {
                 movieStack = kelondroStack.reset(movieStack);
             }
             try {
                 i = musicStack.iterator();
-                while (i.hasNext()) stackIndex.add(new String(((kelondroRecords.Node) i.next()).getKey()));
+                while (i.hasNext()) stackIndex.add(new String(((kelondroRecords.Node) i.next()).getKey(), "UTF-8"));
             } catch (Exception e) {
                 musicStack = kelondroStack.reset(musicStack);
             }

source/de/anomic/plasma/plasmaCrawlStacker.java

@@ -401,17 +401,17 @@ public final class plasmaCrawlStacker {
 
             try {
                 this.urlHash       = urlHash;
-                this.initiator     = new String(entryBytes[1]);
+                this.initiator     = new String(entryBytes[1], "UTF-8");
-                this.url           = new String(entryBytes[2]).trim();
+                this.url           = new String(entryBytes[2], "UTF-8").trim();
-                this.referrerHash      = (entryBytes[3]==null) ? plasmaURL.dummyHash : new String(entryBytes[3]);
+                this.referrerHash      = (entryBytes[3]==null) ? plasmaURL.dummyHash : new String(entryBytes[3], "UTF-8");
-                this.name          = (entryBytes[4] == null) ? "" : new String(entryBytes[4]).trim();
+                this.name          = (entryBytes[4] == null) ? "" : new String(entryBytes[4], "UTF-8").trim();
-                this.loaddate      = new Date(86400000 * kelondroBase64Order.enhancedCoder.decodeLong(new String(entryBytes[5])));
+                this.loaddate      = new Date(86400000 * kelondroBase64Order.enhancedCoder.decodeLong(new String(entryBytes[5], "UTF-8")));
-                this.profileHandle = (entryBytes[6] == null) ? null : new String(entryBytes[6]).trim();
+                this.profileHandle = (entryBytes[6] == null) ? null : new String(entryBytes[6], "UTF-8").trim();
-                this.depth         = (int) kelondroBase64Order.enhancedCoder.decodeLong(new String(entryBytes[7]));
+                this.depth         = (int) kelondroBase64Order.enhancedCoder.decodeLong(new String(entryBytes[7], "UTF-8"));
-                this.anchors       = (int) kelondroBase64Order.enhancedCoder.decodeLong(new String(entryBytes[8]));
+                this.anchors       = (int) kelondroBase64Order.enhancedCoder.decodeLong(new String(entryBytes[8], "UTF-8"));
-                this.forkfactor    = (int) kelondroBase64Order.enhancedCoder.decodeLong(new String(entryBytes[9]));
+                this.forkfactor    = (int) kelondroBase64Order.enhancedCoder.decodeLong(new String(entryBytes[9], "UTF-8"));
                 this.flags         = new bitfield(entryBytes[10]);
-                this.handle        = Integer.parseInt(new String(entryBytes[11]));
+                this.handle        = Integer.parseInt(new String(entryBytes[11], "UTF-8"));
             } catch (Exception e) {
                 e.printStackTrace();
                 throw new IllegalStateException();

source/de/anomic/plasma/plasmaSwitchboardQueue.java

@@ -201,21 +201,21 @@ public class plasmaSwitchboardQueue {
             this.referrerURL = null;
         }
 
-        public Entry(byte[][] row) {
+        public Entry(byte[][] row) throws IOException {
-            long ims = (row[2] == null) ? 0 : kelondroBase64Order.enhancedCoder.decodeLong(new String(row[2]));
+            long ims = (row[2] == null) ? 0 : kelondroBase64Order.enhancedCoder.decodeLong(new String(row[2], "UTF-8"));
             byte flags = (row[3] == null) ? 0 : row[3][0];
             try {
-                this.url = new URL(new String(row[0]));
+                this.url = new URL(new String(row[0], "UTF-8"));
             } catch (MalformedURLException e) {
                 this.url = null;
             }
-            this.referrerHash = (row[1] == null) ? null : new String(row[1]);
+            this.referrerHash = (row[1] == null) ? null : new String(row[1], "UTF-8");
             this.ifModifiedSince = (ims == 0) ? null : new Date(ims);
             this.flags = ((flags & 1) == 1) ? (byte) 1 : (byte) 0;
-            this.initiator = (row[4] == null) ? null : new String(row[4]);
+            this.initiator = (row[4] == null) ? null : new String(row[4], "UTF-8");
-            this.depth = (int) kelondroBase64Order.enhancedCoder.decodeLong(new String(row[5]));
+            this.depth = (int) kelondroBase64Order.enhancedCoder.decodeLong(new String(row[5], "UTF-8"));
-            this.profileHandle = new String(row[6]);
+            this.profileHandle = new String(row[6], "UTF-8");
-            this.anchorName = (row[7] == null) ? null : (new String(row[7])).trim();
+            this.anchorName = (row[7] == null) ? null : (new String(row[7], "UTF-8")).trim();
 
             this.profileEntry = null;
             this.responseHeader = null;

source/de/anomic/plasma/plasmaWordConnotation.java

@@ -73,7 +73,7 @@ public class plasmaWordConnotation {
         //reference = reference.toLowerCase();
         byte[][] record = refDB.get(word, reference.getBytes());
         long c;
-        if (record == null) c = 0; else c = kelondroBase64Order.enhancedCoder.decodeLong(new String(record[1]));
+        if (record == null) c = 0; else c = kelondroBase64Order.enhancedCoder.decodeLong(new String(record[1], "UTF-8"));
         record[1] = kelondroBase64Order.enhancedCoder.encodeLong(c++, countlength).getBytes();
         refDB.put(word, record);
     }

source/de/anomic/plasma/plasmaWordIndexAssortment.java

@@ -200,8 +200,7 @@ public final class plasmaWordIndexAssortment {
         for (int i = 0; i < assortmentLength; i++) {
             container.add(
                     new plasmaWordIndexEntry[] { new plasmaWordIndexEntry(
-                            new String(row[3 + 2 * i]), new String(
+                            new String(row[3 + 2 * i]), new String(row[4 + 2 * i])) }, updateTime);
-                                    row[4 + 2 * i])) }, updateTime);
         }
         return container;
     }

source/de/anomic/plasma/plasmaWordIndexCache.java

@@ -193,9 +193,9 @@ public final class plasmaWordIndexCache implements plasmaWordIndexInterface {
                     // get out one entry
                     row = dumpArray.get(i);
                     if ((row[0] == null) || (row[1] == null) || (row[2] == null) || (row[3] == null) || (row[4] == null)) continue;
-                    wordHash = new String(row[0]);
+                    wordHash = new String(row[0], "UTF-8");
                     creationTime = kelondroRecords.bytes2long(row[2]);
-                    wordEntry = new plasmaWordIndexEntry(new String(row[3]), new String(row[4]));
+                    wordEntry = new plasmaWordIndexEntry(new String(row[3], "UTF-8"), new String(row[4], "UTF-8"));
                     // store to cache
                     addEntry(wordHash, wordEntry, creationTime);
                     urlCount++;

source/de/anomic/server/serverFileUtils.java

@@ -241,7 +241,7 @@ public final class serverFileUtils {
     public static Set loadSet(File file, String sep, boolean tree) throws IOException {
         Set set = (tree) ? (Set) new TreeSet() : (Set) new HashSet();
         byte[] b = read(file);
-        StringTokenizer st = new StringTokenizer(new String(b), sep);
+        StringTokenizer st = new StringTokenizer(new String(b, "UTF-8"), sep);
         while (st.hasMoreTokens()) {
             set.add(st.nextToken());
         }

source/de/anomic/soap/httpdSoapHandler.java

@@ -468,7 +468,7 @@ public final class httpdSoapHandler extends httpdAbstractHandler implements http
             if (templates[i].endsWith(".template")) try {
             //System.out.println("TEMPLATE " + templates[i].substring(0, templates[i].length() - 9) + ": " + new String(buf, 0, c));
             result.put(templates[i].substring(0, templates[i].length() - 9),
-                               new String(serverFileUtils.read(new File(path, templates[i]))));
+                               new String(serverFileUtils.read(new File(path, templates[i])), "UTF-8"));
             } catch (Exception e) {}
         }
         return result;

source/de/anomic/soap/httpdSoapService.java

@@ -232,7 +232,7 @@ public class httpdSoapService
             
             // convert it into a byte array and send it back as result
             byte[] result = o.toByteArray();            
-            return new String(result); 
+            return new String(result, "UTF-8"); 
         } catch (Exception e) {
             throw new AxisFault(e.getMessage());
         }

source/de/anomic/tools/cryptbig.java

@@ -164,7 +164,7 @@ public class cryptbig {
 	    if (b64dec == null) return null; // error in input string (inconsistency)
 	    byte[] dec = decryptArray(b64dec);
 	    if (dec == null) return null;
-	    return new String(dec, "UTF8");
+	    return new String(dec, "UTF-8");
 	} catch (UnsupportedEncodingException e) {
 	}
 	return null;

source/de/anomic/tools/gzip.java

@@ -105,7 +105,7 @@ public class gzip {
 	    copy(fout, fin, 128);
 	    fin.close();
 	    fout.close();
-	    return new String(fout.toByteArray(), "UTF8");
+	    return new String(fout.toByteArray(), "UTF-8");
 	} catch (IOException e) {
 	    System.err.println("ERROR: IO trouble");
 	    return null;

source/de/anomic/yacy/yacyClient.java

@@ -44,6 +44,7 @@
 package de.anomic.yacy;
 
 import java.io.IOException;
+import java.io.UnsupportedEncodingException;
 import java.net.URL;
 import java.util.ArrayList;
 import java.util.Date;
@@ -550,8 +551,11 @@ public final class yacyClient {
             post.put("youare", targetHash);
             post.put("subject", subject);
             post.put(yacySeed.MYTIME, yacyCore.universalDateShortString(new Date()));
-            post.put("message", new String(message));
+            try {
-            
+                post.put("message", new String(message, "UTF-8"));
+            } catch (UnsupportedEncodingException e) {
+                post.put("message", new String(message));
+            }
         // get target address    
         String address = targetAddress(targetHash);
         

source/yacy.java

@@ -1165,7 +1165,7 @@ public final class yacy {
                 entry = (plasmaCrawlLURL.Entry) eiter.next();
                 if ((entry != null) && (entry.url() != null)) {
                     if (html) {
-                        bos.write(("<a href=\"" + entry.url() + "\">" + entry.descr() + "</a><br>").getBytes());
+                        bos.write(("<a href=\"" + entry.url() + "\">" + entry.descr() + "</a><br>").getBytes("UTF-8"));
                         bos.write(serverCore.crlf);
                     } else {
                         bos.write(entry.url().toString().getBytes());