boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

added some UTF-8 handling.

Browse Source 
hope this will help somehow.. for shure not THE solution to our UTF-8 problem


git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@1308 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 19 years ago
parent 7586fcb1a4
commit 9544c47684
34 changed files with 168 additions and 90 deletions
Show Stats Download Patch File Download Diff File

9
htroot/MessageSend_p.java
Unescape View File

@ -44,6 +44,7 @@
// javac -classpath .:../Classes MessageSend_p.java
// if the shell's current path is HTROOT
import java.io.UnsupportedEncodingException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.HashMap;
@ -128,7 +129,13 @@ public class MessageSend_p {
if (messagesize < 1000) messagesize = 1000; // debug
if (subject.length() > 100) subject = subject.substring(0, 100);
if (message.length() > messagesize) message = message.substring(0, messagesize);
HashMap result = yacyClient.postMessage(hash, subject, message.getBytes());
byte[] mb;
try {
mb = message.getBytes("UTF-8");
} catch (UnsupportedEncodingException e) {
mb = message.getBytes();
}
HashMap result = yacyClient.postMessage(hash, subject, mb);
body += "<p>Your message has been sent. The target peer responded:</p>";
body += "<p><i>" + result.get("response") + "</i></p>";
} catch (NumberFormatException e) {

16
htroot/Wiki.java
Unescape View File

@ -73,7 +73,7 @@ public class Wiki {
}
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) {
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) throws IOException {
plasmaSwitchboard switchboard = (plasmaSwitchboard) env;
serverObjects prop = new serverObjects();
if (post == null) {
@ -94,11 +94,15 @@ public class Wiki {
}
}
if (post.containsKey("submit")) {
// store a new page
switchboard.wikiDB.write(switchboard.wikiDB.newEntry(pagename, author, ip,
post.get("reason", "edit"),
post.get("content", "").getBytes()));
if (post.containsKey("submit")) {
// store a new page
byte[] content;
try {
content = post.get("content", "").getBytes("UTF-8");
} catch (UnsupportedEncodingException e) {
content = post.get("content", "").getBytes();
}
switchboard.wikiDB.write(switchboard.wikiDB.newEntry(pagename, author, ip, post.get("reason", "edit"), content));
// create a news message
HashMap map = new HashMap();
map.put("page", pagename);

6
htroot/htdocsdefault/dir.java
Unescape View File

@ -173,7 +173,7 @@ public class dir {
try {
serverFileUtils.write(binary, newfile);
String md5s = serverCodings.encodeMD5Hex(newfile);
serverFileUtils.write((md5s + "\n" + description).getBytes(), newfilemd5); // generate md5
serverFileUtils.write((md5s + "\n" + description).getBytes("UTF-8"), newfilemd5); // generate md5
// index file info
if (post.get("indexing", "").equals("on")) {
@ -262,7 +262,7 @@ public class dir {
// generate md5 on-the-fly
md5s = serverCodings.encodeMD5Hex(f);
description = "";
serverFileUtils.write((md5s + "\n" + description).getBytes(), fmd5);
serverFileUtils.write((md5s + "\n" + description).getBytes("UTF-8"), fmd5);
}
} catch (IOException e) {
md5s = "";
@ -478,7 +478,7 @@ public class dir {
public static void deletePhrase(plasmaSwitchboard switchboard, String urlstring, String phrase, String descr) {
try {
final String urlhash = plasmaURL.urlHash(new URL(urlstring));
final Set words = plasmaCondenser.getWords(("yacyshare " + phrase + " " + descr).getBytes());
final Set words = plasmaCondenser.getWords(("yacyshare " + phrase + " " + descr).getBytes("UTF-8"));
switchboard.removeReferences(urlhash, words);
switchboard.urlPool.loadedURL.remove(urlhash);
} catch (Exception e) {

9
htroot/yacy/message.java
Unescape View File

@ -49,6 +49,7 @@
import java.io.File;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.UnsupportedEncodingException;
import java.text.SimpleDateFormat;
import java.util.Date;
import de.anomic.data.messageBoard;
@ -125,11 +126,17 @@ public final class message {
// save message
messageBoard.entry msgEntry = null;
byte[] mb;
try {
mb = message.getBytes("UTF-8");
} catch (UnsupportedEncodingException e) {
mb = message.getBytes();
}
sb.messageDB.write(msgEntry = sb.messageDB.newEntry(
"remote",
otherSeed.get(yacySeed.NAME, "anonymous"), otherSeed.hash,
yacyCore.seedDB.mySeed.getName(), yacyCore.seedDB.mySeed.hash,
subject, message.getBytes()));
subject, mb));
messageForwardingViaEmail(ss, msgEntry);

1
source/dbtest.java
Unescape View File

@ -24,6 +24,7 @@ public class dbtest {
public final static int keylength = 12;
public final static int valuelength = 223; // sum of all data length as defined in plasmaURL
//public final static long buffer = 0;
public final static long buffer = 8192 * 1024; // 8 MB buffer
public static byte[] dummyvalue1 = new byte[valuelength];
public static byte[] dummyvalue2 = new byte[valuelength];

8
source/de/anomic/data/wikiBoard.java
Unescape View File

@ -149,7 +149,7 @@ public class wikiBoard {
return author;
}
public entry newEntry(String subject, String author, String ip, String reason, byte[] page) {
public entry newEntry(String subject, String author, String ip, String reason, byte[] page) throws IOException {
return new entry(normalize(subject), author, ip, reason, page);
}
@ -158,17 +158,17 @@ public class wikiBoard {
String key;
Map record;
public entry(String subject, String author, String ip, String reason, byte[] page) {
public entry(String subject, String author, String ip, String reason, byte[] page) throws IOException {
record = new HashMap();
key = subject;
if (key.length() > keyLength) key = key.substring(0, keyLength);
record.put("date", dateString());
if ((author == null) || (author.length() == 0)) author = "anonymous";
record.put("author", kelondroBase64Order.enhancedCoder.encode(author.getBytes()));
record.put("author", kelondroBase64Order.enhancedCoder.encode(author.getBytes("UTF-8")));
if ((ip == null) || (ip.length() == 0)) ip = "";
record.put("ip", ip);
if ((reason == null) || (reason.length() == 0)) reason = "";
record.put("reason", kelondroBase64Order.enhancedCoder.encode(reason.getBytes()));
record.put("reason", kelondroBase64Order.enhancedCoder.encode(reason.getBytes("UTF-8")));
if (page == null)
record.put("page", "");
else

7
source/de/anomic/data/wikiCode.java
Unescape View File

@ -50,6 +50,7 @@ import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverCore;
@ -73,7 +74,11 @@ public class wikiCode {
}
public String transform(String content){
return transform(content.getBytes(), sb);
try {
return transform(content.getBytes("UTF-8"), sb);
} catch (UnsupportedEncodingException e) {
return transform(content.getBytes(), sb);
}
}
public String transform(byte[] content){
return transform(content, sb);

2
source/de/anomic/htmlFilter/htmlFilterAbstractScraper.java
View File

File diff suppressed because one or more lines are too long

2
source/de/anomic/htmlFilter/htmlFilterContentScraper.java
Unescape View File

@ -230,7 +230,7 @@ public class htmlFilterContentScraper extends htmlFilterAbstractScraper implemen
}
public byte[] getText() {
return content.getBytes();
return content.getBytes();
}
public Map getAnchors() {

8
source/de/anomic/htmlFilter/htmlFilterContentTransformer.java
Unescape View File

@ -46,6 +46,7 @@ package de.anomic.htmlFilter;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.UnsupportedEncodingException;
import java.text.Collator;
import java.util.ArrayList;
import java.util.Locale;
@ -117,7 +118,12 @@ public class htmlFilterContentTransformer extends htmlFilterAbstractTransformer
private boolean hit(byte[] text) {
if (text == null || bluelist == null) return false;
String lc = new String(text).toLowerCase();
String lc;
try {
lc = new String(text, "UTF-8").toLowerCase();
} catch (UnsupportedEncodingException e) {
lc = new String(text).toLowerCase();
}
for (int i = 0; i < bluelist.size(); i++) {
if (lc.indexOf((String) bluelist.get(i)) >= 0) return true;
}

2
source/de/anomic/http/httpd.java
Unescape View File

@ -710,7 +710,7 @@ public final class httpd implements serverHandler {
bout.close(); bout = null;
}
int argc = parseArgs(args, new String(buffer));
int argc = parseArgs(args, new String(buffer, "UTF-8"));
buffer = null;
return argc;
}

2
source/de/anomic/kelondro/kelondroAbstractRA.java
Unescape View File

@ -152,7 +152,7 @@ abstract class kelondroAbstractRA implements kelondroRA {
return new String(bb, 0, bbsize);
}
if (c == cr) continue;
if (c == lf) return new String(bb, 0, bbsize);
if (c == lf) return new String(bb, 0, bbsize, "UTF-8");
// append to bb
if (bbsize == bb.length) {

32
source/de/anomic/kelondro/kelondroArray.java
Unescape View File

@ -98,7 +98,6 @@ public class kelondroArray extends kelondroRecords {
return getNode(new Handle(index)).getValues();
}
public synchronized int seti(int index, int value) throws IOException {
int before = getHandle(index).hashCode();
setHandle(index, new Handle(value));
@ -109,13 +108,28 @@ public class kelondroArray extends kelondroRecords {
return getHandle(index).hashCode();
}
public synchronized int add(byte[][] row) throws IOException {
if (row.length != columns())
throw new IllegalArgumentException("add: wrong row length " + row.length + "; must be " + columns());
Node n = newNode();
n.commit(CP_LOW);
int index = n.handle().hashCode();
set(index, row);
return index;
}
public synchronized void remove(int index) throws IOException {
deleteNode(new Handle(index));
}
public void print() throws IOException {
System.out.println("PRINTOUT of table, length=" + size());
byte[][] row;
for (int i = 0; i < size(); i++) {
System.out.print("row " + i + ": ");
row = get(i);
for (int j = 0; j < columns(); j++) System.out.print(((row[j] == null) ? "NULL" : new String(row[j])) + ", ");
for (int j = 0; j < columns(); j++) System.out.print(((row[j] == null) ? "NULL" : new String(row[j], "UTF-8")) + ", ");
System.out.println();
}
System.out.println("EndOfTable");
@ -160,6 +174,20 @@ public class kelondroArray extends kelondroRecords {
fm.set(Integer.parseInt(args[2]), row);
fm.close();
} else
if ((args.length == 3) && (args[0].equals("-a"))) {
// add <filename> <value>
kelondroArray fm = new kelondroArray(new File(args[1]));
byte[][] row = new byte[][] { args[2].getBytes() };
int index = fm.add(row);
System.out.println("Added to row " + index);
fm.close();
} else
if ((args.length == 3) && (args[0].equals("-d"))) {
// delete <filename> <index>
kelondroArray fm = new kelondroArray(new File(args[1]));
fm.remove(Integer.parseInt(args[2]));
fm.close();
} else
if ((args.length == 1) && (args[0].equals("-test"))) {
File testfile = new File("test.array");
if (testfile.exists()) testfile.delete();

21
source/de/anomic/kelondro/kelondroBase64Order.java
Unescape View File

@ -52,6 +52,19 @@ public class kelondroBase64Order extends kelondroAbstractOrder implements kelond
private static final char[] alpha_standard = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/".toCharArray();
private static final char[] alpha_enhanced = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_".toCharArray();
private static final byte[] ahpla_standard = new byte[256];
private static final byte[] ahpla_enhanced = new byte[256];
static {
for (int i = 0; i < 256; i++) {
ahpla_standard[i] = -1;
ahpla_enhanced[i] = -1;
}
for (int i = 0; i < alpha_standard.length; i++) {
ahpla_standard[alpha_standard[i]] = (byte) i;
ahpla_enhanced[alpha_enhanced[i]] = (byte) i;
}
}
public static final kelondroBase64Order standardCoder = new kelondroBase64Order(true);
public static final kelondroBase64Order enhancedCoder = new kelondroBase64Order(false);
@ -59,15 +72,14 @@ public class kelondroBase64Order extends kelondroAbstractOrder implements kelond
final boolean rfc1113compliant;
private final char[] alpha;
private final byte[] ahpla = new byte[256];
private final byte[] ahpla;
public kelondroBase64Order(boolean rfc1113compliant) {
// if we choose not to be rfc1113compliant,
// then we get shorter base64 results which are also filename-compatible
this.rfc1113compliant = rfc1113compliant;
alpha = (rfc1113compliant) ? alpha_standard : alpha_enhanced;
for (int i = 0; i < 256; i++) ahpla[i] = -1;
for (int i = 0; i < alpha.length; i++) ahpla[alpha[i]] = (byte) i;
ahpla = (rfc1113compliant) ? ahpla_standard : ahpla_enhanced;
}
public char encodeByte(byte b) {
@ -143,7 +155,8 @@ public class kelondroBase64Order extends kelondroAbstractOrder implements kelond
public String decodeString(String in) {
try {
return new String(decode(in), "ISO-8859-1");
//return new String(decode(in), "ISO-8859-1");
return new String(decode(in), "UTF-8");
} catch (java.io.UnsupportedEncodingException e) {
System.out.println("internal error in base64: " + e.getMessage());
return null;

2
source/de/anomic/kelondro/kelondroHashtable.java
Unescape View File

@ -230,7 +230,7 @@ public class kelondroHashtable {
rowNumber = hash.node();
if (rowNumber >= hashArray.size()) return new Object[]{new Integer(rowNumber), null};
row = hashArray.get(rowNumber);
rowKey = (int) kelondroBase64Order.enhancedCoder.decodeLong(new String(row[0]));
rowKey = (int) kelondroBase64Order.enhancedCoder.decodeLong(new String(row[0], "UTF-8"));
if (rowKey == 0) return new Object[]{new Integer(rowNumber), null};
hash.rehash();
} while (rowKey != hash.key());

4
source/de/anomic/kelondro/kelondroNaturalOrder.java
Unescape View File

@ -68,6 +68,10 @@ public class kelondroNaturalOrder extends kelondroAbstractOrder implements kelon
// two arrays are also equal if one array is a subset of the other's array
// with filled-up char(0)-values
public int compare(byte[] a, byte[] b) {
return compares(a, b);
}
public static final int compares(byte[] a, byte[] b) {
int i = 0;
final int al = a.length;
final int bl = b.length;

2
source/de/anomic/kelondro/kelondroRecords.java
Unescape View File

@ -790,7 +790,7 @@ public class kelondroRecords {
if (h == null) s = s + ":hNULL"; else s = s + ":h" + h.toString();
}
byte[][] content = getValues();
for (int i = 0; i < content.length; i++) s = s + ":" + ((content[i] == null) ? "NULL" : (new String(content[i])).trim());
for (int i = 0; i < content.length; i++) s = s + ":" + ((content[i] == null) ? "NULL" : (new String(content[i], "UTF-8")).trim());
} catch (IOException e) {
s = s + ":***LOAD ERROR***:" + e.getMessage();
}

4
source/de/anomic/plasma/parser/odt/odtParser.java
Unescape View File

@ -130,11 +130,11 @@ public class odtParser extends AbstractParser implements Parser {
if (docShortTitle != null) {
docLongTitle = docShortTitle;
} else if (docContent.length <= 80) {
docLongTitle = new String(docContent);
docLongTitle = new String(docContent, "UTF-8");
} else {
byte[] title = new byte[80];
System.arraycopy(docContent, 0, title, 0, 80);
docLongTitle = new String(title);
docLongTitle = new String(title, "UTF-8");
}
docLongTitle.
replaceAll("\r\n"," ").

2
source/de/anomic/plasma/parser/pdf/pdfParser.java
Unescape View File

@ -127,7 +127,7 @@ public class pdfParser extends AbstractParser implements Parser {
out = null;
if ((docTitle == null) || (docTitle.length() == 0)) {
docTitle = ((contents.length > 80)? new String(contents, 0, 80):new String(contents)).
docTitle = ((contents.length > 80)? new String(contents, 0, 80, "UTF-8"):new String(contents, "UTF-8")).
replaceAll("\r\n"," ").
replaceAll("\n"," ").
replaceAll("\r"," ").

18
source/de/anomic/plasma/plasmaCrawlEURL.java
Unescape View File

@ -171,15 +171,15 @@ public class plasmaCrawlEURL extends plasmaURL {
this.hash = hash;
byte[][] entry = urlHashCache.get(hash.getBytes());
if (entry != null) {
this.referrer = new String(entry[1]);
this.initiator = new String(entry[2]);
this.executor = new String(entry[3]);
this.url = new URL(new String(entry[4]).trim());
this.name = new String(entry[5]).trim();
this.initdate = new Date(86400000 * kelondroBase64Order.enhancedCoder.decodeLong(new String(entry[6])));
this.trydate = new Date(86400000 * kelondroBase64Order.enhancedCoder.decodeLong(new String(entry[7])));
this.trycount = (int) kelondroBase64Order.enhancedCoder.decodeLong(new String(entry[8]));
this.failreason = new String(entry[9]);
this.referrer = new String(entry[1], "UTF-8");
this.initiator = new String(entry[2], "UTF-8");
this.executor = new String(entry[3], "UTF-8");
this.url = new URL(new String(entry[4], "UTF-8").trim());
this.name = new String(entry[5], "UTF-8").trim();
this.initdate = new Date(86400000 * kelondroBase64Order.enhancedCoder.decodeLong(new String(entry[6], "UTF-8")));
this.trydate = new Date(86400000 * kelondroBase64Order.enhancedCoder.decodeLong(new String(entry[7], "UTF-8")));
this.trycount = (int) kelondroBase64Order.enhancedCoder.decodeLong(new String(entry[8], "UTF-8"));
this.failreason = new String(entry[9], "UTF-8");
this.flags = new bitfield(entry[10]);
return;
}

22
source/de/anomic/plasma/plasmaCrawlLURL.java
Unescape View File

@ -475,18 +475,18 @@ public final class plasmaCrawlLURL extends plasmaURL {
if (entry == null) throw new IOException("url hash " + urlHash + " not found in LURL");
try {
if (entry != null) {
this.url = new URL(new String(entry[1]).trim());
this.descr = (entry[2] == null) ? this.url.toString() : new String(entry[2]).trim();
this.moddate = new Date(86400000 * kelondroBase64Order.enhancedCoder.decodeLong(new String(entry[3])));
this.loaddate = new Date(86400000 * kelondroBase64Order.enhancedCoder.decodeLong(new String(entry[4])));
this.referrerHash = (entry[5] == null) ? dummyHash : new String(entry[5]);
this.copyCount = (int) kelondroBase64Order.enhancedCoder.decodeLong(new String(entry[6]));
this.flags = new String(entry[7]);
this.quality = (int) kelondroBase64Order.enhancedCoder.decodeLong(new String(entry[8]));
this.language = new String(entry[9]);
this.url = new URL(new String(entry[1], "UTF-8").trim());
this.descr = (entry[2] == null) ? this.url.toString() : new String(entry[2], "UTF-8").trim();
this.moddate = new Date(86400000 * kelondroBase64Order.enhancedCoder.decodeLong(new String(entry[3], "UTF-8")));
this.loaddate = new Date(86400000 * kelondroBase64Order.enhancedCoder.decodeLong(new String(entry[4], "UTF-8")));
this.referrerHash = (entry[5] == null) ? dummyHash : new String(entry[5], "UTF-8");
this.copyCount = (int) kelondroBase64Order.enhancedCoder.decodeLong(new String(entry[6], "UTF-8"));
this.flags = new String(entry[7], "UTF-8");
this.quality = (int) kelondroBase64Order.enhancedCoder.decodeLong(new String(entry[8], "UTF-8"));
this.language = new String(entry[9], "UTF-8");
this.doctype = (char) entry[10][0];
this.size = kelondroBase64Order.enhancedCoder.decodeLong(new String(entry[11]));
this.wordCount = (int) kelondroBase64Order.enhancedCoder.decodeLong(new String(entry[12]));
this.size = kelondroBase64Order.enhancedCoder.decodeLong(new String(entry[11], "UTF-8"));
this.wordCount = (int) kelondroBase64Order.enhancedCoder.decodeLong(new String(entry[12], "UTF-8"));
this.snippet = null;
return;
}

14
source/de/anomic/plasma/plasmaCrawlNURL.java
Unescape View File

@ -189,43 +189,43 @@ public class plasmaCrawlNURL extends plasmaURL {
Iterator i;
try {
i = coreStack.iterator();
while (i.hasNext()) stackIndex.add(new String((byte[]) i.next()));
while (i.hasNext()) stackIndex.add(new String((byte[]) i.next(), "UTF-8"));
} catch (Exception e) {
coreStack.reset();
}
try {
i = limitStack.iterator();
while (i.hasNext()) stackIndex.add(new String((byte[]) i.next()));
while (i.hasNext()) stackIndex.add(new String((byte[]) i.next(), "UTF-8"));
} catch (Exception e) {
limitStack.reset();
}
try {
i = overhangStack.iterator();
while (i.hasNext()) stackIndex.add(new String((byte[]) i.next()));
while (i.hasNext()) stackIndex.add(new String((byte[]) i.next(), "UTF-8"));
} catch (Exception e) {
overhangStack.reset();
}
try {
i = remoteStack.iterator();
while (i.hasNext()) stackIndex.add(new String((byte[]) i.next()));
while (i.hasNext()) stackIndex.add(new String((byte[]) i.next(), "UTF-8"));
} catch (Exception e) {
remoteStack.reset();
}
try {
i = imageStack.iterator();
while (i.hasNext()) stackIndex.add(new String(((kelondroRecords.Node) i.next()).getKey()));
while (i.hasNext()) stackIndex.add(new String(((kelondroRecords.Node) i.next()).getKey(), "UTF-8"));
} catch (Exception e) {
imageStack = kelondroStack.reset(imageStack);
}
try {
i = movieStack.iterator();
while (i.hasNext()) stackIndex.add(new String(((kelondroRecords.Node) i.next()).getKey()));
while (i.hasNext()) stackIndex.add(new String(((kelondroRecords.Node) i.next()).getKey(), "UTF-8"));
} catch (Exception e) {
movieStack = kelondroStack.reset(movieStack);
}
try {
i = musicStack.iterator();
while (i.hasNext()) stackIndex.add(new String(((kelondroRecords.Node) i.next()).getKey()));
while (i.hasNext()) stackIndex.add(new String(((kelondroRecords.Node) i.next()).getKey(), "UTF-8"));
} catch (Exception e) {
musicStack = kelondroStack.reset(musicStack);
}

20
source/de/anomic/plasma/plasmaCrawlStacker.java
Unescape View File

@ -401,17 +401,17 @@ public final class plasmaCrawlStacker {
try {
this.urlHash = urlHash;
this.initiator = new String(entryBytes[1]);
this.url = new String(entryBytes[2]).trim();
this.referrerHash = (entryBytes[3]==null) ? plasmaURL.dummyHash : new String(entryBytes[3]);
this.name = (entryBytes[4] == null) ? "" : new String(entryBytes[4]).trim();
this.loaddate = new Date(86400000 * kelondroBase64Order.enhancedCoder.decodeLong(new String(entryBytes[5])));
this.profileHandle = (entryBytes[6] == null) ? null : new String(entryBytes[6]).trim();
this.depth = (int) kelondroBase64Order.enhancedCoder.decodeLong(new String(entryBytes[7]));
this.anchors = (int) kelondroBase64Order.enhancedCoder.decodeLong(new String(entryBytes[8]));
this.forkfactor = (int) kelondroBase64Order.enhancedCoder.decodeLong(new String(entryBytes[9]));
this.initiator = new String(entryBytes[1], "UTF-8");
this.url = new String(entryBytes[2], "UTF-8").trim();
this.referrerHash = (entryBytes[3]==null) ? plasmaURL.dummyHash : new String(entryBytes[3], "UTF-8");
this.name = (entryBytes[4] == null) ? "" : new String(entryBytes[4], "UTF-8").trim();
this.loaddate = new Date(86400000 * kelondroBase64Order.enhancedCoder.decodeLong(new String(entryBytes[5], "UTF-8")));
this.profileHandle = (entryBytes[6] == null) ? null : new String(entryBytes[6], "UTF-8").trim();
this.depth = (int) kelondroBase64Order.enhancedCoder.decodeLong(new String(entryBytes[7], "UTF-8"));
this.anchors = (int) kelondroBase64Order.enhancedCoder.decodeLong(new String(entryBytes[8], "UTF-8"));
this.forkfactor = (int) kelondroBase64Order.enhancedCoder.decodeLong(new String(entryBytes[9], "UTF-8"));
this.flags = new bitfield(entryBytes[10]);
this.handle = Integer.parseInt(new String(entryBytes[11]));
this.handle = Integer.parseInt(new String(entryBytes[11], "UTF-8"));
} catch (Exception e) {
e.printStackTrace();
throw new IllegalStateException();

16
source/de/anomic/plasma/plasmaSwitchboardQueue.java
Unescape View File

@ -201,21 +201,21 @@ public class plasmaSwitchboardQueue {
this.referrerURL = null;
}
public Entry(byte[][] row) {
long ims = (row[2] == null) ? 0 : kelondroBase64Order.enhancedCoder.decodeLong(new String(row[2]));
public Entry(byte[][] row) throws IOException {
long ims = (row[2] == null) ? 0 : kelondroBase64Order.enhancedCoder.decodeLong(new String(row[2], "UTF-8"));
byte flags = (row[3] == null) ? 0 : row[3][0];
try {
this.url = new URL(new String(row[0]));
this.url = new URL(new String(row[0], "UTF-8"));
} catch (MalformedURLException e) {
this.url = null;
}
this.referrerHash = (row[1] == null) ? null : new String(row[1]);
this.referrerHash = (row[1] == null) ? null : new String(row[1], "UTF-8");
this.ifModifiedSince = (ims == 0) ? null : new Date(ims);
this.flags = ((flags & 1) == 1) ? (byte) 1 : (byte) 0;
this.initiator = (row[4] == null) ? null : new String(row[4]);
this.depth = (int) kelondroBase64Order.enhancedCoder.decodeLong(new String(row[5]));
this.profileHandle = new String(row[6]);
this.anchorName = (row[7] == null) ? null : (new String(row[7])).trim();
this.initiator = (row[4] == null) ? null : new String(row[4], "UTF-8");
this.depth = (int) kelondroBase64Order.enhancedCoder.decodeLong(new String(row[5], "UTF-8"));
this.profileHandle = new String(row[6], "UTF-8");
this.anchorName = (row[7] == null) ? null : (new String(row[7], "UTF-8")).trim();
this.profileEntry = null;
this.responseHeader = null;

2
source/de/anomic/plasma/plasmaWordConnotation.java
Unescape View File

@ -73,7 +73,7 @@ public class plasmaWordConnotation {
//reference = reference.toLowerCase();
byte[][] record = refDB.get(word, reference.getBytes());
long c;
if (record == null) c = 0; else c = kelondroBase64Order.enhancedCoder.decodeLong(new String(record[1]));
if (record == null) c = 0; else c = kelondroBase64Order.enhancedCoder.decodeLong(new String(record[1], "UTF-8"));
record[1] = kelondroBase64Order.enhancedCoder.encodeLong(c++, countlength).getBytes();
refDB.put(word, record);
}

3
source/de/anomic/plasma/plasmaWordIndexAssortment.java
Unescape View File

@ -200,8 +200,7 @@ public final class plasmaWordIndexAssortment {
for (int i = 0; i < assortmentLength; i++) {
container.add(
new plasmaWordIndexEntry[] { new plasmaWordIndexEntry(
new String(row[3 + 2 * i]), new String(
row[4 + 2 * i])) }, updateTime);
new String(row[3 + 2 * i]), new String(row[4 + 2 * i])) }, updateTime);
}
return container;
}

4
source/de/anomic/plasma/plasmaWordIndexCache.java
Unescape View File

@ -193,9 +193,9 @@ public final class plasmaWordIndexCache implements plasmaWordIndexInterface {
// get out one entry
row = dumpArray.get(i);
if ((row[0] == null) || (row[1] == null) || (row[2] == null) || (row[3] == null) || (row[4] == null)) continue;
wordHash = new String(row[0]);
wordHash = new String(row[0], "UTF-8");
creationTime = kelondroRecords.bytes2long(row[2]);
wordEntry = new plasmaWordIndexEntry(new String(row[3]), new String(row[4]));
wordEntry = new plasmaWordIndexEntry(new String(row[3], "UTF-8"), new String(row[4], "UTF-8"));
// store to cache
addEntry(wordHash, wordEntry, creationTime);
urlCount++;

2
source/de/anomic/server/serverFileUtils.java
Unescape View File

@ -241,7 +241,7 @@ public final class serverFileUtils {
public static Set loadSet(File file, String sep, boolean tree) throws IOException {
Set set = (tree) ? (Set) new TreeSet() : (Set) new HashSet();
byte[] b = read(file);
StringTokenizer st = new StringTokenizer(new String(b), sep);
StringTokenizer st = new StringTokenizer(new String(b, "UTF-8"), sep);
while (st.hasMoreTokens()) {
set.add(st.nextToken());
}

2
source/de/anomic/soap/httpdSoapHandler.java
Unescape View File

@ -468,7 +468,7 @@ public final class httpdSoapHandler extends httpdAbstractHandler implements http
if (templates[i].endsWith(".template")) try {
//System.out.println("TEMPLATE " + templates[i].substring(0, templates[i].length() - 9) + ": " + new String(buf, 0, c));
result.put(templates[i].substring(0, templates[i].length() - 9),
new String(serverFileUtils.read(new File(path, templates[i]))));
new String(serverFileUtils.read(new File(path, templates[i])), "UTF-8"));
} catch (Exception e) {}
}
return result;

2
source/de/anomic/soap/httpdSoapService.java
Unescape View File

@ -232,7 +232,7 @@ public class httpdSoapService
// convert it into a byte array and send it back as result
byte[] result = o.toByteArray();
return new String(result);
return new String(result, "UTF-8");
} catch (Exception e) {
throw new AxisFault(e.getMessage());
}

2
source/de/anomic/tools/cryptbig.java
Unescape View File

@ -164,7 +164,7 @@ public class cryptbig {
if (b64dec == null) return null; // error in input string (inconsistency)
byte[] dec = decryptArray(b64dec);
if (dec == null) return null;
return new String(dec, "UTF8");
return new String(dec, "UTF-8");
} catch (UnsupportedEncodingException e) {
}
return null;

2
source/de/anomic/tools/gzip.java
Unescape View File

@ -105,7 +105,7 @@ public class gzip {
copy(fout, fin, 128);
fin.close();
fout.close();
return new String(fout.toByteArray(), "UTF8");
return new String(fout.toByteArray(), "UTF-8");
} catch (IOException e) {
System.err.println("ERROR: IO trouble");
return null;

8
source/de/anomic/yacy/yacyClient.java
Unescape View File

@ -44,6 +44,7 @@
package de.anomic.yacy;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Date;
@ -550,8 +551,11 @@ public final class yacyClient {
post.put("youare", targetHash);
post.put("subject", subject);
post.put(yacySeed.MYTIME, yacyCore.universalDateShortString(new Date()));
post.put("message", new String(message));
try {
post.put("message", new String(message, "UTF-8"));
} catch (UnsupportedEncodingException e) {
post.put("message", new String(message));
}
// get target address
String address = targetAddress(targetHash);

2
source/yacy.java
Unescape View File

@ -1165,7 +1165,7 @@ public final class yacy {
entry = (plasmaCrawlLURL.Entry) eiter.next();
if ((entry != null) && (entry.url() != null)) {
if (html) {
bos.write(("<a href=\"" + entry.url() + "\">" + entry.descr() + "</a><br>").getBytes());
bos.write(("<a href=\"" + entry.url() + "\">" + entry.descr() + "</a><br>").getBytes("UTF-8"));
bos.write(serverCore.crlf);
} else {
bos.write(entry.url().toString().getBytes());

Write Preview
Loading…
Cancel
Save