- replaced more direct string-based UTF-8 conversions by predefined UTF-8 conversion

- changed menu structure slightly

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7583 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 14 years ago
parent bbb7aea8f3
commit 694fa3a2a5

@ -30,7 +30,6 @@
// javac -classpath .:../classes Blog.java
// if the shell's current path is HTROOT
import java.io.UnsupportedEncodingException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.HashMap;
@ -119,11 +118,7 @@ public class Blog {
}
byte[] author;
try {
author = StrAuthor.getBytes("UTF-8");
} catch (final UnsupportedEncodingException e) {
author = StrAuthor.getBytes();
}
author = UTF8.getBytes(StrAuthor);
if (hasRights && post.containsKey("delete") && "sure".equals(post.get("delete"))) {
page = sb.blogDB.readBlogEntry(pagename);
@ -141,11 +136,7 @@ public class Blog {
if (post.containsKey("submit") && hasRights) {
// store a new/edited blog-entry
byte[] content;
try {
content = post.get("content", "").getBytes("UTF-8");
} catch (final UnsupportedEncodingException e) {
content = post.get("content", "").getBytes();
}
content = UTF8.getBytes(post.get("content", ""));
final Date date;
List<String> comments = null;
@ -162,11 +153,7 @@ public class Blog {
final String commentMode = post.get("commentMode", "2");
final String StrSubject = post.get("subject", "");
byte[] subject;
try {
subject = StrSubject.getBytes("UTF-8");
} catch (final UnsupportedEncodingException e) {
subject = StrSubject.getBytes();
}
subject = UTF8.getBytes(StrSubject);
sb.blogDB.writeBlogEntry(sb.blogDB.newEntry(pagename, subject, author, ip, date, content, comments, commentMode));

@ -32,7 +32,6 @@
import java.io.File;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.UnsupportedEncodingException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.Iterator;
@ -104,11 +103,7 @@ public class BlogComments {
}
byte[] author;
try {
author = StrAuthor.getBytes("UTF-8");
} catch (final UnsupportedEncodingException e) {
author = StrAuthor.getBytes();
}
author = UTF8.getBytes(StrAuthor);
final BlogBoard.BlogEntry page = sb.blogDB.readBlogEntry(pagename); //maybe "if(page == null)"
final boolean pageExists = sb.blogDB.contains(pagename);
@ -123,22 +118,14 @@ public class BlogComments {
if ("".equals(post.get("subject", ""))) {
post.putHTML("subject", "no title");
}
try {
content = post.get("content", "").getBytes("UTF-8");
} catch (final UnsupportedEncodingException e) {
content = post.get("content", "").getBytes();
}
content = UTF8.getBytes(post.get("content", ""));
final Date date = null;
//set name for new entry or date for old entry
final String StrSubject = post.get("subject", "");
byte[] subject;
try {
subject = StrSubject.getBytes("UTF-8");
} catch (final UnsupportedEncodingException e) {
subject = StrSubject.getBytes();
}
subject = UTF8.getBytes(StrSubject);
final String commentID = String.valueOf(System.currentTimeMillis());
final BlogEntry blogEntry = sb.blogDB.readBlogEntry(pagename);
blogEntry.addComment(commentID);

@ -5,7 +5,7 @@
<title>YaCy '#[clientname]#': Integration of a Search Portal</title>
#%env/templates/metas.template%#
</head>
<body id="ConfigSkins">
<body id="ConfigSkins" onload="document.getElementById('promoteSearchPageGreeting').focus()">
#%env/templates/header.template%#
#%env/templates/submenuPortalIntegration.template%#
<h2>Integration of a Search Portal</h2>
@ -15,11 +15,11 @@
and a link to a home page that is reached when the 'corporate identity'-images are clicked.
To change also colours and styles use the <a href="ConfigAppearance_p.html">Appearance Servlet</a> for different skins and languages.
</p>
<form action="ConfigPortal.html" accept-charset="UTF-8">
<form action="ConfigPortal.html" id="ConfigPortal" accept-charset="UTF-8">
<fieldset>
<dl>
<dt>Greeting Line</dt>
<dd><input type="text" name="promoteSearchPageGreeting" value="#[promoteSearchPageGreeting]#" size="60" /></dd>
<dd><input type="text" name="promoteSearchPageGreeting" id="promoteSearchPageGreeting" value="#[promoteSearchPageGreeting]#" size="60" /></dd>
<dt>URL of Home Page</dt>
<dd><input type="text" name="promoteSearchPageGreeting.homepage" value="#[promoteSearchPageGreeting.homepage]#" size="60" /></dd>
@ -125,7 +125,8 @@
height="180"
frameborder="0"
scrolling="auto"
id="target2">
id="target2"
onfocus="removeFocus(this);">
</iframe>
<h3>
@ -148,7 +149,8 @@
height="180"
frameborder="0"
scrolling="auto"
id="target3">
id="target3"
onfocus="removeFocus(this);">
</iframe>
#%env/templates/footer.template%#

@ -0,0 +1,108 @@
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<title>YaCy '#[clientname]#': Index Cleaner</title>
#%env/templates/metas.template%#
</head>
<body id="IndexControlCleaner">
<div id="fullcontent">
#(inline)##%env/templates/header.template%#
<h2>Steering of API Actions</h2>
<p>This table shows search results that had been sorted out from the search result display because their content had not been verified.
This means that the searched word does not appear on the search page.
</p>::#(/inline)#
#(showtable)#::
<form action="IndexControlCleaner_p.html" method="post" enctype="multipart/form-data" accept-charset="UTF-8" id="cleanerlist">
<fieldset>
<legend><label for="apilist">Recorded Actions</label></legend>
<p><span id="resCounter" style="display: inline;">
#(navigation)#
::
#(left)#<img src="env/grafics/navdl.gif" alt="no previous page" />::<a href="Table_API_p.html?startRecord=#[startRecord]#&amp;maximumRecords=#[maximumRecords]#&amp;inline=#(inline)#false::true#(/inline)#&amp;filter=#[filter]#" target="_self"><img src="env/grafics/navsl.gif" alt="previous page" /></a>#(/left)#
#[startRecord]#-#[to]# of #[of]#
#(right)#<img src="env/grafics/navdr.gif" alt="no next page" />::<a href="Table_API_p.html?startRecord=#[startRecord]#&amp;maximumRecords=#[maximumRecords]#&amp;inline=#(inline)#false::true#(/inline)#&amp;filter=#[filter]#" target="_self"><img src="env/grafics/navsr.gif" alt="next page" /></a>#(/right)#
<img src="env/grafics/nave.gif" alt="" />
#(/navigation)#
<div>
<input type="hidden" name="startRecord" value="#[startRecord]#" />
<input type="hidden" name="maximumRecords" value="#[maximumRecords]#" />
<input type="hidden" name="inline" value="#(inline)#false::true#(/inline)#" />
<input type="hidden" name="filter" value="#[filter]#" />
<input type="text" name="query" value="#[query]#" style="font-size:16px;float:left;border:0px;height:20px;background-image:url('env/grafics/find.gif');background-repeat:no-repeat;background-position:right top;" />
</div>
</span><br/></p>
<p style="clear:both;">
<table class="sortable" border="0" cellpadding="2" cellspacing="1">
<tr class="TableHeader" valign="bottom">
<th class="sorttable_nosort"><input type="checkbox" id="allswitch" onclick="checkAll(this.form.id, this.checked);" /></th>
<th>Type</th>
<th width="100">Comment</th>
<th>Call<br/>Count</th>
<th>Recording<br/>Date</th>
<th>Last&nbsp;Exec<br/>Date</th>
<th>Next&nbsp;Exec<br/>Date</th>
<th class="sorttable_nosort">Scheduler</th>
#(inline)#<th class="sorttable_nosort">URL</th>::#(/inline)#
</tr>
#{list}#
<tr class="TableCell#(dark)#Light::Dark::Summary#(/dark)#">
<td align="left"><input type="checkbox" name="item_#[count]#" value="mark_#[pk]#" /></td>
<td>#[type]#</td>
<td>#[comment]#</td>
<td>#[callcount]#</td>
<td>#[dateRecording]#</td>
<td>#[dateLastExec]#</td>
<td>#[dateNextExec]#</td>
<td>
#(scheduler)#
<form action="Table_API_p.html" method="post" enctype="multipart/form-data" id="modify_repeat" accept-charset="UTF-8">
<select name="repeat_select" onchange='this.form.submit()'>
<option value="off" selected="selected">no repetition</option>
<option value="on">activate scheduler</option>
</select>
<input type="hidden" name="pk" value="#[pk]#" />
<input type="hidden" name="inline" value="#[inline]#" />
<input type="hidden" name="filter" value="#[filter]#" />
</form>
::
<form action="Table_API_p.html" method="post" enctype="multipart/form-data" id="modify_repeat">
<table><tr><td>
<select name="repeat_time" onchange='this.form.submit()'>
#{scale}#
<option value="#[time]#" #(selected)#::selected="selected"#(/selected)#>#[time]#</option>
#{/scale}#
</select>
</td><td>
<select name="repeat_unit" onchange='this.form.submit()'>
<option value="selminutes" #(selectedMinutes)#::selected="selected"#(/selectedMinutes)#>minutes</option>
<option value="selhours" #(selectedHours)#::selected="selected"#(/selectedHours)#>hours</option>
<option value="seldays" #(selectedDays)#::selected="selected"#(/selectedDays)#>days</option>
</select>
</td></tr></table>
<input type="hidden" name="pk" value="#[pk]#" />
<input type="hidden" name="inline" value="#[inline]#" />
<input type="hidden" name="filter" value="#[filter]#" />
<noscript><input type="submit" value="Submit" /></noscript>
</form>
#(/scheduler)#
</td>
#(inline)#<td>#[url]#</td>::#(/inline)#
</tr>
#{/list}#
</fieldset>
</table>
</p>
<p>
<input type="hidden" name="num" value="#[num]#" />
<input type="submit" name="deletefromindex" value="Delete selected references from the search index" />
<input type="submit" name="deleterows" value="Delete selected entries from the list" />
</p>
</form>
#(/showtable)#
</div>
#%env/templates/footer.template%#
</body>
</html>

@ -25,12 +25,12 @@
//javac -classpath .:../Classes MessageSend_p.java
//if the shell's current path is HTROOT
import java.io.UnsupportedEncodingException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.Locale;
import java.util.Map;
import net.yacy.cora.document.UTF8;
import net.yacy.cora.protocol.RequestHeader;
import de.anomic.search.Switchboard;
@ -128,11 +128,7 @@ public class MessageSend_p {
if (subject.length() > 100) subject = subject.substring(0, 100);
if (message.length() > messagesize) message = message.substring(0, messagesize);
byte[] mb;
try {
mb = message.getBytes("UTF-8");
} catch (final UnsupportedEncodingException e) {
mb = message.getBytes();
}
mb = UTF8.getBytes(message);
final Map<String, String> result = yacyClient.postMessage(sb.peers, hash, subject, mb);
//message has been sent

@ -25,7 +25,6 @@
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
import java.io.IOException;
import java.net.MalformedURLException;
import java.util.Date;
import java.util.HashMap;
@ -218,30 +217,30 @@ public class Supporter {
entry = null;
if ((record.category().equals(yacyNewsPool.CATEGORY_PROFILE_UPDATE)) &&
((seed = sb.peers.getConnected(record.originator())) != null)) try {
((seed = sb.peers.getConnected(record.originator())) != null)) {
url = record.attribute("homepage", "");
if (url.length() < 12) continue;
entry = rowdef.newEntry(new byte[][]{
url.getBytes(),
url.getBytes(),
("Home Page of " + seed.getName()).getBytes("UTF-8"),
UTF8.getBytes(("Home Page of " + seed.getName())),
record.id().getBytes()
});
score = 1 + timeFactor(record.created());
} catch (final IOException e) {}
}
if ((record.category().equals(yacyNewsPool.CATEGORY_PROFILE_BROADCAST)) &&
((seed = sb.peers.getConnected(record.originator())) != null)) try {
((seed = sb.peers.getConnected(record.originator())) != null)) {
url = record.attribute("homepage", "");
if (url.length() < 12) continue;
entry = rowdef.newEntry(new byte[][]{
url.getBytes(),
url.getBytes(),
("Home Page of " + seed.getName()).getBytes("UTF-8"),
UTF8.getBytes(("Home Page of " + seed.getName())),
record.id().getBytes()
});
score = 1 + timeFactor(record.created());
} catch (final IOException e) {}
}
// add/subtract votes and write record
if (entry != null) {

@ -25,7 +25,6 @@
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
import java.io.IOException;
import java.net.MalformedURLException;
import java.util.Date;
import java.util.HashMap;
@ -225,85 +224,85 @@ public class Surftips {
if (record == null) continue;
entry = null;
if (record.category().equals(yacyNewsPool.CATEGORY_CRAWL_START)) try {
if (record.category().equals(yacyNewsPool.CATEGORY_CRAWL_START)) {
final String intention = record.attribute("intention", "");
url = record.attribute("startURL", "");
if (url.length() < 12) continue;
entry = rowdef.newEntry(new byte[][]{
url.getBytes(),
(((intention.length() == 0) || intention.equals("simple web crawl") || intention.equals("Automatic ReCrawl!")) ? record.attribute("startURL", "") : intention).getBytes(),
intention.equals("Automatic ReCrawl!") ? ("Automatic ReCrawl").getBytes("UTF-8") : ("Crawl Start Point").getBytes("UTF-8"),
intention.equals("Automatic ReCrawl!") ? UTF8.getBytes("Automatic ReCrawl") : UTF8.getBytes("Crawl Start Point"),
record.id().getBytes()
});
score = 2 + Math.min(10, intention.length() / 4) + timeFactor(record.created());
} catch (final IOException e) {}
}
if (record.category().equals(yacyNewsPool.CATEGORY_BOOKMARK_ADD)) try {
if (record.category().equals(yacyNewsPool.CATEGORY_BOOKMARK_ADD)) {
url = record.attribute("url", "");
if (url.length() < 12) continue;
entry = rowdef.newEntry(new byte[][]{
url.getBytes(),
(record.attribute("title", "")).getBytes("UTF-8"),
("Bookmark: " + record.attribute("description", "")).getBytes("UTF-8"),
UTF8.getBytes(record.attribute("title", "")),
UTF8.getBytes("Bookmark: " + record.attribute("description", "")),
record.id().getBytes()
});
score = 8 + timeFactor(record.created());
} catch (final IOException e) {}
}
if (record.category().equals(yacyNewsPool.CATEGORY_SURFTIPP_ADD)) try {
if (record.category().equals(yacyNewsPool.CATEGORY_SURFTIPP_ADD)) {
url = record.attribute("url", "");
if (url.length() < 12) continue;
entry = rowdef.newEntry(new byte[][]{
url.getBytes(),
(record.attribute("title", "")).getBytes("UTF-8"),
("Surf Tipp: " + record.attribute("description", "")).getBytes("UTF-8"),
UTF8.getBytes(record.attribute("title", "")),
UTF8.getBytes("Surf Tipp: " + record.attribute("description", "")),
record.id().getBytes()
});
score = 5 + timeFactor(record.created());
} catch (final IOException e) {}
}
if (record.category().equals(yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD)) try {
if (record.category().equals(yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD)) {
if (!(record.attribute("vote", "negative").equals("positive"))) continue;
url = record.attribute("url", "");
if (url.length() < 12) continue;
entry = rowdef.newEntry(new byte[][]{
url.getBytes(),
record.attribute("title", "").getBytes("UTF-8"),
record.attribute("description", "").getBytes("UTF-8"),
record.attribute("refid", "").getBytes()
UTF8.getBytes(record.attribute("title", "")),
UTF8.getBytes(record.attribute("description", "")),
UTF8.getBytes(record.attribute("refid", ""))
});
score = 5 + timeFactor(record.created());
} catch (final IOException e) {}
}
if (record.category().equals(yacyNewsPool.CATEGORY_WIKI_UPDATE)) try {
if (record.category().equals(yacyNewsPool.CATEGORY_WIKI_UPDATE)) {
yacySeed seed = sb.peers.getConnected(record.originator());
if (seed == null) seed = sb.peers.getDisconnected(record.originator());
if (seed != null) {
url = "http://" + seed.getPublicAddress() + "/Wiki.html?page=" + record.attribute("page", "");
entry = rowdef.newEntry(new byte[][]{
url.getBytes(),
(record.attribute("author", "Anonymous") + ": " + record.attribute("page", "")).getBytes("UTF-8"),
("Wiki Update: " + record.attribute("description", "")).getBytes("UTF-8"),
record.id().getBytes()
UTF8.getBytes(url),
UTF8.getBytes(record.attribute("author", "Anonymous") + ": " + record.attribute("page", "")),
UTF8.getBytes("Wiki Update: " + record.attribute("description", "")),
UTF8.getBytes(record.id())
});
score = 4 + timeFactor(record.created());
}
} catch (final IOException e) {}
}
if (record.category().equals(yacyNewsPool.CATEGORY_BLOG_ADD)) try {
if (record.category().equals(yacyNewsPool.CATEGORY_BLOG_ADD)) {
yacySeed seed = sb.peers.getConnected(record.originator());
if (seed == null) seed = sb.peers.getDisconnected(record.originator());
if (seed != null) {
url = "http://" + seed.getPublicAddress() + "/Blog.html?page=" + record.attribute("page", "");
entry = rowdef.newEntry(new byte[][]{
url.getBytes(),
(record.attribute("author", "Anonymous") + ": " + record.attribute("page", "")).getBytes("UTF-8"),
("Blog Entry: " + record.attribute("subject", "")).getBytes("UTF-8"),
record.id().getBytes()
UTF8.getBytes(url),
UTF8.getBytes(record.attribute("author", "Anonymous") + ": " + record.attribute("page", "")),
UTF8.getBytes("Blog Entry: " + record.attribute("subject", "")),
UTF8.getBytes(record.id())
});
score = 4 + timeFactor(record.created());
}
} catch (final IOException e) {}
}
// add/subtract votes and write record
if (entry != null) {

@ -288,11 +288,7 @@ public class ViewFile {
for (final StringBuilder s: sentences) {
sentence = s.toString();
Enumeration<String> tokens = null;
try {
tokens = new WordTokenizer(new ByteArrayInputStream(sentence.getBytes("UTF-8")), LibraryProvider.dymLib);
} catch (UnsupportedEncodingException e) {
e.printStackTrace();
}
tokens = new WordTokenizer(new ByteArrayInputStream(UTF8.getBytes(sentence)), LibraryProvider.dymLib);
while (tokens.hasMoreElements()) {
token = tokens.nextElement();
if (token.length() > 0) {

@ -31,7 +31,6 @@
// if the shell's current path is HTROOT
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.HashMap;
@ -114,11 +113,7 @@ public class Wiki {
// store a new page
byte[] content;
try {
content = post.get("content", "").getBytes("UTF-8");
} catch (final UnsupportedEncodingException e) {
content = post.get("content", "").getBytes();
}
content = UTF8.getBytes(post.get("content", ""));
final WikiBoard.Entry newEntry = sb.wikiDB.newEntry(pagename, author, ip, post.get("reason", "edit"), content);
sb.wikiDB.write(newEntry);
// create a news message

@ -1,8 +1,8 @@
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.util.HashMap;
import net.yacy.cora.document.UTF8;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.document.Document;
import net.yacy.document.Parser.Failure;
@ -42,14 +42,7 @@ public class import_ymark {
}
if(post.containsKey("bmkfile") && post.containsKey("importer")){
try {
byteIn = new ByteArrayInputStream(post.get("bmkfile$file").getBytes("UTF-8"));
} catch (UnsupportedEncodingException e) {
//TODO: display an error message
Log.logException(e);
prop.put("result", "0");
return prop;
}
byteIn = new ByteArrayInputStream(UTF8.getBytes(post.get("bmkfile$file")));
if(post.get("importer").equals("html") && byteIn != null) {
final YMarksHTMLImporter htmlImporter = new YMarksHTMLImporter(byteIn, 100);
t = new Thread(htmlImporter, "YMarks - HTML Importer");

@ -46,15 +46,22 @@
<ul class="menu">
<li><a href="/index.html" accesskey="s" class="MenuItemLink">Search Page</a></li>
<li><a href="/yacyinteractive.html" class="MenuItemLink">Interactive local Search</a></li>
<li><a href="/yacy/ui/" accesskey="s" class="MenuItemLink">Rich Client Search</a></li>
<!--<li><a href="/yacy/ui/" accesskey="s" class="MenuItemLink">Rich Client Search</a></li>-->
<li><a href="/compare_yacy.html?display=1" class="MenuItemLink">Compare Search</a></li>
<li><a href="/Ranking_p.html" class="MenuItemLink lock">Ranking Config</a></li>
<li><a href="/Surftips.html?display=1" class="MenuItemLink">Surftips</a></li>
<li><a href="/Wiki.html?display=1" class="MenuItemLink">Publication</a></li>
<li><a href="/Bookmarks.html" class="MenuItemLink">Bookmarks</a></li>
<li><a href="/Help.html" class="MenuItemLink">Help</a></li>
</ul>
</li>
<li class="menugroup" id="menugroupIntegration">
<h3>Search Integration</h3>
<ul class="menu">
<li><a href="/ConfigPortal.html" class="MenuItemLink">Search Portals</a></li>
<li><a href="/ConfigAppearance_p.html" class="MenuItemLink lock">Customization</a></li>
<li><a href="/Ranking_p.html" class="MenuItemLink lock">Ranking Config</a></li>
</ul>
</li>
<li class="menugroup" id="menugroupCrawlerControl">
<h3>Index&nbsp;Control</h3>
<ul class="menu">
@ -65,13 +72,6 @@
<li><a href="/Blacklist_p.html" class="MenuItemLink lock">Filter &amp; Blacklists</a></li>
</ul>
</li>
<li class="menugroup" id="menugroupIntegration">
<h3>Search Integration</h3>
<ul class="menu">
<li><a href="/ConfigPortal.html" class="MenuItemLink">Search Portals</a></li>
<li><a href="/ConfigAppearance_p.html" class="MenuItemLink lock">Customization</a></li>
</ul>
</li>
<li class="menugroup" id="menugroupMonitoring">
<h3>Monitoring</h3>
<ul class="menu">

@ -48,6 +48,7 @@ import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.yacy.yacySeed;
import net.yacy.cora.document.MultiProtocolURI;
import net.yacy.cora.document.UTF8;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.document.parser.html.CharacterCoding;
import net.yacy.kelondro.data.meta.DigestURI;
@ -196,11 +197,7 @@ public class sharedBlacklist_p {
final String fileString = post.get("file$file");
if (fileString != null) {
try {
otherBlacklist = FileUtils.strings(fileString.getBytes("UTF-8"));
} catch (IOException ex) {
prop.put("status", STATUS_FILE_ERROR);
}
otherBlacklist = FileUtils.strings(UTF8.getBytes(fileString));
}
}
} else if (post.containsKey("add")) {

@ -30,7 +30,6 @@
import java.io.File;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.UnsupportedEncodingException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.Locale;
@ -135,11 +134,7 @@ public final class message {
// save message
MessageBoard.entry msgEntry = null;
byte[] mb;
try {
mb = message.getBytes("UTF-8");
} catch (final UnsupportedEncodingException e) {
mb = message.getBytes();
}
mb = UTF8.getBytes(message);
sb.messageDB.write(msgEntry = sb.messageDB.newEntry(
"remote",
otherSeed.get(yacySeed.NAME, "anonymous"), otherSeed.hash,

@ -27,9 +27,9 @@
package de.anomic.crawler.retrieval;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.util.Date;
import net.yacy.cora.document.UTF8;
import net.yacy.kelondro.data.meta.DigestURI;
import net.yacy.kelondro.data.word.Word;
import net.yacy.kelondro.index.Row;
@ -179,11 +179,7 @@ public class Request extends WorkflowJob {
final byte[] sizestr = NaturalOrder.encodeLong(this.size, rowdef.width(14));
// store the hash in the hash cache
byte[] namebytes;
try {
namebytes = this.name.getBytes("UTF-8");
} catch (final UnsupportedEncodingException e) {
namebytes = this.name.getBytes();
}
namebytes = UTF8.getBytes(this.name);
final byte[][] entry = new byte[][] {
this.url.hash(),
initiator,

@ -31,7 +31,6 @@ package de.anomic.data;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Comparator;
@ -162,7 +161,7 @@ public class BlogBoard {
final DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
try {
final DocumentBuilder builder = factory.newDocumentBuilder();
return parseXMLimport(builder.parse(new ByteArrayInputStream(input.getBytes("UTF-8"))));
return parseXMLimport(builder.parse(new ByteArrayInputStream(UTF8.getBytes(input))));
} catch (final ParserConfigurationException ex) {
Log.logException(ex);
} catch (final SAXException ex) {
@ -220,24 +219,9 @@ public class BlogBoard {
}
byte[] subject,author,page;
try {
subject = StrSubject.getBytes("UTF-8");
} catch (final UnsupportedEncodingException e1) {
subject = StrSubject.getBytes();
}
try {
author = StrAuthor.getBytes("UTF-8");
} catch (final UnsupportedEncodingException e1) {
author = StrAuthor.getBytes();
}
try {
page = StrPage.getBytes("UTF-8");
} catch (final UnsupportedEncodingException e1) {
page = StrPage.getBytes();
}
subject = UTF8.getBytes(StrSubject);
author = UTF8.getBytes(StrAuthor);
page = UTF8.getBytes(StrPage);
writeBlogEntry (newEntry(key, subject, author, ip, date, page, null, null));
}
return true;

@ -31,7 +31,6 @@ package de.anomic.data;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
@ -209,25 +208,9 @@ public class BlogBoardComments {
return false;
byte[] subject,author,page;
try {
subject = StrSubject.getBytes("UTF-8");
}
catch (final UnsupportedEncodingException ex) {
subject = StrSubject.getBytes();
}
try {
author = StrAuthor.getBytes("UTF-8");
}
catch (final UnsupportedEncodingException ex) {
author = StrAuthor.getBytes();
}
try {
page = StrPage.getBytes("UTF-8");
}
catch (final UnsupportedEncodingException ex) {
page = StrPage.getBytes();
}
subject = UTF8.getBytes(StrSubject);
author = UTF8.getBytes(StrAuthor);
page = UTF8.getBytes(StrPage);
write (newEntry(key, subject, author, ip, date, page));
}

@ -116,7 +116,7 @@ public class BookmarkHelper {
public static int importFromBookmarks(BookmarksDB db, final DigestURI baseURL, final String input, final String tag, final boolean importPublic){
try {
// convert string to input stream
final ByteArrayInputStream byteIn = new ByteArrayInputStream(input.getBytes("UTF-8"));
final ByteArrayInputStream byteIn = new ByteArrayInputStream(UTF8.getBytes(input));
final InputStreamReader reader = new InputStreamReader(byteIn,"UTF-8");
// import stream
@ -165,15 +165,11 @@ public class BookmarkHelper {
public static int importFromXML(BookmarksDB db, final String input, final boolean importPublic){
try {
// convert string to input stream
final ByteArrayInputStream byteIn = new ByteArrayInputStream(input.getBytes("UTF-8"));
// import stream
return importFromXML(db, byteIn,importPublic);
} catch (final UnsupportedEncodingException e) {
return 0;
}
// convert string to input stream
final ByteArrayInputStream byteIn = new ByteArrayInputStream(UTF8.getBytes(input));
// import stream
return importFromXML(db, byteIn,importPublic);
}
private static int importFromXML(BookmarksDB db, final InputStream input, final boolean importPublic){

@ -468,7 +468,7 @@ public class YMarkTables {
buffer.append(document.dc_title());
buffer.append(document.dc_description());
buffer.append(document.dc_subject(' '));
final Enumeration<String> tokens = new WordTokenizer(new ByteArrayInputStream(buffer.toString().getBytes("UTF-8")), LibraryProvider.dymLib);
final Enumeration<String> tokens = new WordTokenizer(new ByteArrayInputStream(UTF8.getBytes(buffer.toString())), LibraryProvider.dymLib);
while(tokens.hasMoreElements()) {
int max = 1;
String token = tokens.nextElement();

@ -29,6 +29,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import net.yacy.cora.document.UTF8;
import net.yacy.kelondro.util.ByteBuffer;
import net.yacy.kelondro.util.FileUtils;
@ -60,7 +61,7 @@ public final class ChunkedOutputStream extends FilterOutputStream {
if (this.finished) throw new IOException("ChunkedOutputStream already finalized.");
if (b.length == 0) return;
this.out.write(Integer.toHexString(b.length).getBytes());
this.out.write(UTF8.getBytes(Integer.toHexString(b.length)));
this.out.write(serverCore.CRLF);
this.out.write(b);
this.out.write(serverCore.CRLF);
@ -71,7 +72,7 @@ public final class ChunkedOutputStream extends FilterOutputStream {
if (this.finished) throw new IOException("ChunkedOutputStream already finalized.");
if (len == 0) return;
this.out.write(Integer.toHexString(len).getBytes());
this.out.write(UTF8.getBytes(Integer.toHexString(len)));
this.out.write(serverCore.CRLF);
this.out.write(b, off, len);
this.out.write(serverCore.CRLF);
@ -82,7 +83,7 @@ public final class ChunkedOutputStream extends FilterOutputStream {
if (this.finished) throw new IOException("ChunkedOutputStream already finalized.");
if (len == 0) return;
this.out.write(Integer.toHexString(len).getBytes());
this.out.write(UTF8.getBytes(Integer.toHexString(len)));
this.out.write(serverCore.CRLF);
this.out.write(b.getBytes(off, len));
this.out.write(serverCore.CRLF);
@ -94,7 +95,7 @@ public final class ChunkedOutputStream extends FilterOutputStream {
final int len = b.available();
if (len == 0) return;
this.out.write(Integer.toHexString(len).getBytes());
this.out.write(UTF8.getBytes(Integer.toHexString(len)));
this.out.write(serverCore.CRLF);
FileUtils.copy(b, out, len);
this.out.write(serverCore.CRLF);
@ -104,7 +105,7 @@ public final class ChunkedOutputStream extends FilterOutputStream {
public void write(final int b) throws IOException {
if (this.finished) throw new IOException("ChunkedOutputStream already finalized.");
this.out.write("1".getBytes());
this.out.write(UTF8.getBytes("1"));
this.out.write(serverCore.CRLF);
this.out.write(b);
this.out.write(serverCore.CRLF);

@ -75,7 +75,6 @@ import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.net.URL;
import java.net.URLDecoder;
import java.nio.charset.Charset;
import java.util.Date;
import java.util.Iterator;
import java.util.List;
@ -505,7 +504,7 @@ public final class HTTPDFileHandler {
// write the list to the client
HTTPDemon.sendRespondHeader(conProp, out, httpVersion, 200, null, "text/html; charset=UTF-8", aBuffer.length(), new Date(targetFile.lastModified()), null, new ResponseHeader(), null, null, true);
if (!method.equals(HeaderFramework.METHOD_HEAD)) {
out.write(aBuffer.toString().getBytes("UTF-8"));
out.write(UTF8.getBytes(aBuffer.toString()));
}
return;
}
@ -747,7 +746,7 @@ public final class HTTPDFileHandler {
if (exitValue == 0 || (cgiBody != null && !cgiBody.equals(""))) {
HTTPDemon.sendRespondHeader(conProp, out, httpVersion, statusCode, null, mimeType, cgiBody.length(), targetDate, null, null, null, null, nocache);
out.write(cgiBody.getBytes());
out.write(UTF8.getBytes(cgiBody));
} else {
HTTPDemon.sendRespondError(conProp, out, exitValue, statusCode, null, HeaderFramework.http1_1.get(Integer.toString(statusCode)), null);
}
@ -941,7 +940,7 @@ public final class HTTPDFileHandler {
// send page in chunks and parse SSIs
final ByteBuffer o = new ByteBuffer();
// apply templates
TemplateEngine.writeTemplate(fis, o, templatePatterns, "-UNRESOLVED_PATTERN-".getBytes("UTF-8"));
TemplateEngine.writeTemplate(fis, o, templatePatterns, UTF8.getBytes("-UNRESOLVED_PATTERN-"));
fis.close();
HTTPDemon.sendRespondHeader(conProp, out,
httpVersion, 200, null, mimeType, -1,
@ -957,7 +956,7 @@ public final class HTTPDFileHandler {
final String contentEncoding = (zipContent) ? "gzip" : null;
// apply templates
final ByteBuffer o1 = new ByteBuffer();
TemplateEngine.writeTemplate(fis, o1, templatePatterns, "-UNRESOLVED_PATTERN-".getBytes("UTF-8"));
TemplateEngine.writeTemplate(fis, o1, templatePatterns, UTF8.getBytes("-UNRESOLVED_PATTERN-"));
fis.close();
final ByteBuffer o = new ByteBuffer();
@ -1316,9 +1315,9 @@ public final class HTTPDFileHandler {
StringWriter buffer = new StringWriter();
if (outgoingHeader.containsKey(HeaderFramework.TRANSFER_ENCODING)) {
FileUtils.copy(new ChunkedInputStream(in), buffer, Charset.forName("UTF-8"));
FileUtils.copy(new ChunkedInputStream(in), buffer, UTF8.charset);
} else {
FileUtils.copy(in, buffer, Charset.forName("UTF-8"));
FileUtils.copy(in, buffer, UTF8.charset);
}
String sbuffer = buffer.toString();
@ -1342,7 +1341,7 @@ public final class HTTPDFileHandler {
HTTPDemon.sendRespondHeader(conProp, out, httpVersion, httpStatus, outgoingHeader);
}
out.write(sbuffer.getBytes("UTF-8"));
out.write(UTF8.getBytes(sbuffer));
} else {
if (!outgoingHeader.containsKey(HeaderFramework.CONTENT_LENGTH))
outgoingHeader.put(HeaderFramework.CONTENT_LENGTH, prop.getProperty(HeaderFramework.CONNECTION_PROP_PROXY_RESPOND_SIZE));

@ -71,6 +71,7 @@ import java.util.logging.LogManager;
import java.util.logging.Logger;
import net.yacy.cora.document.MultiProtocolURI;
import net.yacy.cora.document.UTF8;
import net.yacy.cora.protocol.Domains;
import net.yacy.cora.protocol.HeaderFramework;
import net.yacy.cora.protocol.RequestHeader;
@ -1254,9 +1255,9 @@ public final class HTTPDProxyHandler {
final OutputStream promiscuousOut = sslSocket.getOutputStream();
// now then we can return a success message
clientOut.write((httpVersion + " 200 Connection established" + serverCore.CRLF_STRING +
clientOut.write(UTF8.getBytes(httpVersion + " 200 Connection established" + serverCore.CRLF_STRING +
"Proxy-agent: YACY" + serverCore.CRLF_STRING +
serverCore.CRLF_STRING).getBytes());
serverCore.CRLF_STRING));
log.logInfo("SSL connection to " + host + ":" + port + " established.");

@ -353,10 +353,10 @@ public final class HTTPDemon implements serverHandler, Cloneable {
return false;
}
// ask for authenticate
session.out.write((httpVersion + " 407 Proxy Authentication Required" + serverCore.CRLF_STRING +
RequestHeader.PROXY_AUTHENTICATE + ": Basic realm=\"log-in\"" + serverCore.CRLF_STRING).getBytes());
session.out.write((HeaderFramework.CONTENT_LENGTH + ": 0\r\n").getBytes());
session.out.write("\r\n".getBytes());
session.out.write(UTF8.getBytes(httpVersion + " 407 Proxy Authentication Required" + serverCore.CRLF_STRING +
RequestHeader.PROXY_AUTHENTICATE + ": Basic realm=\"log-in\"" + serverCore.CRLF_STRING));
session.out.write(UTF8.getBytes(HeaderFramework.CONTENT_LENGTH + ": 0\r\n"));
session.out.write(UTF8.getBytes("\r\n"));
session.out.flush();
return false;
}
@ -1122,7 +1122,7 @@ public final class HTTPDemon implements serverHandler, Cloneable {
fis = new FileInputStream(new File(htRootPath, "/proxymsg/error.html")),
o = new ByteArrayOutputStream(512),
tp,
"-UNRESOLVED_PATTERN-".getBytes()
UTF8.getBytes("-UNRESOLVED_PATTERN-")
);
final byte[] result = o.toByteArray();
o.close(); o = null;
@ -1301,7 +1301,7 @@ public final class HTTPDemon implements serverHandler, Cloneable {
header.append("\r\n");
// sending headers to the client
respond.write(header.toString().getBytes());
respond.write(UTF8.getBytes(header.toString()));
// flush stream
respond.flush();

@ -31,6 +31,7 @@ import java.io.IOException;
import java.io.OutputStream;
import java.util.Properties;
import net.yacy.cora.document.UTF8;
import net.yacy.cora.protocol.HeaderFramework;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.kelondro.util.ByteBuffer;
@ -43,10 +44,10 @@ public class ServerSideIncludes {
}
public static void writeSSI(final ByteBuffer in, int off, final OutputStream out, final String authorization, final String requesthost) throws IOException {
int p = in.indexOf("<!--#".getBytes(), off);
int p = in.indexOf(UTF8.getBytes("<!--#"), off);
int q;
while (p >= 0) {
q = in.indexOf("-->".getBytes(), p + 10);
q = in.indexOf(UTF8.getBytes("-->"), p + 10);
if (out instanceof ChunkedOutputStream) {
((ChunkedOutputStream) out).write(in, off, p - off);
} else {
@ -54,7 +55,7 @@ public class ServerSideIncludes {
}
parseSSI(in, p, out, authorization, requesthost);
off = q + 3;
p = in.indexOf("<!--#".getBytes(), off);
p = in.indexOf(UTF8.getBytes("<!--#"), off);
}
if (out instanceof ChunkedOutputStream) {
((ChunkedOutputStream) out).write(in, off, in.length() - off);
@ -64,8 +65,8 @@ public class ServerSideIncludes {
}
private static void parseSSI(final ByteBuffer in, final int off, final OutputStream out, final String authorization, final String requesthost) {
if (in.startsWith("<!--#include virtual=\"".getBytes(), off)) {
final int q = in.indexOf("\"".getBytes(), off + 22);
if (in.startsWith(UTF8.getBytes("<!--#include virtual=\""), off)) {
final int q = in.indexOf(UTF8.getBytes("\""), off + 22);
if (q > 0) {
final String path = in.toString(off + 22, q - off - 22);
writeContent(path, out, authorization, requesthost);

@ -55,7 +55,6 @@ import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.PushbackInputStream;
import java.io.UnsupportedEncodingException;
import java.util.HashMap;
import java.util.Map;
@ -237,16 +236,16 @@ public final class TemplateEngine {
structure.append('<')
.append(multi_key)
.append(" type=\"multi\" num=\"".getBytes())
.append(Integer.toString(num).getBytes())
.append("\">\n".getBytes());
.append(UTF8.getBytes(" type=\"multi\" num=\""))
.append(UTF8.getBytes(Integer.toString(num)))
.append(UTF8.getBytes("\">\n"));
for(int i=0;i < num;i++) {
final PushbackInputStream pis2 = new PushbackInputStream(new ByteArrayInputStream(text));
//System.out.println("recursing with text(prefix="+ multi_key + "_" + i + "_" +"):"); //DEBUG
//System.out.println(text);
structure.append(writeTemplate(pis2, out, pattern, dflt, newPrefix(prefix,multi_key,i)));
}//for
structure.append("</".getBytes()).append(multi_key).append(">\n".getBytes());
structure.append(UTF8.getBytes("</")).append(multi_key).append(UTF8.getBytes(">\n"));
} else {//transferUntil
Log.logSevere("TEMPLATE", "No Close Key found for #{"+UTF8.String(multi_key)+"}#"); //prefix here?
}
@ -273,7 +272,7 @@ public final class TemplateEngine {
}catch(final NumberFormatException e){
whichPattern=0;
byName=true;
patternName=patternId.getBytes("UTF-8");
patternName = UTF8.getBytes(patternId);
}
}
@ -283,16 +282,16 @@ public final class TemplateEngine {
PushbackInputStream pis2;
if (byName) {
//TODO: better Error Handling
transferUntil(pis, keyStream, appendBytes("%%".getBytes(), patternName, null, null));
transferUntil(pis, keyStream, appendBytes(UTF8.getBytes("%%"), patternName, null, null));
if(pis.available()==0){
Log.logSevere("TEMPLATE", "No such Template: %%"+UTF8.String(patternName));
Log.logSevere("TEMPLATE", "No such Template: %%" + UTF8.String(patternName));
return structure.getBytes();
}
keyStream.reset();
transferUntil(pis, keyStream, dpdpa);
pis2 = new PushbackInputStream(new ByteArrayInputStream(keyStream.toByteArray()));
structure.append(writeTemplate(pis2, out, pattern, dflt, newPrefix(prefix,key)));
transferUntil(pis, keyStream, appendBytes("#(/".getBytes(),key,")#".getBytes("UTF-8"),null));
transferUntil(pis, keyStream, appendBytes(UTF8.getBytes("#(/"), key, UTF8.getBytes(")#"), null));
if(pis.available()==0){
Log.logSevere("TEMPLATE", "No Close Key found for #("+UTF8.String(key)+")# (by Name)");
}
@ -308,16 +307,16 @@ public final class TemplateEngine {
if (java.util.Arrays.equals(keyStream.toByteArray(),appendBytes(slashChar, key, null,null))) {
pis2 = new PushbackInputStream(new ByteArrayInputStream(text.getBytes()));
//this maybe the wrong, but its the last
structure.append('<').append(key).append(" type=\"alternative\" which=\"".getBytes()).append(Integer.toString(whichPattern).getBytes()).append("\" found=\"0\">\n".getBytes());
structure.append('<').append(key).append(UTF8.getBytes(" type=\"alternative\" which=\"")).append(UTF8.getBytes(Integer.toString(whichPattern))).append(UTF8.getBytes("\" found=\"0\">\n"));
structure.append(writeTemplate(pis2, out, pattern, dflt, newPrefix(prefix,key)));
structure.append("</".getBytes()).append(key).append(">\n".getBytes());
structure.append(UTF8.getBytes("</")).append(key).append(UTF8.getBytes(">\n"));
found=true;
}else if(others >0 && keyStream.toString().startsWith("/")){ //close nested
others--;
text.append(aOpen).append(keyStream.toByteArray()).append(")#".getBytes());
text.append(aOpen).append(keyStream.toByteArray()).append(UTF8.getBytes(")#"));
} else { //nested
others++;
text.append(aOpen).append(keyStream.toByteArray()).append(")#".getBytes());
text.append(aOpen).append(keyStream.toByteArray()).append(UTF8.getBytes(")#"));
}
keyStream.reset(); //reset stream
continue;
@ -330,11 +329,11 @@ public final class TemplateEngine {
if ((bb & 0xFF) == ':'){
if(currentPattern == whichPattern){ //found the pattern
pis2 = new PushbackInputStream(new ByteArrayInputStream(text.getBytes()));
structure.append('<').append(key).append(" type=\"alternative\" which=\"".getBytes()).append(Integer.toString(whichPattern).getBytes()).append("\" found=\"0\">\n".getBytes());
structure.append('<').append(key).append(UTF8.getBytes(" type=\"alternative\" which=\"")).append(UTF8.getBytes(Integer.toString(whichPattern))).append(UTF8.getBytes("\" found=\"0\">\n"));
structure.append(writeTemplate(pis2, out, pattern, dflt, newPrefix(prefix,key)));
structure.append("</".getBytes()).append(key).append(">\n".getBytes());
structure.append(UTF8.getBytes("</")).append(key).append(UTF8.getBytes(">\n"));
transferUntil(pis, keyStream, appendBytes("#(/".getBytes(),key,")#".getBytes("UTF-8"),null));//to #(/key)#.
transferUntil(pis, keyStream, appendBytes(UTF8.getBytes("#(/"),key,UTF8.getBytes(")#"),null));//to #(/key)#.
found=true;
}
@ -342,7 +341,7 @@ public final class TemplateEngine {
text.clear();
continue;
}
text.append(":".getBytes());
text.append(UTF8.getBytes(":"));
}
if(!found){
text.append((byte)bb);/*
@ -361,9 +360,9 @@ public final class TemplateEngine {
key = keyStream.toByteArray();
final String patternKey = getPatternKey(prefix, key);
replacement = replacePattern(patternKey, pattern, dflt); //replace
structure.append("<".getBytes()).append(key).append(" type=\"normal\">\n".getBytes());
structure.append(UTF8.getBytes("<")).append(key).append(UTF8.getBytes(" type=\"normal\">\n"));
structure.append(replacement);
structure.append("</".getBytes()).append(key).append(">\n".getBytes());
structure.append(UTF8.getBytes("</")).append(key).append(UTF8.getBytes(">\n"));
FileUtils.copy(replacement, out);
} else {
@ -393,7 +392,7 @@ public final class TemplateEngine {
//Read the Include
String line = "";
while ((line = br.readLine()) != null) {
include.append(line.getBytes("UTF-8")).append(de.anomic.server.serverCore.CRLF_STRING.getBytes());
include.append(UTF8.getBytes(line)).append(UTF8.getBytes(de.anomic.server.serverCore.CRLF_STRING));
}
} catch (final IOException e) {
//file not found?
@ -422,18 +421,14 @@ public final class TemplateEngine {
Object value;
if (pattern.containsKey(key)) {
value = pattern.get(key);
try {
if (value instanceof byte[]) {
replacement = (byte[]) value;
} else if (value instanceof String) {
//replacement = ((String) value).getBytes();
replacement = ((String) value).getBytes("UTF-8");
} else {
//replacement = value.toString().getBytes();
replacement = value.toString().getBytes("UTF-8");
}
} catch (final UnsupportedEncodingException e) {
replacement = dflt;
if (value instanceof byte[]) {
replacement = (byte[]) value;
} else if (value instanceof String) {
//replacement = ((String) value).getBytes();
replacement = UTF8.getBytes(((String) value));
} else {
//replacement = value.toString().getBytes();
replacement = UTF8.getBytes(value.toString());
}
} else {
replacement = dflt;
@ -495,10 +490,10 @@ public final class TemplateEngine {
public static void main(final String[] args) {
// arg1 = test input; arg2 = replacement for pattern 'test'; arg3 = default replacement
try {
final InputStream i = new ByteArrayInputStream(args[0].getBytes("UTF-8"));
final InputStream i = new ByteArrayInputStream(UTF8.getBytes(args[0]));
final Map<String, String> h = new HashMap<String, String>();
h.put("test", args[1]);
writeTemplate(new PushbackInputStream(i, 100), System.out, h, args[2].getBytes("UTF-8"));
writeTemplate(new PushbackInputStream(i, 100), System.out, h, UTF8.getBytes(args[2]));
System.out.flush();
} catch (final Exception e) {
Log.logException(e);

@ -34,6 +34,7 @@ import java.util.Iterator;
import java.util.LinkedList;
import net.yacy.cora.date.GenericFormatter;
import net.yacy.cora.document.UTF8;
import net.yacy.document.LibraryProvider;
import net.yacy.kelondro.logging.Log;
@ -104,7 +105,7 @@ public class AccessTracker {
RandomAccessFile raf = new RandomAccessFile(file, "rw");
raf.seek(raf.length());
for (String s: log) {
raf.write(s.getBytes("UTF-8"));
raf.write(UTF8.getBytes(s));
raf.writeByte(10);
}
log.clear();

@ -894,7 +894,7 @@ public final class serverCore extends AbstractBusyThread implements BusyThread {
}
public static void send(final OutputStream os, final String buf) throws IOException {
os.write(buf.getBytes());
os.write(UTF8.getBytes(buf));
os.write(CRLF);
os.flush();
}

@ -340,7 +340,7 @@ public class serverObjects extends HashMap<String, String> implements Cloneable
for (Map.Entry<String, String> entry: entrySet()) {
key = entry.getKey();
value = patternNewline.matcher(entry.getValue()).replaceAll("\\\\n");
fos.write((key + "=" + value + "\r\n").getBytes());
fos.write(UTF8.getBytes(key + "=" + value + "\r\n"));
}
} finally {
if (fos != null) {

@ -116,7 +116,7 @@ public class cryptbig {
final SecretKey key = keyFactory.generateSecret(keySpec);
// create parameter spec for PBE
final PBEParameterSpec paramSpec = new PBEParameterSpec(salt.getBytes(), 1000 /*ITERATIONS*/);
final PBEParameterSpec paramSpec = new PBEParameterSpec(UTF8.getBytes(salt), 1000 /*ITERATIONS*/);
// Create a cipher and initialize it for encrypting end decrypting
cryptMethod = method;

@ -61,7 +61,7 @@ public class FlatWordPartitionScheme implements PartitionScheme {
}
public final long dhtDistance(final byte[] word, final String urlHash, final yacySeed peer) {
return dhtDistance(word, urlHash, peer.hash.getBytes());
return dhtDistance(word, urlHash, UTF8.getBytes(peer.hash));
}
private final long dhtDistance(final byte[] from, final String urlHash, final byte[] to) {
@ -98,7 +98,7 @@ public class FlatWordPartitionScheme implements PartitionScheme {
// transform the position of a peer position into a close peer hash
String s = UTF8.String(Base64Order.enhancedCoder.uncardinal(l));
while (s.length() < 12) s += "A";
return s.getBytes();
return UTF8.getBytes(s);
}

@ -191,7 +191,7 @@ public class PeerSelection {
public static byte[] limitOver(final yacySeedDB seedDB, final byte[] startHash) {
final Iterator<yacySeed> seeds = getAcceptRemoteIndexSeeds(seedDB, startHash, 1, false);
if (seeds.hasNext()) return seeds.next().hash.getBytes();
if (seeds.hasNext()) return UTF8.getBytes(seeds.next().hash);
return null;
}
@ -234,7 +234,7 @@ public class PeerSelection {
this.remaining = max;
this.doublecheck = new HandleSet(12, Base64Order.enhancedCoder, 0);
this.nextSeed = nextInternal();
this.alsoMyOwn = alsoMyOwn && nextSeed != null && (Base64Order.enhancedCoder.compare(seedDB.mySeed().hash.getBytes(), nextSeed.hash.getBytes()) > 0);
this.alsoMyOwn = alsoMyOwn && nextSeed != null && (Base64Order.enhancedCoder.compare(UTF8.getBytes(seedDB.mySeed().hash), UTF8.getBytes(nextSeed.hash)) > 0);
}
public boolean hasNext() {
@ -248,7 +248,7 @@ public class PeerSelection {
while (se.hasNext()) {
s = se.next();
if (s == null) return null;
byte[] hashb = s.hash.getBytes();
byte[] hashb = UTF8.getBytes(s.hash);
if (doublecheck.has(hashb)) return null;
try {
this.doublecheck.put(hashb);
@ -271,7 +271,7 @@ public class PeerSelection {
}
public yacySeed next() {
if (alsoMyOwn && Base64Order.enhancedCoder.compare(seedDB.mySeed().hash.getBytes(), nextSeed.hash.getBytes()) < 0) {
if (alsoMyOwn && Base64Order.enhancedCoder.compare(UTF8.getBytes(seedDB.mySeed().hash), UTF8.getBytes(nextSeed.hash)) < 0) {
// take my own seed hash instead the enumeration result
alsoMyOwn = false;
return seedDB.mySeed();

@ -25,6 +25,7 @@
package de.anomic.yacy.dht;
import net.yacy.cora.document.UTF8;
import de.anomic.yacy.yacySeed;
public class VerticalWordPartitionScheme implements PartitionScheme {
@ -68,7 +69,7 @@ public class VerticalWordPartitionScheme implements PartitionScheme {
// in case that the partitionExpoent is 1, only one bit is taken from the urlHash,
// which means that the partition is in two parts.
// With partitionExponent = 2 it is divided in four parts and so on.
return (FlatWordPartitionScheme.std.dhtPosition(wordHash, null) & partitionMask) | (FlatWordPartitionScheme.std.dhtPosition(urlHash.getBytes(), null) & ~partitionMask);
return (FlatWordPartitionScheme.std.dhtPosition(wordHash, null) & partitionMask) | (FlatWordPartitionScheme.std.dhtPosition(UTF8.getBytes(urlHash), null) & ~partitionMask);
}
public final long dhtPosition(final byte[] wordHash, final int verticalPosition) {
@ -107,7 +108,7 @@ public class VerticalWordPartitionScheme implements PartitionScheme {
}
public final long dhtDistance(final byte[] word, final String urlHash, final yacySeed peer) {
return dhtDistance(word, urlHash, peer.hash.getBytes());
return dhtDistance(word, urlHash, UTF8.getBytes(peer.hash));
}
private long dhtDistance(final byte[] from, final String urlHash, final byte[] to) {
@ -133,10 +134,10 @@ public class VerticalWordPartitionScheme implements PartitionScheme {
// the horizontal and vertical position calculation
String urlHash = args[1];
partitionExponent = Integer.parseInt(args[2]);
dhtl = partition.dhtPosition(wordHash.getBytes(), urlHash);
dhtl = partition.dhtPosition(UTF8.getBytes(wordHash), urlHash);
} else {
// only a horizontal position calculation
dhtl = FlatWordPartitionScheme.std.dhtPosition(wordHash.getBytes(), null);
dhtl = FlatWordPartitionScheme.std.dhtPosition(UTF8.getBytes(wordHash), null);
}
//System.out.println("DHT Double = " + dhtd);
System.out.println("DHT Long = " + dhtl);
@ -156,7 +157,7 @@ public class VerticalWordPartitionScheme implements PartitionScheme {
System.out.println();
*/
System.out.print("all " + (1 << partitionExponent) + " DHT positions from long : ");
long[] l = partition.dhtPositions(wordHash.getBytes());
long[] l = partition.dhtPositions(UTF8.getBytes(wordHash));
for (int i = 0; i < l.length; i++) {
if (i > 0) System.out.print(", ");
System.out.print(FlatWordPartitionScheme.positionToHash(l[i]));

@ -34,6 +34,7 @@ import java.util.Date;
import java.util.Iterator;
import net.yacy.cora.document.Hit;
import net.yacy.cora.document.UTF8;
import net.yacy.kelondro.logging.Log;
import net.yacy.visualization.PrintTool;
import net.yacy.visualization.RasterPlotter;
@ -136,7 +137,7 @@ public class NetworkGraph {
for (int j = 0; j < primarySearches.length; j++) {
if (primarySearches[j] == null) continue;
eventPicture.setColor((primarySearches[j].isAlive()) ? RasterPlotter.RED : RasterPlotter.GREEN);
angle = (int) (360.0 * (((double) FlatWordPartitionScheme.std.dhtPosition(primarySearches[j].target().hash.getBytes(), null)) / ((double) Long.MAX_VALUE)));
angle = (int) (360.0 * (((double) FlatWordPartitionScheme.std.dhtPosition(UTF8.getBytes(primarySearches[j].target().hash), null)) / ((double) Long.MAX_VALUE)));
eventPicture.arcLine(cx, cy, cr - 20, cr, angle, true, null, null, -1, -1, -1, false);
}
@ -145,7 +146,7 @@ public class NetworkGraph {
for (int j = 0; j < secondarySearches.length; j++) {
if (secondarySearches[j] == null) continue;
eventPicture.setColor((secondarySearches[j].isAlive()) ? RasterPlotter.RED : RasterPlotter.GREEN);
angle = (int) (360.0 * (((double) FlatWordPartitionScheme.std.dhtPosition(secondarySearches[j].target().hash.getBytes(), null)) / ((double) Long.MAX_VALUE)));
angle = (int) (360.0 * (((double) FlatWordPartitionScheme.std.dhtPosition(UTF8.getBytes(secondarySearches[j].target().hash), null)) / ((double) Long.MAX_VALUE)));
eventPicture.arcLine(cx, cy, cr - 10, cr, angle - 1, true, null, null, -1, -1, -1, false);
eventPicture.arcLine(cx, cy, cr - 10, cr, angle + 1, true, null, null, -1, -1, -1, false);
}
@ -279,8 +280,8 @@ public class NetworkGraph {
}
private static void drawNetworkPictureDHT(final RasterPlotter img, final int centerX, final int centerY, final int innerradius, final yacySeed mySeed, final yacySeed otherSeed, final String colorLine, final int coronaangle, boolean out) {
final int angleMy = (int) (360.0 * (((double) FlatWordPartitionScheme.std.dhtPosition(mySeed.hash.getBytes(), null)) / ((double) Long.MAX_VALUE)));
final int angleOther = (int) (360.0 * (((double) FlatWordPartitionScheme.std.dhtPosition(otherSeed.hash.getBytes(), null)) / ((double) Long.MAX_VALUE)));
final int angleMy = (int) (360.0 * (((double) FlatWordPartitionScheme.std.dhtPosition(UTF8.getBytes(mySeed.hash), null)) / ((double) Long.MAX_VALUE)));
final int angleOther = (int) (360.0 * (((double) FlatWordPartitionScheme.std.dhtPosition(UTF8.getBytes(otherSeed.hash), null)) / ((double) Long.MAX_VALUE)));
// draw line
img.arcLine(centerX, centerY, innerradius, innerradius - 20, angleMy, !out,
colorLine, null, 12, (coronaangle < 0) ? -1 : coronaangle / 30, 2, true);
@ -299,7 +300,7 @@ public class NetworkGraph {
final String name = seed.getName().toUpperCase() /*+ ":" + seed.hash + ":" + (((double) ((int) (100 * (((double) yacySeed.dhtPosition(seed.hash)) / ((double) yacySeed.maxDHTDistance))))) / 100.0)*/;
if (name.length() < shortestName) shortestName = name.length();
if (name.length() > longestName) longestName = name.length();
final int angle = (int) (360.0 * (((double) FlatWordPartitionScheme.std.dhtPosition(seed.hash.getBytes(), null)) / ((double) Long.MAX_VALUE)));
final int angle = (int) (360.0 * (((double) FlatWordPartitionScheme.std.dhtPosition(UTF8.getBytes(seed.hash), null)) / ((double) Long.MAX_VALUE)));
//System.out.println("Seed " + seed.hash + " has distance " + seed.dhtDistance() + ", angle = " + angle);
int linelength = 20 + outerradius * (20 * (name.length() - shortestName) / (longestName - shortestName) + Math.abs(seed.hash.hashCode() % 20)) / 60;
if (linelength > outerradius) linelength = outerradius;

@ -32,6 +32,8 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import net.yacy.cora.document.UTF8;
import com.jcraft.jsch.Channel;
import com.jcraft.jsch.ChannelExec;
import com.jcraft.jsch.JSch;
@ -149,7 +151,7 @@ class sshc {
command+=localFile.toString();
}
command+="\n";
out.write(command.getBytes()); out.flush();
out.write(UTF8.getBytes(command)); out.flush();
checkAck(in);

@ -46,7 +46,6 @@ package de.anomic.yacy;
import java.io.File;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
@ -55,6 +54,7 @@ import java.util.Properties;
import java.util.Map.Entry;
import net.yacy.cora.date.GenericFormatter;
import net.yacy.cora.document.UTF8;
import net.yacy.kelondro.data.word.Word;
import net.yacy.kelondro.index.Index;
import net.yacy.kelondro.index.Row;
@ -174,23 +174,18 @@ public class yacyNewsDB {
private final Row.Entry r2b(final Record r) {
if (r == null) return null;
try {
String attributes = r.attributes().toString();
if (attributes.length() > attributesMaxLength) {
Log.logWarning("yacyNewsDB", "attribute length=" + attributes.length() + " exceeds maximum size=" + attributesMaxLength);
attributes = new HashMap<String, String>().toString();
}
final Row.Entry entry = this.news.row().newEntry();
entry.setCol(0, r.id().getBytes());
entry.setCol(1, r.category().getBytes("UTF-8"));
entry.setCol(2, (r.received() == null) ? null : my_SHORT_SECOND_FORMATTER.format(r.received()).getBytes());
entry.setCol(3, Base64Order.enhancedCoder.encodeLongBA(r.distributed(), 2));
entry.setCol(4, attributes.getBytes("UTF-8"));
return entry;
} catch(final UnsupportedEncodingException e) {
// ignore this. this should never occure
return null;
String attributes = r.attributes().toString();
if (attributes.length() > attributesMaxLength) {
Log.logWarning("yacyNewsDB", "attribute length=" + attributes.length() + " exceeds maximum size=" + attributesMaxLength);
attributes = new HashMap<String, String>().toString();
}
final Row.Entry entry = this.news.row().newEntry();
entry.setCol(0, r.id().getBytes());
entry.setCol(1, UTF8.getBytes(r.category()));
entry.setCol(2, (r.received() == null) ? null : my_SHORT_SECOND_FORMATTER.format(r.received()).getBytes());
entry.setCol(3, Base64Order.enhancedCoder.encodeLongBA(r.distributed(), 2));
entry.setCol(4, UTF8.getBytes(attributes));
return entry;
}
public Record newRecord(final yacySeed mySeed, final String category, final Properties attributes) {

@ -328,7 +328,7 @@ public final class yacyRelease extends yacyVersion {
}
// Save signature
File signatureFile = new File(download.getAbsoluteFile() + ".sig");
FileUtils.copy(Base64Order.standardCoder.encode(signatureBytes).getBytes("UTF-8"), signatureFile);
FileUtils.copy(UTF8.getBytes(Base64Order.standardCoder.encode(signatureBytes)), signatureFile);
if ((!signatureFile.exists()) || (signatureFile.length() == 0)) throw new IOException("create signature file failed");
} else {
// just copy into file

@ -81,7 +81,7 @@ public class RSSReader extends DefaultHandler {
if (a.length < 100) {
throw new IOException("response=" + UTF8.String(a));
}
if (!equals(a, "<?xml".getBytes()) && !equals(a, "<rss".getBytes())) {
if (!equals(a, UTF8.getBytes("<?xml")) && !equals(a, UTF8.getBytes("<rss"))) {
throw new IOException("response does not contain valid xml");
}
final String end = UTF8.String(a, a.length - 80, 80);

@ -40,6 +40,7 @@ import java.util.concurrent.ConcurrentHashMap;
import net.yacy.cora.document.ASCIIComparator;
import net.yacy.cora.document.MultiProtocolURI;
import net.yacy.cora.document.UTF8;
/**
@ -364,7 +365,7 @@ public class HeaderFramework extends TreeMap<String, String> implements Map<Stri
try {
fos = new FileOutputStream(f);
for (java.util.Map.Entry<String, String> entry: entrySet()) {
fos.write((entry.getKey() + "=" + entry.getValue() + "\r\n").getBytes());
fos.write(UTF8.getBytes((entry.getKey() + "=" + entry.getValue() + "\r\n")));
}
fos.flush();
} finally {

@ -2817,7 +2817,7 @@ public class FTPClient {
final File file = new File("dirindex.html");
FileOutputStream fos;
fos = new FileOutputStream(file);
fos.write(page.toString().getBytes());
fos.write(UTF8.getBytes(page.toString()));
fos.close();
} catch (final FileNotFoundException e) {
log.error(e);

@ -26,7 +26,6 @@ import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.UnsupportedEncodingException;
import java.text.NumberFormat;
import java.util.HashMap;
import java.util.HashSet;
@ -234,11 +233,7 @@ public final class Condenser {
String word;
Word wprop;
WordTokenizer wordenum;
try {
wordenum = new WordTokenizer(new ByteArrayInputStream(text.getBytes("UTF-8")), meaningLib);
} catch (final UnsupportedEncodingException e) {
return;
}
wordenum = new WordTokenizer(new ByteArrayInputStream(UTF8.getBytes(text)), meaningLib);
int pip = 0;
while (wordenum.hasMoreElements()) {
word = (wordenum.nextElement().toString()).toLowerCase(Locale.ENGLISH);
@ -457,11 +452,7 @@ public final class Condenser {
// returns a word/indexWord relation map
if (text == null) return null;
ByteArrayInputStream buffer;
try {
buffer = new ByteArrayInputStream(text.getBytes("UTF-8"));
} catch (UnsupportedEncodingException e1) {
buffer = new ByteArrayInputStream(text.getBytes());
}
buffer = new ByteArrayInputStream(UTF8.getBytes(text));
return new Condenser(buffer, meaningLib).words();
}

@ -233,7 +233,7 @@ dc_rights
try {
if (this.text == null) return new ByteArrayInputStream("".getBytes());
if (this.text instanceof String) {
return new ByteArrayInputStream(((String) this.text).getBytes("UTF-8"));
return new ByteArrayInputStream(UTF8.getBytes(((String) this.text)));
} else if (this.text instanceof InputStream) {
return (InputStream) this.text;
} else if (this.text instanceof File) {
@ -255,7 +255,7 @@ dc_rights
try {
if (this.text == null) return new byte[0];
if (this.text instanceof String) {
return ((String) this.text).getBytes("UTF-8");
return UTF8.getBytes((String) this.text);
} else if (this.text instanceof InputStream) {
return FileUtils.read((InputStream) this.text);
} else if (this.text instanceof File) {

@ -26,13 +26,13 @@ package net.yacy.document;
import java.io.ByteArrayInputStream;
import java.io.InputStream;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.Enumeration;
import java.util.List;
import java.util.SortedMap;
import java.util.TreeMap;
import net.yacy.cora.document.UTF8;
import net.yacy.kelondro.data.word.Word;
import net.yacy.kelondro.order.Base64Order;
@ -173,11 +173,7 @@ public class WordTokenizer implements Enumeration<String> {
public static SortedMap<byte[], Integer> hashSentence(final String sentence, final WordCache meaningLib) {
final SortedMap<byte[], Integer> map = new TreeMap<byte[], Integer>(Base64Order.enhancedCoder);
Enumeration<String> words = null;
try {
words = new WordTokenizer(new ByteArrayInputStream(sentence.getBytes("UTF-8")), meaningLib);
} catch (UnsupportedEncodingException e) {
e.printStackTrace();
}
words = new WordTokenizer(new ByteArrayInputStream(UTF8.getBytes(sentence)), meaningLib);
int pos = 0;
String word;
byte[] hash;

@ -27,7 +27,6 @@ package net.yacy.document.content;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException;
import java.text.Collator;
import java.text.ParseException;
@ -37,6 +36,7 @@ import java.util.Locale;
import java.util.TreeMap;
import net.yacy.cora.date.ISO8601Formatter;
import net.yacy.cora.document.UTF8;
import net.yacy.document.Document;
import net.yacy.kelondro.data.meta.DigestURI;
import net.yacy.kelondro.logging.Log;
@ -243,27 +243,22 @@ public class DCEntry extends TreeMap<String, String> {
HashSet<String> languages = new HashSet<String>();
languages.add(getLanguage());
try {
return new Document(
getIdentifier(true),
"text/html",
"UTF-8",
languages,
getSubject(),
getTitle(),
getCreator(),
getPublisher(),
null,
"",
getDescription().getBytes("UTF-8"),
null,
null,
null,
false);
} catch (UnsupportedEncodingException e) {
Log.logException(e);
return null;
}
return new Document(
getIdentifier(true),
"text/html",
"UTF-8",
languages,
getSubject(),
getTitle(),
getCreator(),
getPublisher(),
null,
"",
UTF8.getBytes(getDescription()),
null,
null,
null,
false);
}
public void writeXML(OutputStreamWriter os) throws IOException {

@ -74,8 +74,8 @@ public class MediawikiImporter extends Thread implements Importer {
private static final String textend = "</text>";
private static final String pagestart = "<page>";
private static final String pageend = "</page>";
private static final byte[] pagestartb = pagestart.getBytes();
private static final byte[] pageendb = pageend.getBytes();
private static final byte[] pagestartb = UTF8.getBytes(pagestart);
private static final byte[] pageendb = UTF8.getBytes(pageend);
private static final int docspermbinxmlbz2 = 800; // documents per megabyte in a xml.bz2 wikimedia dump
public static Importer job; // if started from a servlet, this object is used to store the thread
@ -488,11 +488,9 @@ public class MediawikiImporter extends Thread implements Importer {
public void genDocument() throws Parser.Failure {
try {
url = new DigestURI(urlStub + title);
document = Document.mergeDocuments(url, "text/html", TextParser.parseSource(url, "text/html", "utf-8", html.getBytes("UTF-8")));
document = Document.mergeDocuments(url, "text/html", TextParser.parseSource(url, "text/html", "UTF-8", UTF8.getBytes(html)));
// the wiki parser is not able to find the proper title in the source text, so it must be set here
document.setTitle(title);
} catch (UnsupportedEncodingException e) {
Log.logException(e);
} catch (MalformedURLException e1) {
Log.logException(e1);
}
@ -571,7 +569,7 @@ public class MediawikiImporter extends Thread implements Importer {
long start;
String m = "<title>" + title + "</title>";
String s;
while (in.seek("<page ".getBytes())) {
while (in.seek(UTF8.getBytes("<page "))) {
start = in.pos() - 6;
in.resetBuffer();
if (!in.seek(pageendb)) break;

@ -28,9 +28,8 @@
package net.yacy.document.parser;
import java.io.InputStream;
import java.io.UnsupportedEncodingException;
import net.yacy.cora.document.MultiProtocolURI;
import net.yacy.cora.document.UTF8;
import net.yacy.document.AbstractParser;
import net.yacy.document.Document;
import net.yacy.document.Parser;
@ -85,26 +84,22 @@ public class docParser extends AbstractParser implements Parser {
}
Document[] docs;
try {
docs = new Document[]{new Document(
location,
mimeType,
"UTF-8",
null,
null,
title,
"", // TODO: AUTHOR
extractor.getDocSummaryInformation().getCompany(), // publisher
null,
null,
contents.toString().getBytes("UTF-8"),
null,
null,
null,
false)};
} catch (UnsupportedEncodingException e) {
throw new Parser.Failure("error in docParser, getBytes: " + e.getMessage(), location);
}
docs = new Document[]{new Document(
location,
mimeType,
"UTF-8",
null,
null,
title,
"", // TODO: AUTHOR
extractor.getDocSummaryInformation().getCompany(), // publisher
null,
null,
UTF8.getBytes(contents.toString()),
null,
null,
null,
false)};
return docs;
}

@ -44,6 +44,7 @@ import java.nio.charset.Charset;
import java.util.Enumeration;
import java.util.Properties;
import net.yacy.cora.document.UTF8;
import net.yacy.kelondro.data.meta.DigestURI;
import net.yacy.kelondro.io.CharBuffer;
import net.yacy.kelondro.logging.Log;
@ -585,7 +586,7 @@ public final class TransformerWriter extends Writer {
final Transformer transformer = new ContentTransformer();
final Reader is = new FileReader(args[0]);
final FileOutputStream fos = new FileOutputStream(new File(args[0] + ".out"));
final Writer os = new TransformerWriter(fos, Charset.forName("UTF-8"),scraper, transformer, false);
final Writer os = new TransformerWriter(fos, UTF8.charset, scraper, transformer, false);
int i;
while ((i = is.read(buffer)) > 0) os.write(buffer, 0, i);
os.close();

@ -51,6 +51,7 @@ import com.drew.metadata.exif.ExifReader;
import com.drew.metadata.iptc.IptcReader;
import net.yacy.cora.document.MultiProtocolURI;
import net.yacy.cora.document.UTF8;
import net.yacy.document.AbstractParser;
import net.yacy.document.Document;
import net.yacy.document.Parser;
@ -198,7 +199,7 @@ public class genericImageParser extends AbstractParser implements Parser {
location.getHost(), // Publisher
new String[]{}, // sections
description == null ? "" : description, // description
infoString.getBytes(), // content text
UTF8.getBytes(infoString), // content text
anchors, // anchors
null,
images,

@ -36,6 +36,7 @@ import org.xml.sax.SAXException;
import org.xml.sax.helpers.DefaultHandler;
import net.yacy.cora.document.MultiProtocolURI;
import net.yacy.cora.document.UTF8;
import net.yacy.document.AbstractParser;
import net.yacy.document.Document;
import net.yacy.document.Parser;
@ -73,7 +74,7 @@ public class mmParser extends AbstractParser implements Parser {
sb.append(". ");
}
content = sb.toString().getBytes("UTF-8");
content = UTF8.getBytes(sb.toString());
} catch (ParserConfigurationException ex) {
log.logWarning(ex.getMessage());

@ -39,6 +39,7 @@ import javax.xml.parsers.SAXParser;
import javax.xml.parsers.SAXParserFactory;
import net.yacy.cora.document.MultiProtocolURI;
import net.yacy.cora.document.UTF8;
import net.yacy.document.AbstractParser;
import net.yacy.document.Document;
import net.yacy.document.Parser;
@ -156,7 +157,7 @@ public class odtParser extends AbstractParser implements Parser {
// create the parser document
Document[] docs = null;
final byte[] contentBytes = writer.toString().getBytes("UTF-8");
final byte[] contentBytes = UTF8.getBytes(writer.toString());
docs = new Document[]{new Document(
location,
mimeType,

@ -39,6 +39,7 @@ import javax.xml.parsers.SAXParser;
import javax.xml.parsers.SAXParserFactory;
import net.yacy.cora.document.MultiProtocolURI;
import net.yacy.cora.document.UTF8;
import net.yacy.document.AbstractParser;
import net.yacy.document.Document;
import net.yacy.document.Parser;
@ -141,7 +142,7 @@ public class ooxmlParser extends AbstractParser implements Parser {
// create the parser document
Document[] docs = null;
final byte[] contentBytes = writer.toString().getBytes("UTF-8");
final byte[] contentBytes = UTF8.getBytes(writer.toString());
docs = new Document[]{new Document(
location,
mimeType,

@ -32,8 +32,6 @@ import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.UnsupportedEncodingException;
import org.apache.pdfbox.cos.COSName;
import org.apache.pdfbox.exceptions.CryptographyException;
import org.apache.pdfbox.pdmodel.PDDocument;
@ -45,6 +43,7 @@ import org.apache.pdfbox.pdmodel.font.PDFont;
import org.apache.pdfbox.util.PDFTextStripper;
import net.yacy.cora.document.MultiProtocolURI;
import net.yacy.cora.document.UTF8;
import net.yacy.document.AbstractParser;
import net.yacy.document.Document;
import net.yacy.document.Parser;
@ -149,12 +148,7 @@ public class pdfParser extends AbstractParser implements Parser {
}
byte[] contentBytes;
try {
contentBytes = writer.toString().getBytes("UTF-8");
} catch (UnsupportedEncodingException e) {
Log.logException(e);
throw new Parser.Failure(e.getMessage(), location);
}
contentBytes = UTF8.getBytes(writer.toString());
// clear resources in pdfbox. they say that is resolved but it's not. see:
// https://issues.apache.org/jira/browse/PDFBOX-313

@ -31,6 +31,7 @@ import java.io.BufferedInputStream;
import java.io.InputStream;
import net.yacy.cora.document.MultiProtocolURI;
import net.yacy.cora.document.UTF8;
import net.yacy.document.AbstractParser;
import net.yacy.document.Document;
import net.yacy.document.Parser;
@ -92,7 +93,7 @@ public class pptParser extends AbstractParser implements Parser {
pptExtractor.getDocSummaryInformation().getCompany(),
null,
null,
contents.getBytes("UTF-8"),
UTF8.getBytes(contents),
null,
null,
null,

@ -33,6 +33,7 @@ import javax.swing.text.DefaultStyledDocument;
import javax.swing.text.rtf.RTFEditorKit;
import net.yacy.cora.document.MultiProtocolURI;
import net.yacy.cora.document.UTF8;
import net.yacy.document.AbstractParser;
import net.yacy.document.Document;
import net.yacy.document.Parser;
@ -77,7 +78,7 @@ public class rtfParser extends AbstractParser implements Parser {
"", // TODO: publisher
null,
null,
bodyText.getBytes("UTF-8"),
UTF8.getBytes(bodyText),
null,
null,
null,

@ -33,6 +33,7 @@ import java.util.HashMap;
import java.util.Map;
import net.yacy.cora.document.MultiProtocolURI;
import net.yacy.cora.document.UTF8;
import net.yacy.document.AbstractParser;
import net.yacy.document.Document;
import net.yacy.document.Parser;
@ -116,7 +117,7 @@ public class swfParser extends AbstractParser implements Parser {
"",
sections, // an array of section headlines
abstrct, // an abstract
contents.getBytes("UTF-8"), // the parsed document text
UTF8.getBytes(contents), // the parsed document text
anchors, // a map of extracted anchors
null,
null,

@ -119,7 +119,7 @@ public class torrentParser extends AbstractParser implements Parser {
try {
byte[] b = FileUtils.read(new File(args[0]));
torrentParser parser = new torrentParser();
Document[] d = parser.parse(new MultiProtocolURI("http://localhost/test.torrent"), null, "utf-8", new ByteArrayInputStream(b));
Document[] d = parser.parse(new MultiProtocolURI("http://localhost/test.torrent"), null, "UTF-8", new ByteArrayInputStream(b));
Condenser c = new Condenser(d[0], true, true, LibraryProvider.dymLib);
Map<String, Word> w = c.words();
for (Map.Entry<String, Word> e: w.entrySet()) System.out.println("Word: " + e.getKey() + " - " + e.getValue().posInText);

@ -30,6 +30,7 @@ package net.yacy.document.parser;
import java.io.InputStream;
import net.yacy.cora.document.MultiProtocolURI;
import net.yacy.cora.document.UTF8;
import net.yacy.document.AbstractParser;
import net.yacy.document.Document;
import net.yacy.document.Parser;
@ -112,7 +113,7 @@ public class vsdParser extends AbstractParser implements Parser {
"",
null, // an array of section headlines
abstrct, // an abstract
contents.getBytes("UTF-8"), // the parsed document text
UTF8.getBytes(contents), // the parsed document text
null, // a map of extracted anchors
null,
null, // a treeset of image URLs

@ -30,6 +30,7 @@ package net.yacy.document.parser;
import java.io.InputStream;
import net.yacy.cora.document.MultiProtocolURI;
import net.yacy.cora.document.UTF8;
import net.yacy.document.AbstractParser;
import net.yacy.document.Document;
import net.yacy.document.Parser;
@ -122,7 +123,7 @@ public class xlsParser extends AbstractParser implements Parser {
"", // TODO: publisher
null,
null,
contents.getBytes("UTF-8"),
UTF8.getBytes(contents),
null,
null,
null,

@ -391,8 +391,7 @@ public final class Row {
return rowinstance[offset + colstart[column]] == 0;
}
@Deprecated
public final void setCol(final String nickname, final byte[] cell) {
private final void setCol(final String nickname, final byte[] cell) {
if (nickref == null) genNickRef();
final Object[] ref = nickref.get(nickname);
if (ref == null) return;
@ -432,8 +431,7 @@ public final class Row {
setCol(column, UTF8.getBytes(cell));
}
@Deprecated
public final void setCol(final String nickname, final long cell) {
private final void setCol(final String nickname, final long cell) {
if (nickref == null) genNickRef();
final Object[] ref = nickref.get(nickname);
if (ref == null) return;

@ -27,7 +27,6 @@
package net.yacy.kelondro.order;
import java.io.UnsupportedEncodingException;
import java.util.Comparator;
import net.yacy.cora.document.UTF8;
@ -208,11 +207,7 @@ public class Base64Order extends AbstractOrder<byte[]> implements ByteOrder, Com
}
public final String encodeString(final String in) {
try {
return encode(in.getBytes("UTF-8"));
} catch (final UnsupportedEncodingException e) {
return "";
}
return encode(UTF8.getBytes(in));
}
// we will use this encoding to encode strings with 2^8 values to

@ -204,11 +204,7 @@ public final class ByteBuffer extends OutputStream {
}
public ByteBuffer append(final String s) {
try {
return append(s.getBytes("UTF-8"));
} catch (UnsupportedEncodingException e) {
return append(UTF8.getBytes(s));
}
return append(UTF8.getBytes(s));
}
public ByteBuffer append(final String s, final String charset) throws UnsupportedEncodingException {

@ -466,7 +466,7 @@ public final class FileUtils {
if(os != null) {
for (final Iterator<byte[]> i = set.iterator(); i.hasNext(); ) {
os.write(i.next());
if (sep != null) os.write(sep.getBytes("UTF-8"));
if (sep != null) os.write(UTF8.getBytes(sep));
}
os.close();
}
@ -492,12 +492,12 @@ public final class FileUtils {
String key;
if (i.hasNext()) {
key = UTF8.String(i.next().getPrimaryKeyBytes());
os.write(key.getBytes("UTF-8"));
os.write(UTF8.getBytes(key));
}
while (i.hasNext()) {
key = UTF8.String(i.next().getPrimaryKeyBytes());
if (sep != null) os.write(sep.getBytes("UTF-8"));
os.write(key.getBytes("UTF-8"));
if (sep != null) os.write(UTF8.getBytes(sep));
os.write(UTF8.getBytes(key));
}
os.close();
}

Loading…
Cancel
Save