From 694fa3a2a5d2489845f1475e9ea5f476812ce254 Mon Sep 17 00:00:00 2001 From: orbiter Date: Thu, 10 Mar 2011 23:25:07 +0000 Subject: [PATCH] - replaced more direct string-based UTF-8 conversions by predefined UTF-8 conversion - changed menu structure slightly git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7583 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- htroot/Blog.java | 19 +-- htroot/BlogComments.java | 19 +-- htroot/ConfigPortal.html | 12 +- htroot/IndexControlCleaner_p.html | 108 ++++++++++++++++++ htroot/MessageSend_p.java | 8 +- htroot/Supporter.java | 13 +-- htroot/Surftips.java | 57 +++++---- htroot/ViewFile.java | 6 +- htroot/Wiki.java | 7 +- htroot/api/ymarks/import_ymark.java | 11 +- htroot/env/templates/header.template | 18 +-- htroot/sharedBlacklist_p.java | 7 +- htroot/yacy/message.java | 7 +- .../de/anomic/crawler/retrieval/Request.java | 8 +- source/de/anomic/data/BlogBoard.java | 24 +--- source/de/anomic/data/BlogBoardComments.java | 23 +--- source/de/anomic/data/BookmarkHelper.java | 16 +-- source/de/anomic/data/YMarkTables.java | 2 +- .../http/server/ChunkedOutputStream.java | 11 +- .../anomic/http/server/HTTPDFileHandler.java | 15 ++- .../anomic/http/server/HTTPDProxyHandler.java | 5 +- source/de/anomic/http/server/HTTPDemon.java | 12 +- .../http/server/ServerSideIncludes.java | 11 +- .../de/anomic/http/server/TemplateEngine.java | 63 +++++----- source/de/anomic/search/AccessTracker.java | 3 +- source/de/anomic/server/serverCore.java | 2 +- source/de/anomic/server/serverObjects.java | 2 +- source/de/anomic/tools/cryptbig.java | 2 +- .../yacy/dht/FlatWordPartitionScheme.java | 4 +- source/de/anomic/yacy/dht/PeerSelection.java | 8 +- .../yacy/dht/VerticalWordPartitionScheme.java | 11 +- .../de/anomic/yacy/graphics/NetworkGraph.java | 11 +- .../yacy/seedUpload/yacySeedUploadScp.java | 4 +- source/de/anomic/yacy/yacyNewsDB.java | 29 ++--- source/de/anomic/yacy/yacyRelease.java | 2 +- source/net/yacy/cora/document/RSSReader.java | 2 +- .../yacy/cora/protocol/HeaderFramework.java | 3 +- .../net/yacy/cora/protocol/ftp/FTPClient.java | 2 +- source/net/yacy/document/Condenser.java | 13 +-- source/net/yacy/document/Document.java | 4 +- source/net/yacy/document/WordTokenizer.java | 8 +- source/net/yacy/document/content/DCEntry.java | 39 +++---- .../document/importer/MediawikiImporter.java | 10 +- .../net/yacy/document/parser/docParser.java | 39 +++---- .../parser/html/TransformerWriter.java | 3 +- .../parser/images/genericImageParser.java | 3 +- source/net/yacy/document/parser/mmParser.java | 3 +- .../net/yacy/document/parser/odtParser.java | 3 +- .../net/yacy/document/parser/ooxmlParser.java | 3 +- .../net/yacy/document/parser/pdfParser.java | 10 +- .../net/yacy/document/parser/pptParser.java | 3 +- .../net/yacy/document/parser/rtfParser.java | 3 +- .../net/yacy/document/parser/swfParser.java | 3 +- .../yacy/document/parser/torrentParser.java | 2 +- .../net/yacy/document/parser/vsdParser.java | 3 +- .../net/yacy/document/parser/xlsParser.java | 3 +- source/net/yacy/kelondro/index/Row.java | 6 +- .../net/yacy/kelondro/order/Base64Order.java | 7 +- source/net/yacy/kelondro/util/ByteBuffer.java | 6 +- source/net/yacy/kelondro/util/FileUtils.java | 8 +- 60 files changed, 364 insertions(+), 385 deletions(-) create mode 100644 htroot/IndexControlCleaner_p.html diff --git a/htroot/Blog.java b/htroot/Blog.java index bb6468822..5571b6154 100644 --- a/htroot/Blog.java +++ b/htroot/Blog.java @@ -30,7 +30,6 @@ // javac -classpath .:../classes Blog.java // if the shell's current path is HTROOT -import java.io.UnsupportedEncodingException; import java.text.SimpleDateFormat; import java.util.Date; import java.util.HashMap; @@ -119,11 +118,7 @@ public class Blog { } byte[] author; - try { - author = StrAuthor.getBytes("UTF-8"); - } catch (final UnsupportedEncodingException e) { - author = StrAuthor.getBytes(); - } + author = UTF8.getBytes(StrAuthor); if (hasRights && post.containsKey("delete") && "sure".equals(post.get("delete"))) { page = sb.blogDB.readBlogEntry(pagename); @@ -141,11 +136,7 @@ public class Blog { if (post.containsKey("submit") && hasRights) { // store a new/edited blog-entry byte[] content; - try { - content = post.get("content", "").getBytes("UTF-8"); - } catch (final UnsupportedEncodingException e) { - content = post.get("content", "").getBytes(); - } + content = UTF8.getBytes(post.get("content", "")); final Date date; List comments = null; @@ -162,11 +153,7 @@ public class Blog { final String commentMode = post.get("commentMode", "2"); final String StrSubject = post.get("subject", ""); byte[] subject; - try { - subject = StrSubject.getBytes("UTF-8"); - } catch (final UnsupportedEncodingException e) { - subject = StrSubject.getBytes(); - } + subject = UTF8.getBytes(StrSubject); sb.blogDB.writeBlogEntry(sb.blogDB.newEntry(pagename, subject, author, ip, date, content, comments, commentMode)); diff --git a/htroot/BlogComments.java b/htroot/BlogComments.java index 688c6a5a9..eafa1203a 100644 --- a/htroot/BlogComments.java +++ b/htroot/BlogComments.java @@ -32,7 +32,6 @@ import java.io.File; import java.io.IOException; import java.io.PrintWriter; -import java.io.UnsupportedEncodingException; import java.text.SimpleDateFormat; import java.util.Date; import java.util.Iterator; @@ -104,11 +103,7 @@ public class BlogComments { } byte[] author; - try { - author = StrAuthor.getBytes("UTF-8"); - } catch (final UnsupportedEncodingException e) { - author = StrAuthor.getBytes(); - } + author = UTF8.getBytes(StrAuthor); final BlogBoard.BlogEntry page = sb.blogDB.readBlogEntry(pagename); //maybe "if(page == null)" final boolean pageExists = sb.blogDB.contains(pagename); @@ -123,22 +118,14 @@ public class BlogComments { if ("".equals(post.get("subject", ""))) { post.putHTML("subject", "no title"); } - try { - content = post.get("content", "").getBytes("UTF-8"); - } catch (final UnsupportedEncodingException e) { - content = post.get("content", "").getBytes(); - } + content = UTF8.getBytes(post.get("content", "")); final Date date = null; //set name for new entry or date for old entry final String StrSubject = post.get("subject", ""); byte[] subject; - try { - subject = StrSubject.getBytes("UTF-8"); - } catch (final UnsupportedEncodingException e) { - subject = StrSubject.getBytes(); - } + subject = UTF8.getBytes(StrSubject); final String commentID = String.valueOf(System.currentTimeMillis()); final BlogEntry blogEntry = sb.blogDB.readBlogEntry(pagename); blogEntry.addComment(commentID); diff --git a/htroot/ConfigPortal.html b/htroot/ConfigPortal.html index 3405b322c..872fa27bd 100644 --- a/htroot/ConfigPortal.html +++ b/htroot/ConfigPortal.html @@ -5,7 +5,7 @@ YaCy '#[clientname]#': Integration of a Search Portal #%env/templates/metas.template%# - + #%env/templates/header.template%# #%env/templates/submenuPortalIntegration.template%#

Integration of a Search Portal

@@ -15,11 +15,11 @@ and a link to a home page that is reached when the 'corporate identity'-images are clicked. To change also colours and styles use the Appearance Servlet for different skins and languages.

-
+
Greeting Line
-
+
URL of Home Page
@@ -125,7 +125,8 @@ height="180" frameborder="0" scrolling="auto" - id="target2"> + id="target2" + onfocus="removeFocus(this);">

@@ -148,7 +149,8 @@ height="180" frameborder="0" scrolling="auto" - id="target3"> + id="target3" + onfocus="removeFocus(this);"> #%env/templates/footer.template%# diff --git a/htroot/IndexControlCleaner_p.html b/htroot/IndexControlCleaner_p.html new file mode 100644 index 000000000..7ef61758d --- /dev/null +++ b/htroot/IndexControlCleaner_p.html @@ -0,0 +1,108 @@ + + + + YaCy '#[clientname]#': Index Cleaner + #%env/templates/metas.template%# + + +
+ #(inline)##%env/templates/header.template%# + +

Steering of API Actions

+

This table shows search results that had been sorted out from the search result display because their content had not been verified. + This means that the searched word does not appear on the search page. +

::#(/inline)# + #(showtable)#:: + +
+ +

+ #(navigation)# + :: + #(left)#no previous page::previous page#(/left)# + #[startRecord]#-#[to]# of #[of]# + #(right)#no next page::next page#(/right)# + + #(/navigation)# +

+ + + + + +
+

+

+ + + + + + + + + + + #(inline)#::#(/inline)# + + #{list}# + + + + + + + + + + #(inline)#::#(/inline)# + + #{/list}# + +
TypeCommentCall
Count		Recording
Date		Last Exec
Date		Next Exec
Date		SchedulerURL
#[type]##[comment]##[callcount]##[dateRecording]##[dateLastExec]##[dateNextExec]# + #(scheduler)# + + + + + + + :: +
+
+ + + +
+ + + + +
+ #(/scheduler)# + 		#[url]#
+

+

+ + + +

+ + + #(/showtable)# +
+ #%env/templates/footer.template%# + + + \ No newline at end of file diff --git a/htroot/MessageSend_p.java b/htroot/MessageSend_p.java index e5d7dcde6..db82e4689 100644 --- a/htroot/MessageSend_p.java +++ b/htroot/MessageSend_p.java @@ -25,12 +25,12 @@ //javac -classpath .:../Classes MessageSend_p.java //if the shell's current path is HTROOT -import java.io.UnsupportedEncodingException; import java.text.SimpleDateFormat; import java.util.Date; import java.util.Locale; import java.util.Map; +import net.yacy.cora.document.UTF8; import net.yacy.cora.protocol.RequestHeader; import de.anomic.search.Switchboard; @@ -128,11 +128,7 @@ public class MessageSend_p { if (subject.length() > 100) subject = subject.substring(0, 100); if (message.length() > messagesize) message = message.substring(0, messagesize); byte[] mb; - try { - mb = message.getBytes("UTF-8"); - } catch (final UnsupportedEncodingException e) { - mb = message.getBytes(); - } + mb = UTF8.getBytes(message); final Map result = yacyClient.postMessage(sb.peers, hash, subject, mb); //message has been sent diff --git a/htroot/Supporter.java b/htroot/Supporter.java index 58c9fbe37..3e9083232 100644 --- a/htroot/Supporter.java +++ b/htroot/Supporter.java @@ -25,7 +25,6 @@ // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -import java.io.IOException; import java.net.MalformedURLException; import java.util.Date; import java.util.HashMap; @@ -218,30 +217,30 @@ public class Supporter { entry = null; if ((record.category().equals(yacyNewsPool.CATEGORY_PROFILE_UPDATE)) && - ((seed = sb.peers.getConnected(record.originator())) != null)) try { + ((seed = sb.peers.getConnected(record.originator())) != null)) { url = record.attribute("homepage", ""); if (url.length() < 12) continue; entry = rowdef.newEntry(new byte[][]{ url.getBytes(), url.getBytes(), - ("Home Page of " + seed.getName()).getBytes("UTF-8"), + UTF8.getBytes(("Home Page of " + seed.getName())), record.id().getBytes() }); score = 1 + timeFactor(record.created()); - } catch (final IOException e) {} + } if ((record.category().equals(yacyNewsPool.CATEGORY_PROFILE_BROADCAST)) && - ((seed = sb.peers.getConnected(record.originator())) != null)) try { + ((seed = sb.peers.getConnected(record.originator())) != null)) { url = record.attribute("homepage", ""); if (url.length() < 12) continue; entry = rowdef.newEntry(new byte[][]{ url.getBytes(), url.getBytes(), - ("Home Page of " + seed.getName()).getBytes("UTF-8"), + UTF8.getBytes(("Home Page of " + seed.getName())), record.id().getBytes() }); score = 1 + timeFactor(record.created()); - } catch (final IOException e) {} + } // add/subtract votes and write record if (entry != null) { diff --git a/htroot/Surftips.java b/htroot/Surftips.java index 80b96c7cb..5af109d3a 100644 --- a/htroot/Surftips.java +++ b/htroot/Surftips.java @@ -25,7 +25,6 @@ // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -import java.io.IOException; import java.net.MalformedURLException; import java.util.Date; import java.util.HashMap; @@ -225,85 +224,85 @@ public class Surftips { if (record == null) continue; entry = null; - if (record.category().equals(yacyNewsPool.CATEGORY_CRAWL_START)) try { + if (record.category().equals(yacyNewsPool.CATEGORY_CRAWL_START)) { final String intention = record.attribute("intention", ""); url = record.attribute("startURL", ""); if (url.length() < 12) continue; entry = rowdef.newEntry(new byte[][]{ url.getBytes(), (((intention.length() == 0) || intention.equals("simple web crawl") || intention.equals("Automatic ReCrawl!")) ? record.attribute("startURL", "") : intention).getBytes(), - intention.equals("Automatic ReCrawl!") ? ("Automatic ReCrawl").getBytes("UTF-8") : ("Crawl Start Point").getBytes("UTF-8"), + intention.equals("Automatic ReCrawl!") ? UTF8.getBytes("Automatic ReCrawl") : UTF8.getBytes("Crawl Start Point"), record.id().getBytes() }); score = 2 + Math.min(10, intention.length() / 4) + timeFactor(record.created()); - } catch (final IOException e) {} + } - if (record.category().equals(yacyNewsPool.CATEGORY_BOOKMARK_ADD)) try { + if (record.category().equals(yacyNewsPool.CATEGORY_BOOKMARK_ADD)) { url = record.attribute("url", ""); if (url.length() < 12) continue; entry = rowdef.newEntry(new byte[][]{ url.getBytes(), - (record.attribute("title", "")).getBytes("UTF-8"), - ("Bookmark: " + record.attribute("description", "")).getBytes("UTF-8"), + UTF8.getBytes(record.attribute("title", "")), + UTF8.getBytes("Bookmark: " + record.attribute("description", "")), record.id().getBytes() }); score = 8 + timeFactor(record.created()); - } catch (final IOException e) {} + } - if (record.category().equals(yacyNewsPool.CATEGORY_SURFTIPP_ADD)) try { + if (record.category().equals(yacyNewsPool.CATEGORY_SURFTIPP_ADD)) { url = record.attribute("url", ""); if (url.length() < 12) continue; entry = rowdef.newEntry(new byte[][]{ url.getBytes(), - (record.attribute("title", "")).getBytes("UTF-8"), - ("Surf Tipp: " + record.attribute("description", "")).getBytes("UTF-8"), + UTF8.getBytes(record.attribute("title", "")), + UTF8.getBytes("Surf Tipp: " + record.attribute("description", "")), record.id().getBytes() }); score = 5 + timeFactor(record.created()); - } catch (final IOException e) {} + } - if (record.category().equals(yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD)) try { + if (record.category().equals(yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD)) { if (!(record.attribute("vote", "negative").equals("positive"))) continue; url = record.attribute("url", ""); if (url.length() < 12) continue; entry = rowdef.newEntry(new byte[][]{ url.getBytes(), - record.attribute("title", "").getBytes("UTF-8"), - record.attribute("description", "").getBytes("UTF-8"), - record.attribute("refid", "").getBytes() + UTF8.getBytes(record.attribute("title", "")), + UTF8.getBytes(record.attribute("description", "")), + UTF8.getBytes(record.attribute("refid", "")) }); score = 5 + timeFactor(record.created()); - } catch (final IOException e) {} + } - if (record.category().equals(yacyNewsPool.CATEGORY_WIKI_UPDATE)) try { + if (record.category().equals(yacyNewsPool.CATEGORY_WIKI_UPDATE)) { yacySeed seed = sb.peers.getConnected(record.originator()); if (seed == null) seed = sb.peers.getDisconnected(record.originator()); if (seed != null) { url = "http://" + seed.getPublicAddress() + "/Wiki.html?page=" + record.attribute("page", ""); entry = rowdef.newEntry(new byte[][]{ - url.getBytes(), - (record.attribute("author", "Anonymous") + ": " + record.attribute("page", "")).getBytes("UTF-8"), - ("Wiki Update: " + record.attribute("description", "")).getBytes("UTF-8"), - record.id().getBytes() + UTF8.getBytes(url), + UTF8.getBytes(record.attribute("author", "Anonymous") + ": " + record.attribute("page", "")), + UTF8.getBytes("Wiki Update: " + record.attribute("description", "")), + UTF8.getBytes(record.id()) }); score = 4 + timeFactor(record.created()); } - } catch (final IOException e) {} + } - if (record.category().equals(yacyNewsPool.CATEGORY_BLOG_ADD)) try { + if (record.category().equals(yacyNewsPool.CATEGORY_BLOG_ADD)) { yacySeed seed = sb.peers.getConnected(record.originator()); if (seed == null) seed = sb.peers.getDisconnected(record.originator()); if (seed != null) { url = "http://" + seed.getPublicAddress() + "/Blog.html?page=" + record.attribute("page", ""); entry = rowdef.newEntry(new byte[][]{ - url.getBytes(), - (record.attribute("author", "Anonymous") + ": " + record.attribute("page", "")).getBytes("UTF-8"), - ("Blog Entry: " + record.attribute("subject", "")).getBytes("UTF-8"), - record.id().getBytes() + UTF8.getBytes(url), + UTF8.getBytes(record.attribute("author", "Anonymous") + ": " + record.attribute("page", "")), + UTF8.getBytes("Blog Entry: " + record.attribute("subject", "")), + UTF8.getBytes(record.id()) }); score = 4 + timeFactor(record.created()); } - } catch (final IOException e) {} + } // add/subtract votes and write record if (entry != null) { diff --git a/htroot/ViewFile.java b/htroot/ViewFile.java index 3c1b52c81..25c029e7f 100644 --- a/htroot/ViewFile.java +++ b/htroot/ViewFile.java @@ -288,11 +288,7 @@ public class ViewFile { for (final StringBuilder s: sentences) { sentence = s.toString(); Enumeration tokens = null; - try { - tokens = new WordTokenizer(new ByteArrayInputStream(sentence.getBytes("UTF-8")), LibraryProvider.dymLib); - } catch (UnsupportedEncodingException e) { - e.printStackTrace(); - } + tokens = new WordTokenizer(new ByteArrayInputStream(UTF8.getBytes(sentence)), LibraryProvider.dymLib); while (tokens.hasMoreElements()) { token = tokens.nextElement(); if (token.length() > 0) { diff --git a/htroot/Wiki.java b/htroot/Wiki.java index f7d11383e..f5575b3f7 100644 --- a/htroot/Wiki.java +++ b/htroot/Wiki.java @@ -31,7 +31,6 @@ // if the shell's current path is HTROOT import java.io.IOException; -import java.io.UnsupportedEncodingException; import java.text.SimpleDateFormat; import java.util.Date; import java.util.HashMap; @@ -114,11 +113,7 @@ public class Wiki { // store a new page byte[] content; - try { - content = post.get("content", "").getBytes("UTF-8"); - } catch (final UnsupportedEncodingException e) { - content = post.get("content", "").getBytes(); - } + content = UTF8.getBytes(post.get("content", "")); final WikiBoard.Entry newEntry = sb.wikiDB.newEntry(pagename, author, ip, post.get("reason", "edit"), content); sb.wikiDB.write(newEntry); // create a news message diff --git a/htroot/api/ymarks/import_ymark.java b/htroot/api/ymarks/import_ymark.java index 4fca4d7e7..b25ffa671 100644 --- a/htroot/api/ymarks/import_ymark.java +++ b/htroot/api/ymarks/import_ymark.java @@ -1,8 +1,8 @@ import java.io.ByteArrayInputStream; import java.io.IOException; -import java.io.UnsupportedEncodingException; import java.util.HashMap; +import net.yacy.cora.document.UTF8; import net.yacy.cora.protocol.RequestHeader; import net.yacy.document.Document; import net.yacy.document.Parser.Failure; @@ -42,14 +42,7 @@ public class import_ymark { } if(post.containsKey("bmkfile") && post.containsKey("importer")){ - try { - byteIn = new ByteArrayInputStream(post.get("bmkfile$file").getBytes("UTF-8")); - } catch (UnsupportedEncodingException e) { - //TODO: display an error message - Log.logException(e); - prop.put("result", "0"); - return prop; - } + byteIn = new ByteArrayInputStream(UTF8.getBytes(post.get("bmkfile$file"))); if(post.get("importer").equals("html") && byteIn != null) { final YMarksHTMLImporter htmlImporter = new YMarksHTMLImporter(byteIn, 100); t = new Thread(htmlImporter, "YMarks - HTML Importer"); diff --git a/htroot/env/templates/header.template b/htroot/env/templates/header.template index 3ee3f70c0..49cb45285 100644 --- a/htroot/env/templates/header.template +++ b/htroot/env/templates/header.template @@ -46,15 +46,22 @@ + -