diff --git a/htroot/Blog.java b/htroot/Blog.java index 7ee5a0c3f..a09c7ce67 100644 --- a/htroot/Blog.java +++ b/htroot/Blog.java @@ -34,20 +34,19 @@ import java.text.SimpleDateFormat; import java.util.Date; import java.util.HashMap; import java.util.Iterator; +import java.util.List; import java.util.Locale; +import java.util.Map; import net.yacy.cora.document.UTF8; import net.yacy.cora.protocol.HeaderFramework; import net.yacy.cora.protocol.RequestHeader; import net.yacy.peers.NewsPool; import net.yacy.search.Switchboard; - import de.anomic.data.BlogBoard; import de.anomic.data.UserDB; import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; -import java.util.List; -import java.util.Map; public class Blog { @@ -65,13 +64,13 @@ public class Blog { final serverObjects prop = new serverObjects(); BlogBoard.BlogEntry page = null; - boolean hasRights = sb.verifyAuthentication(header, true); - + boolean hasRights = sb.verifyAuthentication(header); + //final int display = (hasRights || post == null) ? 1 : post.getInt("display", 0); - //prop.put("display", display); + //prop.put("display", display); prop.put("display", 1); // Fixed to 1 - + final boolean xml = (header.get(HeaderFramework.CONNECTION_PROP_PATH)).endsWith(".xml"); final String address = sb.peers.mySeed().getPublicAddress(); @@ -249,7 +248,7 @@ public class Blog { int start, int num, final boolean hasRights, - final boolean xml) + final boolean xml) { final Iterator i = switchboard.blogDB.getBlogIterator(false); @@ -289,7 +288,7 @@ public class Blog { prop.put("mode_preventries_start", prevstart); prop.put("mode_preventries_num", num); } else prop.put("mode_preventries", "0"); - + return prop; } @@ -300,7 +299,7 @@ public class Blog { final String address, final int number, final boolean hasRights, - final boolean xml) + final boolean xml) { prop.putHTML("mode_entries_" + number + "_subject", UTF8.String(entry.getSubject())); diff --git a/htroot/BlogComments.java b/htroot/BlogComments.java index 4e08a02da..95d89c076 100644 --- a/htroot/BlogComments.java +++ b/htroot/BlogComments.java @@ -44,12 +44,11 @@ import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.util.FileUtils; import net.yacy.peers.Network; import net.yacy.search.Switchboard; - import de.anomic.data.BlogBoard; +import de.anomic.data.BlogBoard.BlogEntry; import de.anomic.data.BlogBoardComments; import de.anomic.data.MessageBoard; import de.anomic.data.UserDB; -import de.anomic.data.BlogBoard.BlogEntry; import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; @@ -65,7 +64,7 @@ public class BlogComments { public static serverObjects respond(final RequestHeader header, serverObjects post, final serverSwitch env) { final Switchboard sb = (Switchboard) env; final serverObjects prop = new serverObjects(); - boolean hasRights = sb.verifyAuthentication(header, true); + boolean hasRights = sb.verifyAuthentication(header); prop.put("mode_admin", hasRights ? "1" : "0"); @@ -107,7 +106,7 @@ public class BlogComments { final BlogBoard.BlogEntry page = sb.blogDB.readBlogEntry(pagename); //maybe "if(page == null)" final boolean pageExists = sb.blogDB.contains(pagename); - + // comments not allowed prop.put("mode_allow", (page.getCommentMode() == 0) ? 0 : 1); @@ -217,12 +216,12 @@ public class BlogComments { while (i.hasNext() && count < num) { pageid = i.next(); - + if(start > 0) { start--; continue; } - + entry = sb.blogCommentDB.read(pageid); if (commentMode == 2 && !hasRights && !entry.isAllowed()) { @@ -309,7 +308,7 @@ public class BlogComments { .append("/") .append(msgEntry.authorHash()) .append("\nMessage to: ") - .append(msgEntry.recipient()) + .append(msgEntry.recipient()) .append("/") .append(msgEntry.recipientHash()) .append("\nCategory: ") diff --git a/htroot/Bookmarks.java b/htroot/Bookmarks.java index a4150627b..5b658437a 100644 --- a/htroot/Bookmarks.java +++ b/htroot/Bookmarks.java @@ -85,7 +85,7 @@ public class Bookmarks { prop.clear(); sb = (Switchboard) env; user = sb.userDB.getUser(header); - isAdmin = (sb.verifyAuthentication(header, true) || user!= null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT)); + isAdmin = (sb.verifyAuthentication(header) || user!= null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT)); // set user name if (user != null) { diff --git a/htroot/Collage.java b/htroot/Collage.java index 3a41ae927..3609d2643 100755 --- a/htroot/Collage.java +++ b/htroot/Collage.java @@ -50,7 +50,7 @@ public class Collage { public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { final serverObjects prop = new serverObjects(); final Switchboard sb = (Switchboard) env; - final boolean authenticated = sb.verifyAuthentication(header, false); + final boolean authenticated = sb.verifyAuthentication(header); final ResultImages.OriginEntry nextOrigin = ResultImages.next(!authenticated); int posXMax = 800; int posYMax = 500; diff --git a/htroot/ConfigParser.java b/htroot/ConfigParser.java index 4b41d621d..86182ec96 100644 --- a/htroot/ConfigParser.java +++ b/htroot/ConfigParser.java @@ -41,17 +41,17 @@ public class ConfigParser { final serverObjects prop = new serverObjects(); final Switchboard sb = (Switchboard) env; - + if (post != null) { - if (!sb.verifyAuthentication(header, false)) { + if (!sb.verifyAuthentication(header)) { // force log-in prop.put("AUTHENTICATE", "admin log-in"); return prop; } - + if (post.containsKey("parserSettings")) { post.remove("parserSettings"); - + for (final Parser parser: TextParser.parsers()) { for (final String mimeType: parser.supportedMimeTypes()) { TextParser.grantMime(mimeType, "on".equals(post.get("mimename_" + mimeType, ""))); @@ -60,11 +60,11 @@ public class ConfigParser { env.setConfig(SwitchboardConstants.PARSER_MIME_DENY, TextParser.getDenyMime()); } } - - int i = 0; + + int i = 0; for (final Parser parser: TextParser.parsers()) { prop.put("parser_" + i + "_name", parser.getName()); - + int mimeIdx = 0; for (final String mimeType: parser.supportedMimeTypes()) { prop.put("parser_" + i + "_mime_" + mimeIdx + "_mimetype", mimeType); @@ -74,9 +74,9 @@ public class ConfigParser { prop.put("parser_" + i + "_mime", mimeIdx); i++; } - + prop.put("parser", i); - + // return rewrite properties return prop; } diff --git a/htroot/CrawlResults.html b/htroot/CrawlResults.html index c24b6bac0..bc773e020 100644 --- a/htroot/CrawlResults.html +++ b/htroot/CrawlResults.html @@ -83,7 +83,6 @@
-
diff --git a/htroot/CrawlResults.java b/htroot/CrawlResults.java index 7dc454943..9cdceff83 100644 --- a/htroot/CrawlResults.java +++ b/htroot/CrawlResults.java @@ -60,8 +60,8 @@ public class CrawlResults { boolean showDate = env.getConfigBool("IndexMonitorDate", true); boolean showWords = env.getConfigBool("IndexMonitorWords", true); boolean showTitle = env.getConfigBool("IndexMonitorTitle", true); - boolean showCountry = env.getConfigBool("IndexMonitorCountry", true); - boolean showIP = env.getConfigBool("IndexMonitorIP", true); + boolean showCountry = env.getConfigBool("IndexMonitorCountry", false); + boolean showIP = env.getConfigBool("IndexMonitorIP", false); boolean showURL = env.getConfigBool("IndexMonitorURL", true); if (post == null) { @@ -92,7 +92,7 @@ public class CrawlResults { post.containsKey("deleteentry")))) { final String authorization = (header.get(RequestHeader.AUTHORIZATION, "xxxxxx")); if (authorization.length() != 0) { - if (! sb.verifyAuthentication(header, true)){ + if (! sb.verifyAuthentication(header)){ // force log-in (again, because wrong password was given) prop.put("AUTHENTICATE", "admin log-in"); return prop; @@ -122,8 +122,8 @@ public class CrawlResults { } if (post.containsKey("deletedomain")) { - final String hashpart = post.get("hashpart", null); final String domain = post.get("domain", null); + final String hashpart = domain == null ? null : DigestURI.hosthash6(domain); if (hashpart != null) { // delete all urls for this domain from database try { @@ -289,7 +289,6 @@ public class CrawlResults { prop.put("table_domains_" + cnt + "_feedbackpage", "CrawlResults.html"); prop.put("table_domains_" + cnt + "_tabletype", tabletype.getCode()); prop.put("table_domains_" + cnt + "_domain", domain); - prop.put("table_domains_" + cnt + "_hashpart", DigestURI.hosthash6(domain)); prop.put("table_domains_" + cnt + "_count", ResultURLs.domainCount(tabletype, domain)); dark = !dark; cnt++; diff --git a/htroot/Steering.java b/htroot/Steering.java index 825cd1272..b8a028411 100644 --- a/htroot/Steering.java +++ b/htroot/Steering.java @@ -34,7 +34,6 @@ import net.yacy.cora.protocol.RequestHeader; import net.yacy.kelondro.logging.Log; import net.yacy.peers.operation.yacyRelease; import net.yacy.search.Switchboard; - import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; @@ -48,9 +47,9 @@ public class Steering { prop.put("info", "0"); //no information submitted final String requestIP = post.get(HeaderFramework.CONNECTION_PROP_CLIENTIP, "127.0.0.1"); - + // handle access rights - if (!sb.verifyAuthentication(header, false)) { + if (!sb.verifyAuthentication(header)) { Log.logInfo("STEERING", "log-in attempt for steering from " + requestIP); prop.put("AUTHENTICATE", "admin log-in"); // force log-in return prop; @@ -60,7 +59,7 @@ public class Steering { Log.logInfo("STEERING", "shutdown request from " + requestIP); sb.terminate(10, "shutdown request from Steering; ip = " + requestIP); prop.put("info", "3"); - + return prop; } @@ -68,10 +67,10 @@ public class Steering { Log.logInfo("STEERING", "restart request from " + requestIP); yacyRelease.restart(); prop.put("info", "4"); - + return prop; } - + if (post.containsKey("update")) { Log.logInfo("STEERING", "update request from " + requestIP); final boolean devenvironment = new File(sb.getAppPath(), ".svn").exists(); @@ -82,7 +81,7 @@ public class Steering { } prop.put("info", "5"); prop.putHTML("info_release", releaseFileName); - + return prop; } return prop; diff --git a/htroot/Supporter.java b/htroot/Supporter.java index b7310f7f4..8ed8faa25 100644 --- a/htroot/Supporter.java +++ b/htroot/Supporter.java @@ -64,14 +64,14 @@ public class Supporter { // access control final boolean publicPage = sb.getConfigBool("publicSurftips", true); - final boolean authorizedAccess = sb.verifyAuthentication(header, false); + final boolean authorizedAccess = sb.verifyAuthentication(header); if ((publicPage) || (authorizedAccess)) { // read voting String hash; if ((post != null) && ((hash = post.get("voteNegative", null)) != null)) { - if (!sb.verifyAuthentication(header, false)) { + if (!sb.verifyAuthentication(header)) { prop.put("AUTHENTICATE", "admin log-in"); // force log-in return prop; } @@ -85,7 +85,7 @@ public class Supporter { } } if ((post != null) && ((hash = post.get("votePositive", null)) != null)) { - if (!sb.verifyAuthentication(header, false)) { + if (!sb.verifyAuthentication(header)) { prop.put("AUTHENTICATE", "admin log-in"); // force log-in return prop; } diff --git a/htroot/Surftips.java b/htroot/Surftips.java index e9339d00e..9c5e9eeb5 100644 --- a/htroot/Surftips.java +++ b/htroot/Surftips.java @@ -63,7 +63,7 @@ public class Surftips { // access control boolean publicPage = sb.getConfigBool("publicSurftips", true); - final boolean authorizedAccess = sb.verifyAuthentication(header, false); + final boolean authorizedAccess = sb.verifyAuthentication(header); if ((post != null) && (post.containsKey("publicPage"))) { if (!authorizedAccess) { prop.put("AUTHENTICATE", "admin log-in"); // force log-in @@ -78,7 +78,7 @@ public class Surftips { // read voting String hash; if ((post != null) && ((hash = post.get("voteNegative", null)) != null)) { - if (!sb.verifyAuthentication(header, false)) { + if (!sb.verifyAuthentication(header)) { prop.put("AUTHENTICATE", "admin log-in"); // force log-in return prop; } @@ -92,7 +92,7 @@ public class Surftips { } } if ((post != null) && ((hash = post.get("votePositive", null)) != null)) { - if (!sb.verifyAuthentication(header, false)) { + if (!sb.verifyAuthentication(header)) { prop.put("AUTHENTICATE", "admin log-in"); // force log-in return prop; } diff --git a/htroot/User.java b/htroot/User.java index 9c4dfeb42..2c4c5960f 100644 --- a/htroot/User.java +++ b/htroot/User.java @@ -1,4 +1,4 @@ -//User.java +//User.java //----------------------- //part of the AnomicHTTPD caching proxy //(C) by Michael Peter Christen; mc@yacy.net @@ -35,14 +35,13 @@ import net.yacy.kelondro.order.Base64Order; import net.yacy.kelondro.order.Digest; import net.yacy.search.Switchboard; import net.yacy.search.SwitchboardConstants; - import de.anomic.data.UserDB; import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; import de.anomic.server.servletProperties; public class User{ - + public static servletProperties respond(final RequestHeader requestHeader, final serverObjects post, final serverSwitch env) { final servletProperties prop = new servletProperties(); final Switchboard sb = Switchboard.getSwitchboard(); @@ -70,7 +69,7 @@ public class User{ } } } - + //identified via userDB if(entry != null){ prop.put("logged-in", "1"); @@ -88,7 +87,7 @@ public class User{ prop.put("logged-in_limit_percent2", (100-percent)/3); } //logged in via static Password - }else if(sb.verifyAuthentication(requestHeader, true)){ + }else if(sb.verifyAuthentication(requestHeader)){ prop.put("logged-in", "2"); //identified via form-login //TODO: this does not work for a static admin, yet. @@ -98,7 +97,7 @@ public class User{ final String username=post.get("username"); final String password=post.get("password"); prop.put("logged-in_username", username); - + entry=sb.userDB.passwordAuth(username, password); final boolean staticAdmin = sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, "").equals( Digest.encodeMD5Hex( @@ -111,12 +110,12 @@ public class User{ cookie=sb.userDB.getCookie(entry); else if(staticAdmin) cookie=sb.userDB.getAdminCookie(); - + if(entry != null || staticAdmin){ final ResponseHeader outgoingHeader=new ResponseHeader(); outgoingHeader.setCookie("login", cookie); prop.setOutgoingHeader(outgoingHeader); - + prop.put("logged-in", "1"); prop.put("logged-in_identified-by", "1"); prop.putHTML("logged-in_username", username); @@ -125,7 +124,7 @@ public class User{ } } } - + if(post!= null && entry != null){ if(post.containsKey("changepass")){ prop.put("status", "1"); //password diff --git a/htroot/ViewFile.java b/htroot/ViewFile.java index 123a5aa3b..7e2ab1674 100644 --- a/htroot/ViewFile.java +++ b/htroot/ViewFile.java @@ -92,7 +92,7 @@ public class ViewFile { // get segment Segment indexSegment = null; - final boolean authorized = sb.verifyAuthentication(header, false); + final boolean authorized = sb.verifyAuthentication(header); if (post != null && post.containsKey("segment") && authorized) { indexSegment = sb.indexSegments.segment(post.get("segment")); } else { @@ -145,7 +145,7 @@ public class ViewFile { // this call forces the peer to download web pages // it is therefore protected by the admin password - if (!sb.verifyAuthentication(header, false)) { + if (!sb.verifyAuthentication(header)) { prop.put("AUTHENTICATE", "admin log-in"); // force log-in return prop; } diff --git a/htroot/ViewImage.java b/htroot/ViewImage.java index 363358d3b..eee86329e 100644 --- a/htroot/ViewImage.java +++ b/htroot/ViewImage.java @@ -67,7 +67,7 @@ public class ViewImage { String urlString = post.get("url", ""); final String urlLicense = post.get("code", ""); - final boolean auth = (header.get(HeaderFramework.CONNECTION_PROP_CLIENTIP, "")).equals("localhost") || sb.verifyAuthentication(header, true); // handle access rights + final boolean auth = (header.get(HeaderFramework.CONNECTION_PROP_CLIENTIP, "")).equals("localhost") || sb.verifyAuthentication(header); // handle access rights DigestURI url = null; if ((urlString.length() > 0) && (auth)) try { diff --git a/htroot/Wiki.java b/htroot/Wiki.java index 6a718807d..ef189bcdd 100644 --- a/htroot/Wiki.java +++ b/htroot/Wiki.java @@ -1,4 +1,4 @@ -// Wiki.java +// Wiki.java // ----------------------- // part of the AnomicHTTPD caching proxy // (C) by Michael Peter Christen; mc@yacy.net @@ -36,6 +36,7 @@ import java.util.Date; import java.util.HashMap; import java.util.Iterator; import java.util.Locale; +import java.util.Map; import net.yacy.cora.document.UTF8; import net.yacy.cora.protocol.HeaderFramework; @@ -43,12 +44,10 @@ import net.yacy.cora.protocol.RequestHeader; import net.yacy.kelondro.util.ByteBuffer; import net.yacy.peers.NewsPool; import net.yacy.search.Switchboard; - import de.anomic.data.Diff; import de.anomic.data.wiki.WikiBoard; import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; -import java.util.Map; public class Wiki { private static final String ANONYMOUS = "anonymous"; @@ -71,7 +70,7 @@ public class Wiki { } prop.put("topmenu", sb.getConfigBool("publicTopmenu", true) ? 1 : 0); - + String access = sb.getConfig("WikiAccess", "admin"); final String pagename = get(post, "page", "start"); final String ip = get(post, HeaderFramework.CONNECTION_PROP_CLIENTIP, "127.0.0.1"); @@ -82,15 +81,15 @@ public class Wiki { author = (sb.peers.mySeed() == null) ? ANONYMOUS : sb.peers.mySeed().get("Name", ANONYMOUS); } } - + if (post != null && post.containsKey("access")) { // only the administrator may change the access right - if (!sb.verifyAuthentication(header, true)) { + if (!sb.verifyAuthentication(header)) { // check access right for admin prop.put("AUTHENTICATE", "admin log-in"); // force log-in return prop; } - + access = post.get("access", "admin"); sb.setConfig("WikiAccess", access); } @@ -101,15 +100,15 @@ public class Wiki { } WikiBoard.Entry page = sb.wikiDB.read(pagename); - + if (post != null && post.containsKey("submit")) { - - if ((access.equals("admin") && (!sb.verifyAuthentication(header, true)))) { + + if ((access.equals("admin") && (!sb.verifyAuthentication(header)))) { // check access right for admin prop.put("AUTHENTICATE", "admin log-in"); // force log-in return prop; } - + // store a new page byte[] content; content = UTF8.getBytes(post.get("content", "")); @@ -126,14 +125,14 @@ public class Wiki { prop.putHTML("LOCATION", "/Wiki.html?page=" + pagename); prop.put("LOCATION", prop.get("LOCATION")); } - + if (post != null && post.containsKey("edit")) { - if ((access.equals("admin") && (!sb.verifyAuthentication(header, true)))) { + if ((access.equals("admin") && (!sb.verifyAuthentication(header)))) { // check access right for admin prop.put("AUTHENTICATE", "admin log-in"); // force log-in return prop; } - + prop.put("mode", "1"); //edit prop.putHTML("mode_author", author); prop.putHTML("mode_page-code", UTF8.String(page.page())); @@ -177,7 +176,7 @@ public class Wiki { prop.put("mode", "4"); prop.putHTML("mode_page", pagename); prop.putHTML("mode_error_page", pagename); - + try { final Iterator it = sb.wikiDB.keysBkp(true); WikiBoard.Entry entry; @@ -201,26 +200,26 @@ public class Wiki { count++; } count--; // don't show current version - + if (!oldselected) { // select latest old entry prop.put("mode_error_versions_" + (count - 1) + "_oldselected", "1"); } if (!newselected) { // select latest new entry (== current) prop.put("mode_error_curselected", "1"); } - + if (count == 0) { prop.put("mode_error", "2"); // no entries found } else { prop.put("mode_error_versions", count); } - + entry = sb.wikiDB.read(pagename); if (entry != null) { prop.put("mode_error_curdate", WikiBoard.dateString(entry.date())); prop.put("mode_error_curfdate", dateString(entry.date())); } - + if (nentry == null) { nentry = entry; } @@ -264,7 +263,7 @@ public class Wiki { /** * get key from post, use dflt if (not present or post == null) - * + * * @param post * @param string * @param string2 diff --git a/htroot/YMarks.java b/htroot/YMarks.java index 8fd0019b9..8703d47c2 100644 --- a/htroot/YMarks.java +++ b/htroot/YMarks.java @@ -10,7 +10,7 @@ public class YMarks { final Switchboard sb = (Switchboard) env; final serverObjects prop = new serverObjects(); final UserDB.Entry user = sb.userDB.getUser(header); - final boolean isAdmin = (sb.verifyAuthentication(header, false)); + final boolean isAdmin = (sb.verifyAuthentication(header)); final boolean isAuthUser = user!= null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT); if(isAdmin || isAuthUser) { diff --git a/htroot/api/bookmarks/get_bookmarks.java b/htroot/api/bookmarks/get_bookmarks.java index 900ed2b80..7dca3a29c 100644 --- a/htroot/api/bookmarks/get_bookmarks.java +++ b/htroot/api/bookmarks/get_bookmarks.java @@ -8,15 +8,14 @@ import net.yacy.cora.protocol.HeaderFramework; import net.yacy.cora.protocol.RequestHeader; import net.yacy.document.parser.html.CharacterCoding; import net.yacy.search.Switchboard; - import de.anomic.data.BookmarkHelper; import de.anomic.data.BookmarksDB; import de.anomic.data.UserDB; import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; -public class get_bookmarks { - +public class get_bookmarks { + private static final serverObjects prop = new serverObjects(); private static Switchboard sb = null; private static UserDB.Entry user = null; @@ -29,7 +28,7 @@ public class get_bookmarks { private final static int SORT_SIZE = 2; private final static int SHOW_ALL = -1; */ - + private final static int MAXRESULTS = 10000; // file types and display types @@ -42,14 +41,14 @@ public class get_bookmarks { RSS(3), // .xml (.rss) RDF(4); // .xml - private int value; + private final int value; DisplayType(final int value) { this.value = value; } int getValue() { - return value; + return this.value; } } @@ -58,7 +57,7 @@ public class get_bookmarks { prop.clear(); sb = (Switchboard) env; user = sb.userDB.getUser(header); - isAdmin = (sb.verifyAuthentication(header, true) || user != null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT)); + isAdmin = (sb.verifyAuthentication(header) || user != null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT)); // set user name final String username; @@ -66,7 +65,7 @@ public class get_bookmarks { else if(isAdmin) username="admin"; else username = "unknown"; prop.putHTML("display_user", username); - + // set peer address prop.put("display_address", sb.peers.mySeed().getPublicAddress()); prop.put("display_peer", sb.peers.mySeed().getName()); @@ -78,7 +77,7 @@ public class get_bookmarks { // String sortname = "date"; final String qtype; final String query; - + // check for GET parameters if (post != null){ itemsPerPage = (post.containsKey("rp")) ? post.getInt("rp", MAXRESULTS) : MAXRESULTS; @@ -108,13 +107,13 @@ public class get_bookmarks { itemsPerPage = MAXRESULTS; display = DisplayType.XML.getValue(); } - + int count = 0; int total = 0; int start = 0; final Iterator it; - + if (display == DisplayType.XBEL.getValue()) { String root = "/"; if ("tags".equals(qtype) && !"".equals(query)) { @@ -200,14 +199,14 @@ public class get_bookmarks { prop.put("display_page", page); prop.put("display_total", total); } - + // return rewrite properties return prop; } - + private static int recurseFolders(final Iterator it, String root, int count, final boolean next, final String prev){ - String fn=""; - + String fn=""; + if (next) fn = it.next(); else fn = prev; @@ -220,7 +219,7 @@ public class get_bookmarks { } return count; } - + if (fn.startsWith(("/".equals(root) ? root : root + "/"))) { prop.put("display_xbel_"+count+"_elements", ""); count++; @@ -233,7 +232,7 @@ public class get_bookmarks { if (it.hasNext()) { count = recurseFolders(it, fn, count, true, fn); } - } else { + } else { if (count > 0) { prop.put("display_xbel_"+count+"_elements", ""); count++; @@ -243,14 +242,14 @@ public class get_bookmarks { root = "/"; } count = recurseFolders(it, root, count, false, fn); - } + } return count; } - + private static int print_XBEL(final Iterator bit, int count) { BookmarksDB.Bookmark bookmark = null; Date date; - while(bit.hasNext()){ + while(bit.hasNext()){ bookmark = sb.bookmarksDB.getBookmark(bit.next()); date = new Date(bookmark.getTimeStamp()); prop.put("display_xbel_"+count+"_elements", " it = null; - + // loop through folderList - it = BookmarkHelper.getFolderList(root, sb.bookmarksDB.getTagIterator(isAdmin)); + it = BookmarkHelper.getFolderList(root, sb.bookmarksDB.getTagIterator(isAdmin)); int n = root.split("/").length; if (n == 0) n = 1; int count = 0; - while (it.hasNext()) { - String folder = it.next(); + while (it.hasNext()) { + final String folder = it.next(); foldername = folder.split("/"); if (foldername.length == n+1) { prop.put("folders_"+count+"_foldername", foldername[n]); @@ -69,14 +68,14 @@ public class get_folders { count++; } } - + // loop through bookmarkList - it = sb.bookmarksDB.getBookmarksIterator(root, isAdmin); + it = sb.bookmarksDB.getBookmarksIterator(root, isAdmin); BookmarksDB.Bookmark bm; while (it.hasNext()) { bm = sb.bookmarksDB.getBookmark(it.next()); // TODO: get rid of bmtype - if (post.containsKey("bmtype")) { + if (post.containsKey("bmtype")) { if (post.get("bmtype").equals("title")) { prop.put("folders_"+count+"_foldername", bm.getTitle()); } else if (post.get("bmtype").equals("href")) { @@ -84,22 +83,22 @@ public class get_folders { } else { prop.put("folders_"+count+"_foldername", bm.getUrl()); } - } + } prop.put("folders_"+count+"_expanded", "false"); prop.put("folders_"+count+"_url", bm.getUrl()); prop.put("folders_"+count+"_type", "file"); prop.put("folders_"+count+"_hash", bm.getUrlHash()); prop.put("folders_"+count+"_hasChildren", "false"); prop.put("folders_"+count+"_comma", ","); - count++; - } - + count++; + } + count--; prop.put("folders_"+count+"_comma", ""); count++; prop.put("folders", count); - + // return rewrite properties return prop; - } + } } \ No newline at end of file diff --git a/htroot/api/bookmarks/posts/add_p.java b/htroot/api/bookmarks/posts/add_p.java index 8a98ca284..918a5412b 100755 --- a/htroot/api/bookmarks/posts/add_p.java +++ b/htroot/api/bookmarks/posts/add_p.java @@ -6,7 +6,6 @@ import java.util.Set; import net.yacy.cora.protocol.RequestHeader; import net.yacy.peers.NewsPool; import net.yacy.search.Switchboard; - import de.anomic.data.BookmarkHelper; import de.anomic.data.BookmarksDB; import de.anomic.data.ListManager; @@ -16,23 +15,23 @@ import de.anomic.server.serverSwitch; public class add_p { - + private static final serverObjects prop = new serverObjects(); private static Switchboard sb = null; private static UserDB.Entry user = null; - private static boolean isAdmin = false; - + private static boolean isAdmin = false; + public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { - - sb = (Switchboard) env; - isAdmin=sb.verifyAuthentication(header, true); + + sb = (Switchboard) env; + isAdmin=sb.verifyAuthentication(header); user = sb.userDB.getUser(header); - + // set user name - String username=""; + String username=""; if(user != null) username=user.getUserName(); else if(isAdmin) username="admin"; - + if (post != null) { if (!isAdmin) { // force authentication if desired @@ -45,7 +44,7 @@ public class add_p { final String title=post.get("title",url); final String description=post.get("description",""); String tagsString = post.get("tags",""); - String pathString = post.get("path","/unsorted"); + final String pathString = post.get("path","/unsorted"); tagsString= tagsString + "," + pathString; final Set tags = ListManager.string2set(BookmarkHelper.cleanTagsString(tagsString)); final BookmarksDB.Bookmark bookmark = sb.bookmarksDB.createBookmark(url, username); diff --git a/htroot/api/bookmarks/posts/all.java b/htroot/api/bookmarks/posts/all.java index 439b86e70..1f8c9bd1f 100755 --- a/htroot/api/bookmarks/posts/all.java +++ b/htroot/api/bookmarks/posts/all.java @@ -6,7 +6,6 @@ import net.yacy.cora.date.ISO8601Formatter; import net.yacy.cora.protocol.RequestHeader; import net.yacy.kelondro.order.Digest; import net.yacy.search.Switchboard; - import de.anomic.data.BookmarksDB; import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; @@ -15,19 +14,19 @@ public class all { public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { // return variable that accumulates replacements final Switchboard switchboard = (Switchboard) env; - final boolean isAdmin=switchboard.verifyAuthentication(header, true); + final boolean isAdmin=switchboard.verifyAuthentication(header); final serverObjects prop = new serverObjects(); - + Iterator it; if(post != null && post.containsKey("tag")){ it=switchboard.bookmarksDB.getBookmarksIterator(post.get("tag"), isAdmin); }else{ it=switchboard.bookmarksDB.getBookmarksIterator(isAdmin); } - + // if an extended xml should be used final boolean extendedXML = (post != null && post.containsKey("extendedXML")); - + int count=0; BookmarksDB.Bookmark bookmark; Date date; @@ -40,7 +39,7 @@ public class all { date = new Date(bookmark.getTimeStamp()); prop.putXML("posts_"+count+"_time", ISO8601Formatter.FORMATTER.format(date)); prop.putXML("posts_"+count+"_tags", bookmark.getTagsString().replaceAll(","," ")); - + // additional XML tags prop.put("posts_"+count+"_isExtended",extendedXML ? "1" : "0"); if (extendedXML) { @@ -53,5 +52,5 @@ public class all { // return rewrite properties return prop; } - + } diff --git a/htroot/api/bookmarks/posts/delete_p.java b/htroot/api/bookmarks/posts/delete_p.java index fcfae1a5a..1e838e477 100755 --- a/htroot/api/bookmarks/posts/delete_p.java +++ b/htroot/api/bookmarks/posts/delete_p.java @@ -5,7 +5,6 @@ import net.yacy.cora.document.ASCII; import net.yacy.cora.protocol.RequestHeader; import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.search.Switchboard; - import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; @@ -14,7 +13,7 @@ public class delete_p { // return variable that accumulates replacements final Switchboard switchboard = (Switchboard) env; final serverObjects prop = new serverObjects(); - final boolean isAdmin=switchboard.verifyAuthentication(header, true); + final boolean isAdmin=switchboard.verifyAuthentication(header); if(post!= null){ if(!isAdmin){ // force authentication if desired @@ -22,7 +21,7 @@ public class delete_p { prop.put("AUTHENTICATE","admin log-in"); } return prop; - } + } try { if (post.containsKey("url") && switchboard.bookmarksDB.removeBookmark(ASCII.String((new DigestURI(post.get("url", "nourl"))).hash()))) { prop.put("result", "1"); @@ -36,10 +35,10 @@ public class delete_p { } }else{ prop.put("result", "0"); - } + } // return rewrite properties return prop; - } + } } diff --git a/htroot/api/bookmarks/posts/get.java b/htroot/api/bookmarks/posts/get.java index db221b591..785f358ad 100755 --- a/htroot/api/bookmarks/posts/get.java +++ b/htroot/api/bookmarks/posts/get.java @@ -16,12 +16,12 @@ public class get { public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { // return variable that accumulates replacements final Switchboard switchboard = (Switchboard) env; - final boolean isAdmin=switchboard.verifyAuthentication(header, true); + final boolean isAdmin=switchboard.verifyAuthentication(header); final serverObjects prop = new serverObjects(); String tag = null; final String date; //String url=""; //urlfilter not yet implemented - + if (post != null && post.containsKey("tag")) { tag = post.get("tag"); } @@ -30,19 +30,19 @@ public class get { } else { date = ISO8601Formatter.FORMATTER.format(); } - + // if an extended xml should be used final boolean extendedXML = (post != null && post.containsKey("extendedXML")); - + int count=0; - - Date parsedDate = null; + + Date parsedDate = null; try { parsedDate = ISO8601Formatter.FORMATTER.parse(date); } catch (final ParseException e) { parsedDate = new Date(); } - + final List bookmark_hashes = switchboard.bookmarksDB.getDate(Long.toString(parsedDate.getTime())).getBookmarkList(); BookmarksDB.Bookmark bookmark = null; for (final String bookmark_hash : bookmark_hashes){ @@ -56,7 +56,7 @@ public class get { prop.put("posts_"+count+"_md5", Digest.encodeMD5Hex(bookmark.getUrl())); prop.put("posts_"+count+"_time", date); prop.putHTML("posts_"+count+"_tags", bookmark.getTagsString().replaceAll(","," ")); - + // additional XML tags prop.put("posts_"+count+"_isExtended",extendedXML ? "1" : "0"); if (extendedXML) { @@ -70,5 +70,5 @@ public class get { // return rewrite properties return prop; } - + } diff --git a/htroot/api/bookmarks/tags/addTag_p.java b/htroot/api/bookmarks/tags/addTag_p.java index 057b7c5e1..5b42053a6 100644 --- a/htroot/api/bookmarks/tags/addTag_p.java +++ b/htroot/api/bookmarks/tags/addTag_p.java @@ -1,9 +1,9 @@ import net.yacy.cora.protocol.RequestHeader; import net.yacy.search.Switchboard; +import de.anomic.data.BookmarksDB.Bookmark; import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; -import de.anomic.data.BookmarksDB.Bookmark; public class addTag_p { @@ -12,22 +12,22 @@ public class addTag_p { final Switchboard switchboard = (Switchboard) env; final serverObjects prop = new serverObjects(); boolean isAdmin = false; - isAdmin = switchboard.verifyAuthentication(header, true); - + isAdmin = switchboard.verifyAuthentication(header); + prop.put("result", "0");//error //rename tags if(post != null && isAdmin) { if (post.containsKey("selectTag") && post.containsKey("addTag")) { switchboard.bookmarksDB.addTag(post.get("selectTag"), post.get("addTag")); - prop.put("result", "1");//success + prop.put("result", "1");//success } else if (post.containsKey("urlhash") && post.containsKey("addTag")) { final Bookmark bm = switchboard.bookmarksDB.getBookmark(post.get("urlhash")); bm.addTag(post.get("addTag")); - prop.put("result", "1");//success + prop.put("result", "1");//success } - } + } // return rewrite properties return prop; } - + } \ No newline at end of file diff --git a/htroot/api/bookmarks/tags/editTag_p.java b/htroot/api/bookmarks/tags/editTag_p.java index d3aa932b1..e70f7b3f4 100755 --- a/htroot/api/bookmarks/tags/editTag_p.java +++ b/htroot/api/bookmarks/tags/editTag_p.java @@ -6,12 +6,12 @@ import de.anomic.server.serverSwitch; public class editTag_p { public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { - - final Switchboard switchboard = (Switchboard) env; + + final Switchboard switchboard = (Switchboard) env; final serverObjects prop = new serverObjects(); boolean isAdmin = false; - isAdmin = switchboard.verifyAuthentication(header, true); - + isAdmin = switchboard.verifyAuthentication(header); + prop.put("result", "0");//error //rename tags if(post != null && isAdmin && post.containsKey("old") && post.containsKey("new")){ @@ -21,5 +21,5 @@ public class editTag_p { // return rewrite properties return prop; } - + } diff --git a/htroot/api/bookmarks/tags/getTag.java b/htroot/api/bookmarks/tags/getTag.java index c4bb49dfe..3135e32f0 100755 --- a/htroot/api/bookmarks/tags/getTag.java +++ b/htroot/api/bookmarks/tags/getTag.java @@ -3,7 +3,6 @@ import java.util.Iterator; import net.yacy.cora.protocol.RequestHeader; import net.yacy.search.Switchboard; - import de.anomic.data.BookmarksDB; import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; @@ -12,18 +11,18 @@ public class getTag { final static int SORT_ALPHA = 1; final static int SORT_SIZE = 2; final static int SHOW_ALL = -1; - + public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { // return variable that accumulates replacements final Switchboard switchboard = (Switchboard) env; - final boolean isAdmin = switchboard.verifyAuthentication(header, true); + final boolean isAdmin = switchboard.verifyAuthentication(header); final serverObjects prop = new serverObjects(); Iterator it = null; String tagName = ""; int top = SHOW_ALL; int comp = SORT_ALPHA; - - + + if (post != null) { if (!isAdmin) { // force authentication if desired @@ -43,16 +42,16 @@ public class getTag { } if (post != null && post.containsKey("tag")) { - tagName=post.get("tag"); + tagName=post.get("tag"); if (!tagName.isEmpty()) { - it = switchboard.bookmarksDB.getTagIterator(tagName, isAdmin, comp, top); - } + it = switchboard.bookmarksDB.getTagIterator(tagName, isAdmin, comp, top); + } } else { it = switchboard.bookmarksDB.getTagIterator(isAdmin, comp, top); - } + } // Iterator it = switchboard.bookmarksDB.getTagIterator(isAdmin); - + int count = 0; if (it != null) { BookmarksDB.Tag tag; @@ -70,5 +69,5 @@ public class getTag { // return rewrite properties return prop; } - + } diff --git a/htroot/api/bookmarks/xbel/xbel.java b/htroot/api/bookmarks/xbel/xbel.java index bdea87e9b..8c682542c 100755 --- a/htroot/api/bookmarks/xbel/xbel.java +++ b/htroot/api/bookmarks/xbel/xbel.java @@ -18,16 +18,16 @@ public class xbel { private static Switchboard switchboard = null; private static boolean isAdmin = false; private static int R = 1; // TODO: solve the recursion problem an remove global variable - + public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { - + int count = 0;; String root = "/"; int style = 0; - + prop.clear(); - switchboard = (Switchboard) env; - isAdmin=switchboard.verifyAuthentication(header, true); + switchboard = (Switchboard) env; + isAdmin=switchboard.verifyAuthentication(header); if(post != null) { if(!isAdmin) { @@ -36,7 +36,7 @@ public class xbel { } } if(post.containsKey("tag")) { - final String tagName=post.get("tag"); + final String tagName=post.get("tag"); prop.putHTML("folder", tagName); if (!tagName.equals("")) { final Iterator bit=switchboard.bookmarksDB.getBookmarksIterator(tagName, isAdmin); @@ -47,98 +47,98 @@ public class xbel { } if(post.containsKey("folder")) { final String folderName=post.get("folder"); - if (folderName.length() > 0 && folderName.charAt(0) == '/') { root = folderName; } + if (folderName.length() > 0 && folderName.charAt(0) == '/') { root = folderName; } else { root = "/" + folderName; } } if(post.containsKey("style") && !post.get("style").equals("")) { - style = 1; - prop.putHTML("style_href", post.get("style")); - prop.putHTML("style_type", post.get("style").replaceAll("^.*\\.", "")); - } + style = 1; + prop.putHTML("style_href", post.get("style")); + prop.putHTML("style_type", post.get("style").replaceAll("^.*\\.", "")); + } } prop.put("style", style); R = root.replaceAll("[^/]","").length() - 1; count = recurseFolders(BookmarkHelper.getFolderList(root, switchboard.bookmarksDB.getTagIterator(isAdmin)),root,0,true,""); prop.put("xbel", count); - return prop; // return from serverObjects respond() + return prop; // return from serverObjects respond() } private static int recurseFolders(final Iterator it, String root, int count, final boolean next, final String prev){ - String fn=""; - - if(next) fn = it.next(); + String fn=""; + + if(next) fn = it.next(); else fn = prev; if(fn.equals("\uffff")) { - int i = prev.replaceAll("[^/]","").length() - R; + int i = prev.replaceAll("[^/]","").length() - R; while(i>0){ prop.put("xbel_"+count+"_elements", ""); count++; i--; - } + } return count; } - + if(fn.startsWith((root.equals("/") ? root : root+"/"))){ prop.put("xbel_"+count+"_elements", ""); count++; - - final String title = fn; // just to make sure fn stays untouched - prop.put("xbel_"+count+"_elements", "" + CharacterCoding.unicode2xml(title.replaceAll("(/.[^/]*)*/", ""), true) + ""); - count++; + + final String title = fn; // just to make sure fn stays untouched + prop.put("xbel_"+count+"_elements", "" + CharacterCoding.unicode2xml(title.replaceAll("(/.[^/]*)*/", ""), true) + ""); + count++; final Iterator bit=switchboard.bookmarksDB.getBookmarksIterator(fn, isAdmin); count = print_XBEL(bit, count); if(it.hasNext()){ count = recurseFolders(it, fn, count, true, fn); } - } else { + } else { if (count > 0) { prop.put("xbel_"+count+"_elements", ""); count++; } - root = root.replaceAll("(/.[^/]*$)", ""); - if(root.equals("")) root = "/"; + root = root.replaceAll("(/.[^/]*$)", ""); + if(root.equals("")) root = "/"; count = recurseFolders(it, root, count, false, fn); - } + } return count; } private static int print_XBEL(final Iterator bit, int count) { BookmarksDB.Bookmark bookmark; Date date; - while(bit.hasNext()){ + while(bit.hasNext()){ bookmark=switchboard.bookmarksDB.getBookmark(bit.next()); date=new Date(bookmark.getTimeStamp()); prop.put("xbel_"+count+"_elements", ""); - count++; + count++; prop.put("xbel_"+count+"_elements", ""); count++; - prop.putXML("xbel_"+count+"_elements", bookmark.getTitle()); - count++; + prop.putXML("xbel_"+count+"_elements", bookmark.getTitle()); + count++; prop.put("xbel_"+count+"_elements", ""); count++; - prop.put("xbel_"+count+"_elements", ""); + prop.put("xbel_"+count+"_elements", ""); count++; prop.put("xbel_"+count+"_elements", ""); + + "\"/>"); count++; - prop.put("xbel_"+count+"_elements", ""); + prop.put("xbel_"+count+"_elements", ""); count++; - prop.put("xbel_"+count+"_elements", ""); + prop.put("xbel_"+count+"_elements", ""); count++; prop.put("xbel_"+count+"_elements", ""); count++; - prop.putXML("xbel_"+count+"_elements", bookmark.getDescription()); - count++; + prop.putXML("xbel_"+count+"_elements", bookmark.getDescription()); + count++; prop.put("xbel_"+count+"_elements", ""); count++; - prop.put("xbel_"+count+"_elements", ""); - count++; + prop.put("xbel_"+count+"_elements", ""); + count++; } return count; - } + } } diff --git a/htroot/api/feed.java b/htroot/api/feed.java index c8150304a..7b96dc81e 100755 --- a/htroot/api/feed.java +++ b/htroot/api/feed.java @@ -23,7 +23,7 @@ public class feed { prop.put("item", "0"); if ((post == null) || (env == null)) return prop; - final boolean authorized = sb.verifyAuthentication(header, false); + final boolean authorized = sb.verifyAuthentication(header); final String channelNames = post.get("set"); if (channelNames == null) return prop; diff --git a/htroot/api/queues_p.java b/htroot/api/queues_p.java index 8e355e0ce..e630ccbdd 100755 --- a/htroot/api/queues_p.java +++ b/htroot/api/queues_p.java @@ -11,46 +11,45 @@ import net.yacy.search.Switchboard; import net.yacy.search.SwitchboardConstants; import net.yacy.search.index.Segment; import net.yacy.search.index.Segments; - import de.anomic.crawler.NoticedURL; import de.anomic.crawler.retrieval.Request; import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; public class queues_p { - + public static final String STATE_RUNNING = "running"; public static final String STATE_PAUSED = "paused"; - + private static SimpleDateFormat dayFormatter = new SimpleDateFormat("yyyy/MM/dd", Locale.US); private static String daydate(final Date date) { if (date == null) return ""; return dayFormatter.format(date); } - + public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { // return variable that accumulates replacements final Switchboard sb = (Switchboard) env; //wikiCode wikiTransformer = new wikiCode(switchboard); final serverObjects prop = new serverObjects(); Segment segment = null; - boolean html = post != null && post.containsKey("html"); + final boolean html = post != null && post.containsKey("html"); prop.setLocalized(html); - if (post != null && post.containsKey("segment") && sb.verifyAuthentication(header, false)) { + if (post != null && post.containsKey("segment") && sb.verifyAuthentication(header)) { segment = sb.indexSegments.segment(post.get("segment")); } if (segment == null) segment = sb.indexSegments.segment(Segments.Process.PUBLIC); prop.put("rejected", "0"); //int showRejectedCount = 10; - + Seed initiator; - + // index size prop.putNum("urlpublictextSize", segment.urlMetadata().size()); prop.putNum("rwipublictextSize", segment.termIndex().sizesMax()); // loader queue - prop.putNum("loaderSize", sb.crawlQueues.workerSize()); + prop.putNum("loaderSize", sb.crawlQueues.workerSize()); prop.putNum("loaderMax", sb.getConfigLong(SwitchboardConstants.CRAWLER_THREADS_ACTIVE_MAX, 10)); if (sb.crawlQueues.workerSize() == 0) { prop.put("list-loader", "0"); @@ -68,7 +67,7 @@ public class queues_p { } prop.put("list-loader", count); } - + //local crawl queue prop.putNum("localCrawlSize", sb.getThread(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL).getJobCount()); prop.put("localCrawlState", sb.crawlJobIsPaused(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL) ? STATE_PAUSED : STATE_RUNNING); @@ -94,8 +93,8 @@ public class queues_p { // return rewrite properties return prop; } - - + + public static final void addNTable(final Switchboard sb, final serverObjects prop, final String tableName, final List crawlerList) { int showNum = 0; diff --git a/htroot/api/status_p.java b/htroot/api/status_p.java index cd42560eb..0faad5718 100644 --- a/htroot/api/status_p.java +++ b/htroot/api/status_p.java @@ -2,8 +2,6 @@ import net.yacy.cora.protocol.RequestHeader; import net.yacy.kelondro.io.ByteCount; -//import net.yacy.kelondro.io.ByteCountInputStream; -//import net.yacy.kelondro.io.ByteCountOutputStream; import net.yacy.kelondro.util.MemoryControl; import net.yacy.kelondro.workflow.WorkflowProcessor; import net.yacy.search.Switchboard; @@ -14,20 +12,20 @@ import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; public class status_p { - - + + public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { // return variable that accumulates replacements final Switchboard sb = (Switchboard) env; final serverObjects prop = new serverObjects(); Segment segment = null; - boolean html = post != null && post.containsKey("html"); + final boolean html = post != null && post.containsKey("html"); prop.setLocalized(html); - if (post != null && post.containsKey("segment") && sb.verifyAuthentication(header, false)) { + if (post != null && post.containsKey("segment") && sb.verifyAuthentication(header)) { segment = sb.indexSegments.segment(post.get("segment")); } if (segment == null) segment = sb.indexSegments.segment(Segments.Process.PUBLIC); - + prop.put("rejected", "0"); sb.updateMySeed(); final int cacheMaxSize = (int) sb.getConfigLong(SwitchboardConstants.WORDCACHE_MAX_COUNT, 10000); @@ -35,14 +33,14 @@ public class status_p { prop.putNum("qpm", sb.peers.mySeed().getQPM()); prop.putNum("wordCacheSize", segment.termIndex().getBufferSize()); prop.putNum("wordCacheMaxSize", cacheMaxSize); - + // crawl queues prop.putNum("localCrawlSize", sb.getThread(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL).getJobCount()); prop.putNum("limitCrawlSize", sb.crawlQueues.limitCrawlJobSize()); prop.putNum("remoteCrawlSize", sb.getThread(SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL).getJobCount()); - prop.putNum("loaderSize", sb.crawlQueues.workerSize()); + prop.putNum("loaderSize", sb.crawlQueues.workerSize()); prop.putNum("loaderMax", sb.getConfigLong(SwitchboardConstants.CRAWLER_THREADS_ACTIVE_MAX, 10)); - + // memory usage and system attributes prop.putNum("freeMemory", MemoryControl.free()); prop.putNum("totalMemory", MemoryControl.total()); @@ -57,5 +55,5 @@ public class status_p { // return rewrite properties return prop; } - + } diff --git a/htroot/api/termlist_p.java b/htroot/api/termlist_p.java index 3b1e3b40e..f98c87002 100644 --- a/htroot/api/termlist_p.java +++ b/htroot/api/termlist_p.java @@ -46,7 +46,7 @@ public class termlist_p { Segment segment = null; final boolean delete = post != null && post.containsKey("delete"); final long mincount = post == null ? 10000 : post.getLong("mincount", 10000); - if (post != null && post.containsKey("segment") && sb.verifyAuthentication(header, false)) { + if (post != null && post.containsKey("segment") && sb.verifyAuthentication(header)) { segment = sb.indexSegments.segment(post.get("segment")); } if (segment == null) segment = sb.indexSegments.segment(Segments.Process.PUBLIC); diff --git a/htroot/api/yacydoc.java b/htroot/api/yacydoc.java index 9172d3b73..1599243ea 100644 --- a/htroot/api/yacydoc.java +++ b/htroot/api/yacydoc.java @@ -10,7 +10,7 @@ // $LastChangedBy: orbiter $ // // LICENSE -// +// // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by // the Free Software Foundation; either version 2 of the License, or @@ -38,27 +38,26 @@ import net.yacy.kelondro.logging.Log; import net.yacy.search.Switchboard; import net.yacy.search.index.Segment; import net.yacy.search.index.Segments; - import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; public class yacydoc { - + public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { // return variable that accumulates replacements final Switchboard sb = (Switchboard) env; - + final serverObjects prop = new serverObjects(); final Segment segment; - boolean html = post != null && post.containsKey("html"); + final boolean html = post != null && post.containsKey("html"); prop.setLocalized(html); - boolean authorized = sb.verifyAuthentication(header, false); + final boolean authorized = sb.verifyAuthentication(header); if (post != null && post.containsKey("segment") && authorized) { segment = sb.indexSegments.segment(post.get("segment")); } else { segment = sb.indexSegments.segment(Segments.Process.PUBLIC); } - + prop.put("dc_title", ""); prop.put("dc_creator", ""); prop.put("dc_description", ""); @@ -71,21 +70,21 @@ public class yacydoc { prop.put("dc_language", ""); if (post == null) return prop; - - String urlstring = post.get("url", "").trim(); + + final String urlstring = post.get("url", "").trim(); String urlhash = post.get("urlhash", "").trim(); if (urlstring.length() == 0 && urlhash.length() == 0) return prop; if (urlstring.length() > 0 && urlhash.length() == 0) { try { - DigestURI url = new DigestURI(urlstring); + final DigestURI url = new DigestURI(urlstring); urlhash = ASCII.String(url.hash()); - } catch (MalformedURLException e) { + } catch (final MalformedURLException e) { Log.logException(e); } } if (urlhash == null || urlhash.length() == 0) return prop; - + final URIMetadataRow entry = segment.urlMetadata().load(urlhash.getBytes()); if (entry == null) return prop; @@ -94,7 +93,7 @@ public class yacydoc { return prop; } final URIMetadataRow le = (entry.referrerHash() == null || entry.referrerHash().length != Word.commonHashLength) ? null : segment.urlMetadata().load(entry.referrerHash()); - + prop.putXML("dc_title", metadata.dc_title()); prop.putXML("dc_creator", metadata.dc_creator()); prop.putXML("dc_description", ""); // this is the fulltext part in the surrogate @@ -114,7 +113,7 @@ public class yacydoc { prop.putXML("yacy_referrer_url", (le == null) ? "" : le.metadata().url().toNormalform(false, true)); prop.put("yacy_size", entry.size()); prop.put("yacy_words",entry.wordCount()); - + // return rewrite properties return prop; } diff --git a/htroot/api/ymarks/add_ymark.java b/htroot/api/ymarks/add_ymark.java index 77f137d34..60f3b6669 100644 --- a/htroot/api/ymarks/add_ymark.java +++ b/htroot/api/ymarks/add_ymark.java @@ -1,4 +1,5 @@ import java.io.IOException; + import net.yacy.cora.protocol.RequestHeader; import net.yacy.document.Parser.Failure; import net.yacy.kelondro.data.meta.DigestURI; @@ -15,15 +16,15 @@ import de.anomic.server.serverSwitch; public class add_ymark { - + public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { final Switchboard sb = (Switchboard) env; final serverObjects prop = new serverObjects(); - + final UserDB.Entry user = sb.userDB.getUser(header); - final boolean isAdmin = (sb.verifyAuthentication(header, true)); + final boolean isAdmin = (sb.verifyAuthentication(header)); final boolean isAuthUser = user!= null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT); - + if(isAdmin || isAuthUser) { final String bmk_user = (isAuthUser ? user.getUserName() : YMarkTables.USER_ADMIN); @@ -31,30 +32,30 @@ public class add_ymark { prop.put("redirect_url", post.get("redirect")); prop.put("redirect", "1"); } - + if(post.containsKey("urlHash")) { final String urlHash = post.get("urlHash",YMarkUtil.EMPTY_STRING); - final DigestURI url = sb.indexSegments.segment(Segments.Process.PUBLIC).urlMetadata().load(urlHash.getBytes()).metadata().url(); + final DigestURI url = sb.indexSegments.segment(Segments.Process.PUBLIC).urlMetadata().load(urlHash.getBytes()).metadata().url(); final String folders = post.get(YMarkEntry.BOOKMARK.FOLDERS.key(),YMarkEntry.FOLDERS_UNSORTED); final String tags = post.get(YMarkEntry.BOOKMARK.TAGS.key(),YMarkUtil.EMPTY_STRING); try { sb.tables.bookmarks.createBookmark(sb.loader, url, bmk_user, true, tags, folders); prop.put("status", "1"); - } catch (IOException e) { + } catch (final IOException e) { // TODO Auto-generated catch block Log.logException(e); - } catch (Failure e) { + } catch (final Failure e) { // TODO Auto-generated catch block Log.logException(e); - } catch (RowSpaceExceededException e) { + } catch (final RowSpaceExceededException e) { // TODO Auto-generated catch block Log.logException(e); } - + } else if(post.containsKey(YMarkEntry.BOOKMARK.URL.key())) { String url = post.get(YMarkEntry.BOOKMARK.URL.key(),YMarkEntry.BOOKMARK.URL.deflt()); boolean hasProtocol = false; - for (YMarkTables.PROTOCOLS p : YMarkTables.PROTOCOLS.values()) { + for (final YMarkTables.PROTOCOLS p : YMarkTables.PROTOCOLS.values()) { if(url.toLowerCase().startsWith(p.protocol())) { hasProtocol = true; break; @@ -63,22 +64,22 @@ public class add_ymark { if (!hasProtocol) { url=YMarkTables.PROTOCOLS.HTTP.protocol(url); } - - final YMarkEntry bmk = new YMarkEntry(); - + + final YMarkEntry bmk = new YMarkEntry(); + bmk.put(YMarkEntry.BOOKMARK.URL.key(), url); bmk.put(YMarkEntry.BOOKMARK.TITLE.key(), post.get(YMarkEntry.BOOKMARK.TITLE.key(),YMarkEntry.BOOKMARK.TITLE.deflt())); bmk.put(YMarkEntry.BOOKMARK.DESC.key(), post.get(YMarkEntry.BOOKMARK.DESC.key(),YMarkEntry.BOOKMARK.DESC.deflt())); bmk.put(YMarkEntry.BOOKMARK.PUBLIC.key(), post.get(YMarkEntry.BOOKMARK.PUBLIC.key(),YMarkEntry.BOOKMARK.PUBLIC.deflt())); bmk.put(YMarkEntry.BOOKMARK.TAGS.key(), YMarkUtil.cleanTagsString(post.get(YMarkEntry.BOOKMARK.TAGS.key(),YMarkEntry.BOOKMARK.TAGS.deflt()))); bmk.put(YMarkEntry.BOOKMARK.FOLDERS.key(), YMarkUtil.cleanFoldersString(post.get(YMarkEntry.BOOKMARK.FOLDERS.key(),YMarkEntry.FOLDERS_UNSORTED))); - + try { sb.tables.bookmarks.addBookmark(bmk_user, bmk, false, false); - } catch (IOException e) { + } catch (final IOException e) { Log.logException(e); - } catch (RowSpaceExceededException e) { - } + } catch (final RowSpaceExceededException e) { + } prop.put("status", "1"); } else { prop.put("status", "0"); diff --git a/htroot/api/ymarks/delete_ymark.java b/htroot/api/ymarks/delete_ymark.java index bf8d74c14..e93a6160f 100644 --- a/htroot/api/ymarks/delete_ymark.java +++ b/htroot/api/ymarks/delete_ymark.java @@ -13,18 +13,18 @@ import de.anomic.server.serverSwitch; public class delete_ymark { - + private static Switchboard sb = null; - + public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { sb = (Switchboard) env; final serverObjects prop = new serverObjects(); - + final UserDB.Entry user = sb.userDB.getUser(header); - final boolean isAdmin = (sb.verifyAuthentication(header, true)); + final boolean isAdmin = (sb.verifyAuthentication(header)); final boolean isAuthUser = user!= null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT); - - if(isAdmin || isAuthUser) { + + if(isAdmin || isAuthUser) { final String bmk_user = (isAuthUser ? user.getUserName() : YMarkTables.USER_ADMIN); byte[] urlHash = null; try { @@ -38,14 +38,14 @@ public class delete_ymark { } sb.tables.bookmarks.deleteBookmark(bmk_user, urlHash); prop.put("result", "1"); - } catch (IOException e) { + } catch (final IOException e) { Log.logException(e); - } catch (RowSpaceExceededException e) { + } catch (final RowSpaceExceededException e) { Log.logException(e); } } else { prop.put(YMarkTables.USER_AUTHENTICATE,YMarkTables.USER_AUTHENTICATE_MSG); - } + } // return rewrite properties return prop; } diff --git a/htroot/api/ymarks/get_metadata.java b/htroot/api/ymarks/get_metadata.java index eb770f9f9..daf167313 100644 --- a/htroot/api/ymarks/get_metadata.java +++ b/htroot/api/ymarks/get_metadata.java @@ -18,25 +18,25 @@ import de.anomic.data.ymark.YMarkUtil; import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; -public class get_metadata { - +public class get_metadata { + static serverObjects prop; - + public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { final Switchboard sb = (Switchboard) env; - prop = new serverObjects(); + prop = new serverObjects(); final UserDB.Entry user = sb.userDB.getUser(header); - final boolean isAdmin = (sb.verifyAuthentication(header, true)); - final boolean isAuthUser = user!= null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT); - + final boolean isAdmin = (sb.verifyAuthentication(header)); + final boolean isAuthUser = user!= null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT); + if(isAdmin || isAuthUser) { - final String bmk_user = (isAuthUser ? user.getUserName() : YMarkTables.USER_ADMIN); - + final String bmk_user = (isAuthUser ? user.getUserName() : YMarkTables.USER_ADMIN); + String url = post.get(YMarkEntry.BOOKMARK.URL.key(),YMarkEntry.BOOKMARK.URL.deflt()); boolean hasProtocol = false; - for (YMarkTables.PROTOCOLS p : YMarkTables.PROTOCOLS.values()) { + for (final YMarkTables.PROTOCOLS p : YMarkTables.PROTOCOLS.values()) { if(url.toLowerCase().startsWith(p.protocol())) { hasProtocol = true; break; @@ -45,17 +45,17 @@ public class get_metadata { if (!hasProtocol) { url=YMarkTables.PROTOCOLS.HTTP.protocol(url); } - - try { - YMarkMetadata meta = new YMarkMetadata(new DigestURI(url), sb.indexSegments); + + try { + final YMarkMetadata meta = new YMarkMetadata(new DigestURI(url), sb.indexSegments); final Document document = meta.loadDocument(sb.loader); final EnumMap metadata = meta.loadMetadata(); - + prop.putXML("title", metadata.get(YMarkMetadata.METADATA.TITLE)); - prop.putXML("desc", metadata.get(YMarkMetadata.METADATA.DESCRIPTION)); + prop.putXML("desc", metadata.get(YMarkMetadata.METADATA.DESCRIPTION)); prop.put("keywords", putTags(document.dc_subject(','), "keywords")); prop.put("autotags", putTags(YMarkAutoTagger.autoTag(document, 5, sb.tables.bookmarks.getTags(bmk_user)), "autotags")); - + final YMarkCrawlStart crawlStart = new YMarkCrawlStart(sb.tables, url); int count = 0; if(!crawlStart.isEmpty()) { @@ -70,23 +70,23 @@ public class get_metadata { } prop.put("crawlstart", count); - } catch (MalformedURLException e1) { + } catch (final MalformedURLException e1) { // TODO Auto-generated catch block e1.printStackTrace(); - } catch (IOException e) { + } catch (final IOException e) { // TODO Auto-generated catch block e.printStackTrace(); - } catch (Failure e) { + } catch (final Failure e) { // TODO Auto-generated catch block e.printStackTrace(); - } + } } else { prop.put(YMarkTables.USER_AUTHENTICATE,YMarkTables.USER_AUTHENTICATE_MSG); - } + } // return rewrite properties return prop; } - + public static int putTags(final String tagString, final String var) { final String list[] = tagString.split(YMarkUtil.TAGS_SEPARATOR); int count = 0; diff --git a/htroot/api/ymarks/get_tags.java b/htroot/api/ymarks/get_tags.java index 64d05d118..df7a00993 100644 --- a/htroot/api/ymarks/get_tags.java +++ b/htroot/api/ymarks/get_tags.java @@ -17,25 +17,25 @@ import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; public class get_tags { - + final static String TAG = "tag"; final static String TOP = "top"; final static String SORT = "sort"; final static String SIZE = "size"; final static String ALPHA = "alpha"; - - + + private static Switchboard sb = null; private static serverObjects prop = null; - + public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { sb = (Switchboard) env; prop = new serverObjects(); - + final UserDB.Entry user = sb.userDB.getUser(header); - final boolean isAdmin = (sb.verifyAuthentication(header, true)); + final boolean isAdmin = (sb.verifyAuthentication(header)); final boolean isAuthUser = user!= null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT); - + if(isAdmin || isAuthUser) { final String bmk_user = (isAuthUser ? user.getUserName() : YMarkTables.USER_ADMIN); Integer top = Integer.MAX_VALUE; @@ -49,13 +49,13 @@ public class get_tags { final String[] tagArray = YMarkUtil.cleanTagsString(post.get(TAG)).split(YMarkUtil.TAGS_SEPARATOR); try { tags = new TreeSet(sb.tables.bookmarks.getTags(sb.tables.bookmarks.getBookmarksByTag(bmk_user, tagArray)).values()); - } catch (IOException e) { + } catch (final IOException e) { return prop; } } else { try { tags = new TreeSet(sb.tables.bookmarks.getTags(bmk_user).values()); - } catch (IOException e) { + } catch (final IOException e) { return prop; } } @@ -63,17 +63,17 @@ public class get_tags { if (post != null && post.containsKey(TOP)) { top = post.getInt(TOP, Integer.MAX_VALUE); } - + if (post != null && post.containsKey(SORT)) { if (SIZE.equals(post.get(SORT))) { sortAlpha = false; } } - + if(sortAlpha) { final TreeMap sort = new TreeMap(); - final Collator collator = Collator.getInstance(); - collator.setStrength(Collator.SECONDARY); + final Collator collator = Collator.getInstance(); + collator.setStrength(Collator.SECONDARY); tit = tags.iterator(); while(tit.hasNext() && count < top) { t = tit.next(); diff --git a/htroot/api/ymarks/get_treeview.java b/htroot/api/ymarks/get_treeview.java index a44d97331..a075d81c1 100644 --- a/htroot/api/ymarks/get_treeview.java +++ b/htroot/api/ymarks/get_treeview.java @@ -26,25 +26,25 @@ import de.anomic.data.ymark.YMarkUtil; import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; -public class get_treeview { - +public class get_treeview { + public static final String ROOT = "root"; public static final String SOURCE = "source"; - + static serverObjects prop; - + public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { final Switchboard sb = (Switchboard) env; - prop = new serverObjects(); + prop = new serverObjects(); final UserDB.Entry user = sb.userDB.getUser(header); - final boolean isAdmin = (sb.verifyAuthentication(header, true)); + final boolean isAdmin = (sb.verifyAuthentication(header)); final boolean isAuthUser = user!= null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT); - + if(isAdmin || isAuthUser) { final String bmk_user = (isAuthUser ? user.getUserName() : YMarkTables.USER_ADMIN); - - String root = YMarkTables.FOLDERS_ROOT; + + String root = YMarkTables.FOLDERS_ROOT; String[] foldername = null; boolean isFolder = true; boolean isBookmark = false; @@ -58,7 +58,7 @@ public class get_treeview { if(post.containsKey("display") && post.get("display").equals("bmk")) { displayBmk = true; } - + if (post.containsKey(ROOT)) { if (post.get(ROOT).equals(SOURCE) || post.get(ROOT).equals(YMarkTables.FOLDERS_ROOT)) { root = ""; @@ -82,24 +82,24 @@ public class get_treeview { } } } - + Iterator it = null; Iterator bit = null; Tables.Row bmk_row = null; int count = 0; - + if(isFolder) { - // loop through folderList - try { + // loop through folderList + try { // it = sb.tables.bookmarks.folders.getFolders(bmk_user, root); it = sb.tables.bookmarks.getFolders(bmk_user, root).iterator(); - } catch (IOException e) { + } catch (final IOException e) { Log.logException(e); } int n = root.split(YMarkUtil.FOLDERS_SEPARATOR).length; if (n == 0) n = 1; - while (it.hasNext()) { - String folder = it.next(); + while (it.hasNext()) { + final String folder = it.next(); foldername = folder.split(YMarkUtil.FOLDERS_SEPARATOR); if (foldername.length == n+1) { prop.put("folders_"+count+"_foldername", foldername[n]); @@ -118,17 +118,17 @@ public class get_treeview { bit = sb.tables.bookmarks.getBookmarksByFolder(bmk_user, root); while (bit.hasNext()) { bmk_row = bit.next(); - if(bmk_row != null) { + if(bmk_row != null) { final String url = UTF8.String(bmk_row.get(YMarkEntry.BOOKMARK.URL.key())); final String title = bmk_row.get(YMarkEntry.BOOKMARK.TITLE.key(), YMarkEntry.BOOKMARK.TITLE.deflt()); - + // TODO: get_treeview - get rid of bmtype - if (post.containsKey("bmtype")) { + if (post.containsKey("bmtype")) { if (post.get("bmtype").equals("title")) { prop.putJSON("folders_"+count+"_foldername", title); } else if (post.get("bmtype").equals("href")) { prop.putJSON("folders_"+count+"_foldername", ""+title+""); - } + } } else { prop.putJSON("folders_"+count+"_foldername", url); } @@ -138,15 +138,15 @@ public class get_treeview { prop.put("folders_"+count+"_hash", "b:"+new String(bmk_row.getPK())); prop.put("folders_"+count+"_hasChildren", "true"); prop.put("folders_"+count+"_comma", ","); - count++; + count++; } - } + } } count--; prop.put("folders_"+count+"_comma", ""); count++; prop.put("folders", count); - } catch (IOException e) { + } catch (final IOException e) { Log.logException(e); } } else if(displayBmk && isBookmark) { @@ -170,12 +170,12 @@ public class get_treeview { final String value = UTF8.String(bmk_row.get(key)); if (key.equals("url")) url = value; - prop.put("folders_"+count+"_foldername",""+key+": " + value + ""); + prop.put("folders_"+count+"_foldername",""+key+": " + value + ""); if(YMarkEntry.BOOKMARK.contains(key)) putProp(count, YMarkEntry.BOOKMARK.get(key).type()); else putProp(count, "meta"); - count++; + count++; } } prop.put("folders_"+count+"_foldername","MetaData"); @@ -198,12 +198,12 @@ public class get_treeview { prop.put("folders_"+count+"_hash", "w:"+url); prop.put("folders_"+count+"_hasChildren", "true"); prop.put("folders_"+count+"_comma", ""); - count++; + count++; prop.put("folders", count); } - } catch (IOException e) { + } catch (final IOException e) { Log.logException(e); - } catch (RowSpaceExceededException e) { + } catch (final RowSpaceExceededException e) { Log.logException(e); } } else if (isAutoTagger || isMetadata || isURLdb || isCrawlStart) { @@ -220,7 +220,7 @@ public class get_treeview { count++; prop.put("folders_"+count+"_foldername","without preference: "+YMarkAutoTagger.autoTag(document, 4, new TreeMap())+""); putProp(count, "meta"); - count++; + count++; prop.put("folders", count); } else if(isMetadata) { count = putMeta(count, meta.loadMetadata()); @@ -240,17 +240,17 @@ public class get_treeview { prop.put("folders", count); } - } catch (MalformedURLException e) { + } catch (final MalformedURLException e) { Log.logException(e); - } catch (IOException e) { + } catch (final IOException e) { Log.logException(e); - } catch (Failure e) { + } catch (final Failure e) { Log.logException(e); } - } + } } else { prop.put(YMarkTables.USER_AUTHENTICATE,YMarkTables.USER_AUTHENTICATE_MSG); - } + } // return rewrite properties return prop; } diff --git a/htroot/api/ymarks/get_xbel.java b/htroot/api/ymarks/get_xbel.java index 482f3daad..59a3d9f5c 100644 --- a/htroot/api/ymarks/get_xbel.java +++ b/htroot/api/ymarks/get_xbel.java @@ -23,20 +23,20 @@ public class get_xbel { public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { final Switchboard sb = (Switchboard) env; - final serverObjects prop = new serverObjects(); + final serverObjects prop = new serverObjects(); final HashSet alias = new HashSet(); final StringBuilder buffer = new StringBuilder(250); final UserDB.Entry user = sb.userDB.getUser(header); - final boolean isAdmin = (sb.verifyAuthentication(header, true)); + final boolean isAdmin = (sb.verifyAuthentication(header)); final boolean isAuthUser = user!= null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT); final String bmk_user; - + if(isAdmin || isAuthUser) { bmk_user = (isAuthUser ? user.getUserName() : YMarkTables.USER_ADMIN); - - String root = YMarkTables.FOLDERS_ROOT; + + String root = YMarkTables.FOLDERS_ROOT; String[] foldername = null; - + // TODO: better handling of query if (post != null){ if (post.containsKey(ROOT)) { @@ -45,27 +45,27 @@ public class get_xbel { } else if (post.get(ROOT).startsWith(YMarkTables.FOLDERS_ROOT)) { root = post.get(ROOT); } else { - root = ""; + root = ""; } } } else { root = ""; } - + final int root_depth = root.split(YMarkUtil.FOLDERS_SEPARATOR).length - 1; // Log.logInfo(YMarkTables.BOOKMARKS_LOG, "root: "+root+" root_depth: "+root_depth); Iterator fit = null; Iterator bit = null; - int count = 0; + int count = 0; int n = root_depth; - + try { fit = sb.tables.bookmarks.getFolders(bmk_user, root).iterator(); - } catch (IOException e) { + } catch (final IOException e) { Log.logException(e); } - - while (fit.hasNext()) { + + while (fit.hasNext()) { final String folder = fit.next(); foldername = folder.split(YMarkUtil.FOLDERS_SEPARATOR); final int len = foldername.length -1; @@ -76,73 +76,73 @@ public class get_xbel { count++; } } - if (len >= n) { + if (len >= n) { n = len; if(n > root_depth) { // Log.logInfo(YMarkTables.BOOKMARKS_LOG, ": "+folder+" n: "+n); prop.put("xbel_"+count+"_elements", ""); count++; - prop.put("xbel_"+count+"_elements", "" + CharacterCoding.unicode2xml(foldername[n], true) + ""); - count++; + prop.put("xbel_"+count+"_elements", "" + CharacterCoding.unicode2xml(foldername[n], true) + ""); + count++; } try { bit = sb.tables.bookmarks.getBookmarksByFolder(bmk_user, folder); - } catch (IOException e) { + } catch (final IOException e) { // TODO: better error handling (avoid NPE) bit = null; } Tables.Row bmk_row = null; String urlHash; final YMarkDate date = new YMarkDate(); - while(bit.hasNext()){ + while(bit.hasNext()){ bmk_row = bit.next(); urlHash = new String(bmk_row.getPK()); - + if(alias.contains(urlHash)) { buffer.setLength(0); buffer.append(YMarkXBELImporter.XBEL.ALIAS.startTag(true)); buffer.append(" ref=\"b:"); buffer.append(urlHash); - buffer.append("\"/>"); - prop.put("xbel_"+count+"_elements", buffer.toString()); - count++; + buffer.append("\"/>"); + prop.put("xbel_"+count+"_elements", buffer.toString()); + count++; } else { alias.add(urlHash); if(bmk_row != null) { buffer.setLength(0); - + buffer.append(YMarkXBELImporter.XBEL.BOOKMARK.startTag(true)); buffer.append(" id=\"b:"); buffer.append(urlHash); - + buffer.append(YMarkEntry.BOOKMARK.URL.xbel()); buffer.append(CharacterCoding.unicode2xml(bmk_row.get(YMarkEntry.BOOKMARK.URL.key(), YMarkEntry.BOOKMARK.URL.deflt()), true)); - + buffer.append(YMarkEntry.BOOKMARK.DATE_ADDED.xbel()); date.set(bmk_row.get(YMarkEntry.BOOKMARK.DATE_ADDED.key())); buffer.append(CharacterCoding.unicode2xml(date.toISO8601(), true)); - + buffer.append(YMarkEntry.BOOKMARK.DATE_MODIFIED.xbel()); date.set(bmk_row.get(YMarkEntry.BOOKMARK.DATE_MODIFIED.key())); buffer.append(CharacterCoding.unicode2xml(date.toISO8601(), true)); - + buffer.append(YMarkEntry.BOOKMARK.DATE_VISITED.xbel()); date.set(bmk_row.get(YMarkEntry.BOOKMARK.DATE_VISITED.key())); buffer.append(CharacterCoding.unicode2xml(date.toISO8601(), true)); - + buffer.append(YMarkEntry.BOOKMARK.TAGS.xbel()); buffer.append(bmk_row.get(YMarkEntry.BOOKMARK.TAGS.key(), YMarkEntry.BOOKMARK.TAGS.deflt())); - + buffer.append(YMarkEntry.BOOKMARK.PUBLIC.xbel()); buffer.append(bmk_row.get(YMarkEntry.BOOKMARK.PUBLIC.key(), YMarkEntry.BOOKMARK.PUBLIC.deflt())); - + buffer.append(YMarkEntry.BOOKMARK.VISITS.xbel()); buffer.append(bmk_row.get(YMarkEntry.BOOKMARK.VISITS.key(), YMarkEntry.BOOKMARK.VISITS.deflt())); - + buffer.append("\"\n>"); prop.put("xbel_"+count+"_elements", buffer.toString()); - count++; - + count++; + buffer.setLength(0); buffer.append(YMarkXBELImporter.XBEL.TITLE.startTag(false)); buffer.append(CharacterCoding.unicode2xml(bmk_row.get(YMarkEntry.BOOKMARK.TITLE.key(), YMarkEntry.BOOKMARK.TITLE.deflt()), true)); @@ -156,9 +156,9 @@ public class get_xbel { buffer.append(YMarkXBELImporter.XBEL.DESC.endTag(false)); prop.put("xbel_"+count+"_elements", buffer.toString()); count++; - - prop.put("xbel_"+count+"_elements", YMarkXBELImporter.XBEL.BOOKMARK.endTag(false)); - count++; + + prop.put("xbel_"+count+"_elements", YMarkXBELImporter.XBEL.BOOKMARK.endTag(false)); + count++; } } } @@ -173,13 +173,13 @@ public class get_xbel { prop.put("root", root); prop.put("user", bmk_user.substring(0,1).toUpperCase() + bmk_user.substring(1)); prop.put("xbel", count); - + } else { prop.put(YMarkTables.USER_AUTHENTICATE,YMarkTables.USER_AUTHENTICATE_MSG); - } + } // return rewrite properties return prop; } } - + diff --git a/htroot/api/ymarks/get_ymark.java b/htroot/api/ymarks/get_ymark.java index 1a9310d78..49fc9ebbe 100644 --- a/htroot/api/ymarks/get_ymark.java +++ b/htroot/api/ymarks/get_ymark.java @@ -14,22 +14,22 @@ import de.anomic.data.ymark.YMarkCrawlStart; import de.anomic.data.ymark.YMarkDate; import de.anomic.data.ymark.YMarkEntry; import de.anomic.data.ymark.YMarkTables; -import de.anomic.data.ymark.YMarkUtil; import de.anomic.data.ymark.YMarkTables.TABLES; +import de.anomic.data.ymark.YMarkUtil; import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; public class get_ymark { - + private static Switchboard sb = null; private static serverObjects prop = null; final static String FOLDER_IMG = ""; - + public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { sb = (Switchboard) env; prop = new serverObjects(); - + int rp; // items per page int page; // page int total; @@ -37,15 +37,15 @@ public class get_ymark { String sortname; String qtype; String query; - + final UserDB.Entry user = sb.userDB.getUser(header); - final boolean isAdmin = (sb.verifyAuthentication(header, true)); + final boolean isAdmin = (sb.verifyAuthentication(header)); final boolean isAuthUser = user!= null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT); Iterator bookmarks = null; - + if(isAdmin || isAuthUser) { final String bmk_user = (isAuthUser ? user.getUserName() : YMarkTables.USER_ADMIN); - + query = ".*"; qtype = YMarkEntry.BOOKMARK.TITLE.key(); page = 1; @@ -53,7 +53,7 @@ public class get_ymark { total = 0; sortname = YMarkEntry.BOOKMARK.TITLE.key(); sortorder = "asc"; - + if(post != null) { rp = (post.containsKey("rp")) ? post.getInt("rp", 10) : 10; page = (post.containsKey("page")) ? post.getInt("page", 1): 1; @@ -61,7 +61,7 @@ public class get_ymark { qtype = (post.containsKey("qtype")) ? post.get("qtype", YMarkEntry.BOOKMARK.TAGS.key()) : YMarkEntry.BOOKMARK.TAGS.key(); sortname = (post.containsKey("sortname")) ? post.get("sortname", YMarkEntry.BOOKMARK.TITLE.key()) : YMarkEntry.BOOKMARK.TITLE.key(); sortorder = (post.containsKey("sortorder")) ? post.get("sortorder", "asc") : "asc"; - } + } try { final String bmk_table = TABLES.BOOKMARKS.tablename(bmk_user); final Collection result; @@ -72,7 +72,7 @@ public class get_ymark { result = sb.tables.bookmarks.orderBookmarksBy(sb.tables.bookmarks.getBookmarksByTag(bmk_user, tagArray), sortname, sortorder); } else if(qtype.equals("_folder")) { result = sb.tables.bookmarks.orderBookmarksBy(sb.tables.bookmarks.getBookmarksByFolder(bmk_user, query), sortname, sortorder); - } else { + } else { result = sb.tables.bookmarks.orderBookmarksBy(sb.tables.iterator(bmk_table, qtype, Pattern.compile(query)), sortname, sortorder); } } else { @@ -83,20 +83,20 @@ public class get_ymark { } total = result.size(); bookmarks = result.iterator(); - } catch (IOException e) { + } catch (final IOException e) { Log.logException(e); } prop.put("page", page); prop.put("total", total); putProp(bookmarks, rp, page); - + } else { prop.put(YMarkTables.USER_AUTHENTICATE,YMarkTables.USER_AUTHENTICATE_MSG); } // return rewrite properties return prop; } - + private static void putProp(final Iterator bit, final int rp, final int page) { Tables.Row bmk_row; int count = 0; @@ -112,11 +112,11 @@ public class get_ymark { while (count < rp && bit.hasNext()) { bmk_row = bit.next(); if (bmk_row != null) { - + // put JSON prop.put("json_"+count+"_id", count); prop.put("json_"+count+"_hash", UTF8.String(bmk_row.getPK())); - for (YMarkEntry.BOOKMARK bmk : YMarkEntry.BOOKMARK.values()) { + for (final YMarkEntry.BOOKMARK bmk : YMarkEntry.BOOKMARK.values()) { if(bmk == YMarkEntry.BOOKMARK.PUBLIC) prop.put("json_"+count+"_"+bmk.key(), bmk_row.get(bmk.key(),bmk.deflt()).equals("false") ? 1 : 0); else if(bmk == YMarkEntry.BOOKMARK.TAGS) @@ -128,7 +128,7 @@ public class get_ymark { else prop.putJSON("json_"+count+"_"+bmk.key(), bmk_row.get(bmk.key(),bmk.deflt())); } - + final YMarkCrawlStart crawlstart = new YMarkCrawlStart(sb.tables, bmk_row.get(YMarkEntry.BOOKMARK.URL.key(),YMarkEntry.BOOKMARK.URL.deflt())); int crawl = 0; if (!crawlstart.isEmpty()) { @@ -144,20 +144,20 @@ public class get_ymark { prop.put("json_"+count+"_crawlstart_info", "Crawl is running ..."); } prop.put("json_"+count+"_crawlstart", crawl); - + prop.put("json_"+count+"_apicall_pk", crawlstart.getPK()); prop.put("json_"+count+"_date_recording", YMarkDate.ISO8601(crawlstart.date_recording()).replaceAll("T", "
")); prop.put("json_"+count+"_date_next_exec", YMarkDate.ISO8601(crawlstart.date_next_exec()).replaceAll("T", "
")); - prop.put("json_"+count+"_date_last_exec", YMarkDate.ISO8601(crawlstart.date_last_exec()).replaceAll("T", "
")); - + prop.put("json_"+count+"_date_last_exec", YMarkDate.ISO8601(crawlstart.date_last_exec()).replaceAll("T", "
")); + prop.put("json_"+count+"_comma", ","); - + // put XML prop.putXML("xml_"+count+"_id", UTF8.String(bmk_row.getPK())); - for (YMarkEntry.BOOKMARK bmk : YMarkEntry.BOOKMARK.values()) { + for (final YMarkEntry.BOOKMARK bmk : YMarkEntry.BOOKMARK.values()) { prop.putXML("xml_"+count+"_"+bmk.key(), bmk_row.get(bmk.key(),bmk.deflt())); } - + count++; } } diff --git a/htroot/api/ymarks/import_ymark.java b/htroot/api/ymarks/import_ymark.java index 6be1902ca..b773692fb 100644 --- a/htroot/api/ymarks/import_ymark.java +++ b/htroot/api/ymarks/import_ymark.java @@ -38,7 +38,7 @@ public class import_ymark { final Switchboard sb = (Switchboard) env; final serverObjects prop = new serverObjects(); final UserDB.Entry user = sb.userDB.getUser(header); - final boolean isAdmin = (sb.verifyAuthentication(header, true)); + final boolean isAdmin = (sb.verifyAuthentication(header)); final boolean isAuthUser = user!= null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT); final int queueSize = 200; diff --git a/htroot/api/ymarks/manage_tags.java b/htroot/api/ymarks/manage_tags.java index 118df10c0..582f4ae06 100644 --- a/htroot/api/ymarks/manage_tags.java +++ b/htroot/api/ymarks/manage_tags.java @@ -16,10 +16,10 @@ import de.anomic.server.serverSwitch; public class manage_tags { - + private static Switchboard sb = null; private static serverObjects prop = null; - + public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { sb = (Switchboard) env; prop = new serverObjects(); @@ -28,27 +28,27 @@ public class manage_tags { String query; String tags; String replace; - + final UserDB.Entry user = sb.userDB.getUser(header); - final boolean isAdmin = (sb.verifyAuthentication(header, true)); + final boolean isAdmin = (sb.verifyAuthentication(header)); final boolean isAuthUser = user!= null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT); - + if(isAdmin || isAuthUser) { final String bmk_user = (isAuthUser ? user.getUserName() : YMarkTables.USER_ADMIN); - + if(post != null) { query = post.get("query", post.get("tags", YMarkUtil.EMPTY_STRING)); qtype = post.get("qtype", "_tags"); tags = YMarkUtil.cleanTagsString(post.get("tags", YMarkUtil.EMPTY_STRING)); replace = post.get("replace", YMarkUtil.EMPTY_STRING); - + } else { query = ".*"; qtype = YMarkUtil.EMPTY_STRING; tags = YMarkUtil.EMPTY_STRING; replace = YMarkUtil.EMPTY_STRING; } - + try { final String bmk_table = TABLES.BOOKMARKS.tablename(bmk_user); final Iterator row_iter; @@ -61,7 +61,7 @@ public class manage_tags { row_iter = sb.tables.bookmarks.getBookmarksByTag(bmk_user, tagArray); } else if(qtype.equals("_folder")) { row_iter = sb.tables.bookmarks.getBookmarksByFolder(bmk_user, query); - } else { + } else { row_iter = sb.tables.iterator(bmk_table, qtype, Pattern.compile(query)); } } else { @@ -74,15 +74,15 @@ public class manage_tags { } sb.tables.bookmarks.replaceTags(row_iter, bmk_user, tags, replace); prop.put("status", 1); - } catch (IOException e) { + } catch (final IOException e) { Log.logException(e); - } catch (RowSpaceExceededException e) { + } catch (final RowSpaceExceededException e) { Log.logException(e); - } + } } else { prop.put(YMarkTables.USER_AUTHENTICATE,YMarkTables.USER_AUTHENTICATE_MSG); } // return rewrite properties return prop; - } + } } diff --git a/htroot/api/ynetSearch.java b/htroot/api/ynetSearch.java index 4884d5ae9..fa12c2984 100755 --- a/htroot/api/ynetSearch.java +++ b/htroot/api/ynetSearch.java @@ -9,18 +9,17 @@ import java.util.Scanner; import net.yacy.cora.protocol.RequestHeader; import net.yacy.search.Switchboard; - import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; public class ynetSearch { - - public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { + + public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { final Switchboard switchboard = (Switchboard) env; - final boolean isAdmin=switchboard.verifyAuthentication(header, true); - final serverObjects prop = new serverObjects(); - - if(post != null){ + final boolean isAdmin=switchboard.verifyAuthentication(header); + final serverObjects prop = new serverObjects(); + + if(post != null){ if(!isAdmin){ // force authentication if desired if(post.containsKey("login")){ @@ -28,7 +27,7 @@ public class ynetSearch { } return prop; } else { - InputStream is = null; + InputStream is = null; try { String searchaddress = post.get("url"); if (!searchaddress.startsWith("http://")) { @@ -42,23 +41,23 @@ public class ynetSearch { Map.Entry k; while(it.hasNext()) { k = it.next(); - s = s + "&" + k.getKey() + "=" + k.getValue(); + s = s + "&" + k.getKey() + "=" + k.getValue(); } - // final String s = searchaddress+"&query="+post.get("search")+"&maximumRecords="+post.get("maximumRecords")+"&startRecord="+post.get("startRecord"); - final URL url = new URL(s); - is = url.openStream(); - final String httpout = new Scanner(is).useDelimiter( "\\Z" ).next(); + // final String s = searchaddress+"&query="+post.get("search")+"&maximumRecords="+post.get("maximumRecords")+"&startRecord="+post.get("startRecord"); + final URL url = new URL(s); + is = url.openStream(); + final String httpout = new Scanner(is).useDelimiter( "\\Z" ).next(); prop.put("http", httpout); - } - catch ( final Exception e ) { + } + catch ( final Exception e ) { prop.put("url", "error!"); - } - finally { - if ( is != null ) - try { is.close(); } catch ( final IOException e ) { } + } + finally { + if ( is != null ) + try { is.close(); } catch ( final IOException e ) { } } } - } + } return prop; } } \ No newline at end of file diff --git a/htroot/index.java b/htroot/index.java index 400274020..ee034c075 100644 --- a/htroot/index.java +++ b/htroot/index.java @@ -51,7 +51,7 @@ public class index { } // access control - final boolean authorizedAccess = sb.verifyAuthentication(header, false); + final boolean authorizedAccess = sb.verifyAuthentication(header); if ((post != null) && (post.containsKey("publicPage"))) { if (!authorizedAccess) { prop.put("AUTHENTICATE", "admin log-in"); // force log-in diff --git a/htroot/yacysearch.java b/htroot/yacysearch.java index ff3febc5b..9ae07a6fe 100644 --- a/htroot/yacysearch.java +++ b/htroot/yacysearch.java @@ -87,7 +87,7 @@ public class yacysearch { final Switchboard sb = (Switchboard) env; sb.localSearchLastAccess = System.currentTimeMillis(); - final boolean searchAllowed = sb.getConfigBool("publicSearchpage", true) || sb.verifyAuthentication(header, false); + final boolean searchAllowed = sb.getConfigBool("publicSearchpage", true) || sb.verifyAuthentication(header); boolean authenticated = sb.adminAuthenticated(header) >= 2; if (!authenticated) { @@ -484,7 +484,7 @@ public class yacysearch { // if a minus-button was hit, remove a special reference first if (post != null && post.containsKey("deleteref")) { try { - if (!sb.verifyAuthentication(header, false)) { + if (!sb.verifyAuthentication(header)) { prop.put("AUTHENTICATE", "admin log-in"); // force log-in return prop; } @@ -511,7 +511,7 @@ public class yacysearch { // if a plus-button was hit, create new voting message if (post != null && post.containsKey("recommendref")) { - if (!sb.verifyAuthentication(header, false)) { + if (!sb.verifyAuthentication(header)) { prop.put("AUTHENTICATE", "admin log-in"); // force log-in return prop; } @@ -541,7 +541,7 @@ public class yacysearch { // if a bookmarks-button was hit, create new bookmark entry if (post != null && post.containsKey("bookmarkref")) { - if (!sb.verifyAuthentication(header, false)) { + if (!sb.verifyAuthentication(header)) { prop.put("AUTHENTICATE", "admin log-in"); // force log-in return prop; } diff --git a/htroot/yacysearchitem.java b/htroot/yacysearchitem.java index 491840b2f..c0608772f 100644 --- a/htroot/yacysearchitem.java +++ b/htroot/yacysearchitem.java @@ -71,7 +71,7 @@ public class yacysearchitem { final String eventID = post.get("eventID", ""); final boolean authenticated = sb.adminAuthenticated(header) >= 2; final int item = post.getInt("item", -1); - final boolean auth = (header.get(HeaderFramework.CONNECTION_PROP_CLIENTIP, "")).equals("localhost") || sb.verifyAuthentication(header, true); + final boolean auth = (header.get(HeaderFramework.CONNECTION_PROP_CLIENTIP, "")).equals("localhost") || sb.verifyAuthentication(header); final RequestHeader.FileType fileType = header.fileType(); // default settings for blank item @@ -141,7 +141,7 @@ public class yacysearchitem { prop.putHTML("content_link", result.urlstring()); prop.putHTML("content_showPictures_link", result.urlstring()); prop.putHTML("content_target", target); - if (faviconURL != null && fileType == FileType.HTML) sb.loader.loadIfNotExistBackground(faviconURL.toNormalform(true, false), 1024 * 1024 * 10); + if (faviconURL != null && fileType == FileType.HTML) sb.loader.loadIfNotExistBackground(faviconURL, 1024 * 1024 * 10); prop.putHTML("content_faviconCode", sb.licensedURLs.aquireLicense(faviconURL)); // acquire license for favicon url loading prop.put("content_urlhash", resulthashString); prop.put("content_ranking", result.ranking); @@ -216,7 +216,7 @@ public class yacysearchitem { prop.put("content_item", "0"); } else { final String license = sb.licensedURLs.aquireLicense(ms.href); - sb.loader.loadIfNotExistBackground(ms.href.toNormalform(true, false), 1024 * 1024 * 10); + sb.loader.loadIfNotExistBackground(ms.href, 1024 * 1024 * 10); prop.putHTML("content_item_hrefCache", (auth) ? "/ViewImage.png?url=" + ms.href.toNormalform(true, false) : ms.href.toNormalform(true, false)); prop.putHTML("content_item_href", ms.href.toNormalform(true, false)); prop.putHTML("content_item_target", target); diff --git a/source/net/yacy/cora/document/MultiProtocolURI.java b/source/net/yacy/cora/document/MultiProtocolURI.java index d7681eb49..9f33f4670 100644 --- a/source/net/yacy/cora/document/MultiProtocolURI.java +++ b/source/net/yacy/cora/document/MultiProtocolURI.java @@ -1140,6 +1140,11 @@ public class MultiProtocolURI implements Serializable, Comparable 0) return locale; } + /* + if (this.hostAddress != null) { + return Domains.getLocale(this.hostAddress); + } + */ return Domains.getLocale(this.host); } diff --git a/source/net/yacy/cora/protocol/Domains.java b/source/net/yacy/cora/protocol/Domains.java index 843bec7aa..e236132e2 100644 --- a/source/net/yacy/cora/protocol/Domains.java +++ b/source/net/yacy/cora/protocol/Domains.java @@ -67,8 +67,8 @@ public class Domains { private static final String PRESENT = ""; private static final String LOCAL_PATTERNS = "10\\..*,127\\..*,172\\.(1[6-9]|2[0-9]|3[0-1])\\..*,169\\.254\\..*,192\\.168\\..*,localhost"; - private static final int MAX_NAME_CACHE_HIT_SIZE = 20000; - private static final int MAX_NAME_CACHE_MISS_SIZE = 20000; + private static final int MAX_NAME_CACHE_HIT_SIZE = 100000; + private static final int MAX_NAME_CACHE_MISS_SIZE = 100000; private static final int CONCURRENCY_LEVEL = Runtime.getRuntime().availableProcessors() + 1; // a dns cache @@ -76,7 +76,7 @@ public class Domains { private static final ARC NAME_CACHE_MISS = new ConcurrentARC(MAX_NAME_CACHE_MISS_SIZE, CONCURRENCY_LEVEL); private static final ConcurrentHashMap LOOKUP_SYNC = new ConcurrentHashMap(100, 0.75f, Runtime.getRuntime().availableProcessors() * 2); private static List nameCacheNoCachingPatterns = Collections.synchronizedList(new LinkedList()); - private static final List LOCALHOST_PATTERNS = makePatterns(LOCAL_PATTERNS); + private static final List INTRANET_PATTERNS = makePatterns(LOCAL_PATTERNS); public static long cacheHit_Hit = 0, cacheHit_Miss = 0, cacheHit_Insert = 0; // for statistics only; do not write public static long cacheMiss_Hit = 0, cacheMiss_Miss = 0, cacheMiss_Insert = 0; // for statistics only; do not write @@ -809,7 +809,7 @@ public class Domains { if (localHostAddresses.isEmpty()) return list; // give up for (final InetAddress a: localHostAddresses) { if (((0Xff & a.getAddress()[0]) == 127) || - (!matchesList(a.getHostAddress(), LOCALHOST_PATTERNS))) continue; + (!matchesList(a.getHostAddress(), INTRANET_PATTERNS))) continue; list.add(a); } return list; @@ -894,7 +894,7 @@ public class Domains { // FIXME IPv4 only // check local ip addresses - if (matchesList(host, LOCALHOST_PATTERNS)) return true; + if (matchesList(host, INTRANET_PATTERNS)) return true; if (host.startsWith("0:0:0:0:0:0:0:1")) return true; // check if there are other local IP addresses that are not in diff --git a/source/net/yacy/cora/storage/ConcurrentARC.java b/source/net/yacy/cora/storage/ConcurrentARC.java index 4de97774f..700d6b3c7 100644 --- a/source/net/yacy/cora/storage/ConcurrentARC.java +++ b/source/net/yacy/cora/storage/ConcurrentARC.java @@ -11,12 +11,12 @@ * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. - * + * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. - * + * * You should have received a copy of the GNU Lesser General Public License * along with this program in the file lgpl21.txt * If not, see . @@ -28,9 +28,11 @@ import java.util.AbstractMap; import java.util.ArrayList; import java.util.Collection; import java.util.Comparator; +import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.Map; +import java.util.Random; import java.util.Set; @@ -47,7 +49,7 @@ public final class ConcurrentARC extends AbstractMap implements Map< private final int mask; private final ARC arc[]; - + /** * create a concurrent ARC based on a HashARC. The type of the key elements must implement a hashing function * @param cacheSize the number of maximum entries @@ -64,7 +66,7 @@ public final class ConcurrentARC extends AbstractMap implements Map< m -= 1; this.mask = m; } - + /** * create a concurrent ARC based on a ComparableARC * @param cacheSize the number of maximum entries @@ -72,7 +74,7 @@ public final class ConcurrentARC extends AbstractMap implements Map< * @param comparator a comparator for the key object which may be of type byte[] */ @SuppressWarnings("unchecked") - public ConcurrentARC(final int cacheSize, final int partitions, Comparator comparator) { + public ConcurrentARC(final int cacheSize, final int partitions, final Comparator comparator) { int m = 1; while (m < partitions) m = m * 2; int partitionSize = cacheSize / m; @@ -82,7 +84,7 @@ public final class ConcurrentARC extends AbstractMap implements Map< m -= 1; this.mask = m; } - + /** * put a value to the cache. * @param s @@ -98,7 +100,7 @@ public final class ConcurrentARC extends AbstractMap implements Map< * @param s * @param v */ - public void insertIfAbsent(K s, V v) { + public void insertIfAbsent(final K s, final V v) { this.arc[getPartition(s)].insertIfAbsent(s, v); } @@ -109,10 +111,10 @@ public final class ConcurrentARC extends AbstractMap implements Map< * @param v * @return the value before inserting the new value */ - public V putIfAbsent(K s, V v) { + public V putIfAbsent(final K s, final V v) { return this.arc[getPartition(s)].putIfAbsent(s, v); } - + /** * put a value to the cache. * @param s @@ -122,7 +124,7 @@ public final class ConcurrentARC extends AbstractMap implements Map< public final V put(final K s, final V v) { return this.arc[getPartition(s)].put(s, v); } - + /** * get a value from the cache. * @param s @@ -133,18 +135,19 @@ public final class ConcurrentARC extends AbstractMap implements Map< public final V get(final Object s) { return this.arc[getPartition(s)].get((K) s); } - + /** * check if the map contains the value * @param value * @return the keys that have the given value */ - public Collection getKeys(V value) { - ArrayList keys = new ArrayList(); - for (int i = 0; i < this.arc.length; i++) keys.addAll(this.arc[i].getKeys(value)); + public Collection getKeys(final V value) { + final ArrayList keys = new ArrayList(); + for (final ARC element : this.arc) + keys.addAll(element.getKeys(value)); return keys; } - + /** * check if the map contains the key * @param s @@ -155,7 +158,7 @@ public final class ConcurrentARC extends AbstractMap implements Map< public final boolean containsKey(final Object s) { return this.arc[getPartition(s)].containsKey((K) s); } - + /** * remove an entry from the cache * @param s @@ -166,13 +169,13 @@ public final class ConcurrentARC extends AbstractMap implements Map< public final V remove(final Object s) { return this.arc[getPartition(s)].remove((K) s); } - + /** * clear the cache */ @Override public final void clear() { - for (ARC a: this.arc) a.clear(); + for (final ARC a: this.arc) a.clear(); } /** @@ -182,7 +185,7 @@ public final class ConcurrentARC extends AbstractMap implements Map< @Override public final int size() { int s = 0; - for (ARC a: this.arc) s += a.size(); + for (final ARC a: this.arc) s += a.size(); return s; } @@ -202,13 +205,13 @@ public final class ConcurrentARC extends AbstractMap implements Map< */ @Override public Set> entrySet() { - Set> m = new HashSet>(); - for (ARC a: this.arc) { - for (Map.Entry entry: a.entrySet()) m.add(entry); + final Set> m = new HashSet>(); + for (final ARC a: this.arc) { + for (final Map.Entry entry: a.entrySet()) m.add(entry); } return m; } - + /** * a hash code for this ARC * @return a hash code @@ -217,7 +220,7 @@ public final class ConcurrentARC extends AbstractMap implements Map< public int hashCode() { return this.arc.hashCode(); } - + //private static String latestObject = ""; /** * return in which partition the Object belongs @@ -228,13 +231,47 @@ public final class ConcurrentARC extends AbstractMap implements Map< private int getPartition(final Object x) { if (x instanceof byte[]) { int h = 0; - for (byte c: (byte[])x) h = 31 * h + (c & 0xFF); - int p = h & mask; + for (final byte c: (byte[])x) h = 31 * h + (c & 0xFF); + final int p = h & this.mask; //String o = UTF8.String((byte[]) x); try { if (o.equals(latestObject)) throw new RuntimeException("ConcurrentARC: p = " + p + ", objectb = " + o); } catch (Exception e) { Log.logException(e); } latestObject = o; return p; } - int p = x.hashCode() & mask; + final int p = x.hashCode() & this.mask; //String o = x.toString(); try { if (o.equals(latestObject)) throw new RuntimeException("ConcurrentARC: p = " + p + ", objecto = " + o); } catch (Exception e) { Log.logException(e); } latestObject = o; return p; } + + public static void main(final String[] args) { + final Random r = new Random(); + final int testsize = 10000; + final ARC a = new ConcurrentARC(testsize * 3, Runtime.getRuntime().availableProcessors()); + final Map b = new HashMap(); + String key, value; + for (int i = 0; i < testsize; i++) { + key = "k" + r.nextInt(); + value = "v" + r.nextInt(); + a.insertIfAbsent(key, value); + b.put(key, value); + } + + // now put half of the entries AGAIN into the ARC + int h = testsize / 2; + for (final Map.Entry entry: b.entrySet()) { + a.put(entry.getKey(), entry.getValue()); + if (h-- <= 0) break; + } + + // test correctness + for (final Map.Entry entry: b.entrySet()) { + if (!a.containsKey(entry.getKey())) { + System.out.println("missing: " + entry.getKey()); + continue; + } + if (!a.get(entry.getKey()).equals(entry.getValue())) { + System.out.println("wrong: a = " + entry.getKey() + "," + a.get(entry.getKey()) + "; v = " + entry.getValue()); + } + } + System.out.println("finished test!"); + } + } diff --git a/source/net/yacy/cora/storage/HashARC.java b/source/net/yacy/cora/storage/HashARC.java index 6d299fdf1..b67a95d33 100644 --- a/source/net/yacy/cora/storage/HashARC.java +++ b/source/net/yacy/cora/storage/HashARC.java @@ -12,12 +12,12 @@ * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. - * + * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. - * + * * You should have received a copy of the GNU Lesser General Public License * along with this program in the file lgpl21.txt * If not, see . @@ -26,13 +26,15 @@ package net.yacy.cora.storage; import java.util.Collections; +import java.util.HashMap; import java.util.LinkedHashMap; import java.util.Map; +import java.util.Random; public final class HashARC extends SimpleARC implements Map, Iterable>, ARC { public final static boolean accessOrder = false; // if false, then a insertion-order is used - + public HashARC(final int cacheSize) { this.cacheSize = cacheSize / 2; super.levelA = Collections.synchronizedMap(new LinkedHashMap(cacheSize, 0.1f, accessOrder) { @@ -48,4 +50,37 @@ public final class HashARC extends SimpleARC implements Map, I } }); } + + public static void main(final String[] args) { + final Random r = new Random(); + final int testsize = 10000; + final ARC a = new HashARC(testsize * 2); + final Map b = new HashMap(); + String key, value; + for (int i = 0; i < testsize; i++) { + key = "k" + r.nextInt(); + value = "v" + r.nextInt(); + a.insertIfAbsent(key, value); + b.put(key, value); + } + + // now put half of the entries AGAIN into the ARC + int h = testsize / 2; + for (final Map.Entry entry: b.entrySet()) { + a.put(entry.getKey(), entry.getValue()); + if (h-- <= 0) break; + } + + // test correctness + for (final Map.Entry entry: b.entrySet()) { + if (!a.containsKey(entry.getKey())) { + System.out.println("missing: " + entry.getKey()); + continue; + } + if (!a.get(entry.getKey()).equals(entry.getValue())) { + System.out.println("wrong: a = " + entry.getKey() + "," + a.get(entry.getKey()) + "; v = " + entry.getValue()); + } + } + System.out.println("finished test!"); + } } diff --git a/source/net/yacy/cora/storage/SimpleARC.java b/source/net/yacy/cora/storage/SimpleARC.java index 40c20fc5b..8ee383d90 100644 --- a/source/net/yacy/cora/storage/SimpleARC.java +++ b/source/net/yacy/cora/storage/SimpleARC.java @@ -12,12 +12,12 @@ * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. - * + * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. - * + * * You should have received a copy of the GNU Lesser General Public License * along with this program in the file lgpl21.txt * If not, see . @@ -41,7 +41,7 @@ import java.util.Set; * or http://en.wikipedia.org/wiki/Adaptive_Replacement_Cache * This version omits the ghost entry handling which is described in ARC, and keeps both cache levels * at the same size. - * + * * This class is defined abstract because it shall be used with either the HashARC or the ComparableARC classes */ @@ -49,7 +49,7 @@ abstract class SimpleARC extends AbstractMap implements Map, I protected int cacheSize; protected Map levelA, levelB; // we can assume that these maps are synchronized - + /** * put a value to the cache. * @param s @@ -58,10 +58,10 @@ abstract class SimpleARC extends AbstractMap implements Map, I public final synchronized void insert(final K s, final V v) { if (this.levelB.containsKey(s)) { this.levelB.put(s, v); - assert (this.levelB.size() <= cacheSize); // the cache should shrink automatically + assert (this.levelB.size() <= this.cacheSize); // the cache should shrink automatically } else { this.levelA.put(s, v); - assert (this.levelA.size() <= cacheSize); // the cache should shrink automatically + assert (this.levelA.size() <= this.cacheSize); // the cache should shrink automatically } } @@ -71,7 +71,7 @@ abstract class SimpleARC extends AbstractMap implements Map, I * @param s * @param v */ - public void insertIfAbsent(K s, V v) { + public void insertIfAbsent(final K s, final V v) { if (this.levelB.containsKey(s)) { return; } else if (this.levelA.containsKey(s)) { @@ -85,12 +85,12 @@ abstract class SimpleARC extends AbstractMap implements Map, I return; } else { this.levelA.put(s, v); - assert (this.levelA.size() <= cacheSize); // the cache should shrink automatically + assert (this.levelA.size() <= this.cacheSize); // the cache should shrink automatically } } } } - + /** * put a value to the cache if there was not an entry before * return a previous content value @@ -98,18 +98,18 @@ abstract class SimpleARC extends AbstractMap implements Map, I * @param v * @return the value before inserting the new value */ - public V putIfAbsent(K s, V v) { + public V putIfAbsent(final K s, final V v) { synchronized (this) { V o = this.levelB.get(s); if (o != null) return o; o = this.levelA.get(s); if (o != null) return o; this.levelA.put(s, v); - assert (this.levelA.size() <= cacheSize); // the cache should shrink automatically + assert (this.levelA.size() <= this.cacheSize); // the cache should shrink automatically return null; } } - + /** * put a value to the cache. * @param s @@ -117,16 +117,16 @@ abstract class SimpleARC extends AbstractMap implements Map, I */ public final synchronized V put(final K s, final V v) { if (this.levelB.containsKey(s)) { - V r = this.levelB.put(s, v); - assert (this.levelB.size() <= cacheSize); // the cache should shrink automatically + final V r = this.levelB.put(s, v); + assert (this.levelB.size() <= this.cacheSize); // the cache should shrink automatically return r; } else { - V r = this.levelA.put(s, v); - assert (this.levelA.size() <= cacheSize); // the cache should shrink automatically + final V r = this.levelA.put(s, v); + assert (this.levelA.size() <= this.cacheSize); // the cache should shrink automatically return r; } } - + /** * get a value from the cache. * @param s @@ -148,7 +148,7 @@ abstract class SimpleARC extends AbstractMap implements Map, I // move value from A to B; since it was already removed from A, just put it to B //System.out.println("ARC: moving A->B, size(A) = " + this.levelA.size() + ", size(B) = " + this.levelB.size()); this.levelB.put((K) s, v); - assert (this.levelB.size() <= cacheSize); // the cache should shrink automatically + assert (this.levelB.size() <= this.cacheSize); // the cache should shrink automatically } return v; } @@ -158,21 +158,21 @@ abstract class SimpleARC extends AbstractMap implements Map, I * @param value * @return the keys that have the given value */ - public Collection getKeys(V value) { - ArrayList keys = new ArrayList(); + public Collection getKeys(final V value) { + final ArrayList keys = new ArrayList(); synchronized (this.levelB) { - for (Map.Entry entry: this.levelB.entrySet()) { + for (final Map.Entry entry: this.levelB.entrySet()) { if (value.equals(entry.getValue())) keys.add(entry.getKey()); } } synchronized (this) { - for (Map.Entry entry: this.levelA.entrySet()) { + for (final Map.Entry entry: this.levelA.entrySet()) { if (value.equals(entry.getValue())) keys.add(entry.getKey()); } } return keys; } - + /** * check if the map contains the key * @param s @@ -183,8 +183,8 @@ abstract class SimpleARC extends AbstractMap implements Map, I if (this.levelB.containsKey(s)) return true; return this.levelA.containsKey(s); } - - + + /** * remove an entry from the cache * @param s @@ -196,7 +196,7 @@ abstract class SimpleARC extends AbstractMap implements Map, I if (r != null) return r; return this.levelA.remove(s); } - + /** * clear the cache */ @@ -214,7 +214,7 @@ abstract class SimpleARC extends AbstractMap implements Map, I public final synchronized int size() { return this.levelA.size() + this.levelB.size(); } - + /** * iterator implements the Iterable interface */ @@ -231,12 +231,12 @@ abstract class SimpleARC extends AbstractMap implements Map, I */ @Override public final synchronized Set> entrySet() { - Set> m = new HashSet>(); - for (Map.Entry entry: this.levelA.entrySet()) m.add(entry); - for (Map.Entry entry: this.levelB.entrySet()) m.add(entry); + final Set> m = new HashSet>(); + for (final Map.Entry entry: this.levelA.entrySet()) m.add(entry); + for (final Map.Entry entry: this.levelB.entrySet()) m.add(entry); return m; } - + /** * a hash code for this ARC * @return the hash code of one of the ARC partial hash tables @@ -245,4 +245,5 @@ abstract class SimpleARC extends AbstractMap implements Map, I public final int hashCode() { return this.levelA.hashCode(); } + } diff --git a/source/net/yacy/document/importer/OAIListFriendsLoader.java b/source/net/yacy/document/importer/OAIListFriendsLoader.java index 9af9011af..0360c6ee9 100644 --- a/source/net/yacy/document/importer/OAIListFriendsLoader.java +++ b/source/net/yacy/document/importer/OAIListFriendsLoader.java @@ -27,6 +27,7 @@ import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; +import java.net.MalformedURLException; import java.util.HashMap; import java.util.Map; import java.util.Map.Entry; @@ -59,7 +60,10 @@ public class OAIListFriendsLoader { public static void init(final LoaderDispatcher loader, final Map moreFriends) { listFriends.putAll(moreFriends); if (loader != null) for (final Map.Entry oaiFriend: listFriends.entrySet()) { - loader.loadIfNotExistBackground(oaiFriend.getKey(), oaiFriend.getValue(), Integer.MAX_VALUE); + try { + loader.loadIfNotExistBackground(new DigestURI(oaiFriend.getKey()), oaiFriend.getValue(), Integer.MAX_VALUE); + } catch (final MalformedURLException e) { + } } } diff --git a/source/net/yacy/repository/LoaderDispatcher.java b/source/net/yacy/repository/LoaderDispatcher.java index aa4d7612e..2400de845 100644 --- a/source/net/yacy/repository/LoaderDispatcher.java +++ b/source/net/yacy/repository/LoaderDispatcher.java @@ -77,7 +77,7 @@ public final class LoaderDispatcher { private final FTPLoader ftpLoader; private final SMBLoader smbLoader; private final FileLoader fileLoader; - private final ConcurrentHashMap loaderSteering; // a map that delivers a 'finish' semaphore for urls + private final ConcurrentHashMap loaderSteering; // a map that delivers a 'finish' semaphore for urls private final Log log; public LoaderDispatcher(final Switchboard sb) { @@ -90,7 +90,7 @@ public final class LoaderDispatcher { this.ftpLoader = new FTPLoader(sb, this.log); this.smbLoader = new SMBLoader(sb, this.log); this.fileLoader = new FileLoader(sb, this.log); - this.loaderSteering = new ConcurrentHashMap(); + this.loaderSteering = new ConcurrentHashMap(); } public boolean isSupportedProtocol(final String protocol) { @@ -153,8 +153,7 @@ public final class LoaderDispatcher { } public Response load(final Request request, final CacheStrategy cacheStrategy, final int maxFileSize, final boolean checkBlacklist) throws IOException { - final String url = request.url().toNormalform(true, false); - Semaphore check = this.loaderSteering.get(url); + Semaphore check = this.loaderSteering.get(request.url()); if (check != null) { // a loading process may be going on for that url try { check.tryAcquire(5, TimeUnit.SECONDS);} catch (final InterruptedException e) {} @@ -162,15 +161,15 @@ public final class LoaderDispatcher { // which may be successful faster because of a cache hit } - this.loaderSteering.put(url, new Semaphore(0)); + this.loaderSteering.put(request.url(), new Semaphore(0)); try { final Response response = loadInternal(request, cacheStrategy, maxFileSize, checkBlacklist); - check = this.loaderSteering.remove(url); + check = this.loaderSteering.remove(request.url()); if (check != null) check.release(1000); return response; } catch (final IOException e) { // release the semaphore anyway - check = this.loaderSteering.remove(url); + check = this.loaderSteering.remove(request.url()); if (check != null) check.release(1000); //Log.logException(e); throw new IOException(e); @@ -386,22 +385,22 @@ public final class LoaderDispatcher { } } - public void loadIfNotExistBackground(final String url, final File cache, final int maxFileSize) { + public void loadIfNotExistBackground(final DigestURI url, final File cache, final int maxFileSize) { new Loader(url, cache, maxFileSize, CacheStrategy.IFEXIST).start(); } - public void loadIfNotExistBackground(final String url, final int maxFileSize) { + public void loadIfNotExistBackground(final DigestURI url, final int maxFileSize) { new Loader(url, null, maxFileSize, CacheStrategy.IFEXIST).start(); } private class Loader extends Thread { - private final String url; + private final DigestURI url; private final File cache; private final int maxFileSize; private final CacheStrategy cacheStrategy; - public Loader(final String url, final File cache, final int maxFileSize, final CacheStrategy cacheStrategy) { + public Loader(final DigestURI url, final File cache, final int maxFileSize, final CacheStrategy cacheStrategy) { this.url = url; this.cache = cache; this.maxFileSize = maxFileSize; @@ -412,7 +411,7 @@ public final class LoaderDispatcher { if (this.cache != null && this.cache.exists()) return; try { // load from the net - final Response response = load(request(new DigestURI(this.url), false, true), this.cacheStrategy, this.maxFileSize, true); + final Response response = load(request(this.url, false, true), this.cacheStrategy, this.maxFileSize, true); final byte[] b = response.getContent(); if (this.cache != null) FileUtils.copy(b, this.cache); } catch (final MalformedURLException e) {} catch (final IOException e) {} diff --git a/source/net/yacy/search/Switchboard.java b/source/net/yacy/search/Switchboard.java index 5a484fb4b..56d472bce 100644 --- a/source/net/yacy/search/Switchboard.java +++ b/source/net/yacy/search/Switchboard.java @@ -2348,7 +2348,7 @@ public final class Switchboard extends serverSwitch { return 1; } - public boolean verifyAuthentication(final RequestHeader header, final boolean strict) { + public boolean verifyAuthentication(final RequestHeader header) { // handle access rights switch (adminAuthenticated(header)) { case 0: // wrong password given @@ -2357,7 +2357,7 @@ public final class Switchboard extends serverSwitch { case 1: // no password given return false; case 2: // no password stored - return !strict; + return true; case 3: // soft-authenticated for localhost only return true; case 4: // hard-authenticated, all ok