- tested the ARC methods

- removed strict authentication (if password is empty; this was buggy and not useful; can be switched on if necessary globally and not for each interface method)
- increased speed of CrawlResults page (no dns lookup any more)
- increased speed of favicon display (removed dns lookup)

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@8104 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 13 years ago
parent bc5df0eef5
commit e22f8497c9

@ -34,20 +34,19 @@ import java.text.SimpleDateFormat;
import java.util.Date; import java.util.Date;
import java.util.HashMap; import java.util.HashMap;
import java.util.Iterator; import java.util.Iterator;
import java.util.List;
import java.util.Locale; import java.util.Locale;
import java.util.Map;
import net.yacy.cora.document.UTF8; import net.yacy.cora.document.UTF8;
import net.yacy.cora.protocol.HeaderFramework; import net.yacy.cora.protocol.HeaderFramework;
import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.protocol.RequestHeader;
import net.yacy.peers.NewsPool; import net.yacy.peers.NewsPool;
import net.yacy.search.Switchboard; import net.yacy.search.Switchboard;
import de.anomic.data.BlogBoard; import de.anomic.data.BlogBoard;
import de.anomic.data.UserDB; import de.anomic.data.UserDB;
import de.anomic.server.serverObjects; import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch; import de.anomic.server.serverSwitch;
import java.util.List;
import java.util.Map;
public class Blog { public class Blog {
@ -65,7 +64,7 @@ public class Blog {
final serverObjects prop = new serverObjects(); final serverObjects prop = new serverObjects();
BlogBoard.BlogEntry page = null; BlogBoard.BlogEntry page = null;
boolean hasRights = sb.verifyAuthentication(header, true); boolean hasRights = sb.verifyAuthentication(header);
//final int display = (hasRights || post == null) ? 1 : post.getInt("display", 0); //final int display = (hasRights || post == null) ? 1 : post.getInt("display", 0);
//prop.put("display", display); //prop.put("display", display);

@ -44,12 +44,11 @@ import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.util.FileUtils; import net.yacy.kelondro.util.FileUtils;
import net.yacy.peers.Network; import net.yacy.peers.Network;
import net.yacy.search.Switchboard; import net.yacy.search.Switchboard;
import de.anomic.data.BlogBoard; import de.anomic.data.BlogBoard;
import de.anomic.data.BlogBoard.BlogEntry;
import de.anomic.data.BlogBoardComments; import de.anomic.data.BlogBoardComments;
import de.anomic.data.MessageBoard; import de.anomic.data.MessageBoard;
import de.anomic.data.UserDB; import de.anomic.data.UserDB;
import de.anomic.data.BlogBoard.BlogEntry;
import de.anomic.server.serverObjects; import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch; import de.anomic.server.serverSwitch;
@ -65,7 +64,7 @@ public class BlogComments {
public static serverObjects respond(final RequestHeader header, serverObjects post, final serverSwitch env) { public static serverObjects respond(final RequestHeader header, serverObjects post, final serverSwitch env) {
final Switchboard sb = (Switchboard) env; final Switchboard sb = (Switchboard) env;
final serverObjects prop = new serverObjects(); final serverObjects prop = new serverObjects();
boolean hasRights = sb.verifyAuthentication(header, true); boolean hasRights = sb.verifyAuthentication(header);
prop.put("mode_admin", hasRights ? "1" : "0"); prop.put("mode_admin", hasRights ? "1" : "0");

@ -85,7 +85,7 @@ public class Bookmarks {
prop.clear(); prop.clear();
sb = (Switchboard) env; sb = (Switchboard) env;
user = sb.userDB.getUser(header); user = sb.userDB.getUser(header);
isAdmin = (sb.verifyAuthentication(header, true) || user!= null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT)); isAdmin = (sb.verifyAuthentication(header) || user!= null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT));
// set user name // set user name
if (user != null) { if (user != null) {

@ -50,7 +50,7 @@ public class Collage {
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
final serverObjects prop = new serverObjects(); final serverObjects prop = new serverObjects();
final Switchboard sb = (Switchboard) env; final Switchboard sb = (Switchboard) env;
final boolean authenticated = sb.verifyAuthentication(header, false); final boolean authenticated = sb.verifyAuthentication(header);
final ResultImages.OriginEntry nextOrigin = ResultImages.next(!authenticated); final ResultImages.OriginEntry nextOrigin = ResultImages.next(!authenticated);
int posXMax = 800; int posXMax = 800;
int posYMax = 500; int posYMax = 500;

@ -43,7 +43,7 @@ public class ConfigParser {
if (post != null) { if (post != null) {
if (!sb.verifyAuthentication(header, false)) { if (!sb.verifyAuthentication(header)) {
// force log-in // force log-in
prop.put("AUTHENTICATE", "admin log-in"); prop.put("AUTHENTICATE", "admin log-in");
return prop; return prop;

@ -83,7 +83,6 @@
<form action="#[feedbackpage]#" method="post" enctype="multipart/form-data" accept-charset="UTF-8"> <form action="#[feedbackpage]#" method="post" enctype="multipart/form-data" accept-charset="UTF-8">
<div> <div>
<input type="hidden" name="process" value="#[tabletype]#" /> <input type="hidden" name="process" value="#[tabletype]#" />
<input type="hidden" name="hashpart" value="#[hashpart]#" />
<input type="hidden" name="domain" value="#[domain]#" /> <input type="hidden" name="domain" value="#[domain]#" />
<input type="submit" name="deletedomain" value="delete all" /> <input type="submit" name="deletedomain" value="delete all" />
</div> </div>

@ -60,8 +60,8 @@ public class CrawlResults {
boolean showDate = env.getConfigBool("IndexMonitorDate", true); boolean showDate = env.getConfigBool("IndexMonitorDate", true);
boolean showWords = env.getConfigBool("IndexMonitorWords", true); boolean showWords = env.getConfigBool("IndexMonitorWords", true);
boolean showTitle = env.getConfigBool("IndexMonitorTitle", true); boolean showTitle = env.getConfigBool("IndexMonitorTitle", true);
boolean showCountry = env.getConfigBool("IndexMonitorCountry", true); boolean showCountry = env.getConfigBool("IndexMonitorCountry", false);
boolean showIP = env.getConfigBool("IndexMonitorIP", true); boolean showIP = env.getConfigBool("IndexMonitorIP", false);
boolean showURL = env.getConfigBool("IndexMonitorURL", true); boolean showURL = env.getConfigBool("IndexMonitorURL", true);
if (post == null) { if (post == null) {
@ -92,7 +92,7 @@ public class CrawlResults {
post.containsKey("deleteentry")))) { post.containsKey("deleteentry")))) {
final String authorization = (header.get(RequestHeader.AUTHORIZATION, "xxxxxx")); final String authorization = (header.get(RequestHeader.AUTHORIZATION, "xxxxxx"));
if (authorization.length() != 0) { if (authorization.length() != 0) {
if (! sb.verifyAuthentication(header, true)){ if (! sb.verifyAuthentication(header)){
// force log-in (again, because wrong password was given) // force log-in (again, because wrong password was given)
prop.put("AUTHENTICATE", "admin log-in"); prop.put("AUTHENTICATE", "admin log-in");
return prop; return prop;
@ -122,8 +122,8 @@ public class CrawlResults {
} }
if (post.containsKey("deletedomain")) { if (post.containsKey("deletedomain")) {
final String hashpart = post.get("hashpart", null);
final String domain = post.get("domain", null); final String domain = post.get("domain", null);
final String hashpart = domain == null ? null : DigestURI.hosthash6(domain);
if (hashpart != null) { if (hashpart != null) {
// delete all urls for this domain from database // delete all urls for this domain from database
try { try {
@ -289,7 +289,6 @@ public class CrawlResults {
prop.put("table_domains_" + cnt + "_feedbackpage", "CrawlResults.html"); prop.put("table_domains_" + cnt + "_feedbackpage", "CrawlResults.html");
prop.put("table_domains_" + cnt + "_tabletype", tabletype.getCode()); prop.put("table_domains_" + cnt + "_tabletype", tabletype.getCode());
prop.put("table_domains_" + cnt + "_domain", domain); prop.put("table_domains_" + cnt + "_domain", domain);
prop.put("table_domains_" + cnt + "_hashpart", DigestURI.hosthash6(domain));
prop.put("table_domains_" + cnt + "_count", ResultURLs.domainCount(tabletype, domain)); prop.put("table_domains_" + cnt + "_count", ResultURLs.domainCount(tabletype, domain));
dark = !dark; dark = !dark;
cnt++; cnt++;

@ -34,7 +34,6 @@ import net.yacy.cora.protocol.RequestHeader;
import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.logging.Log;
import net.yacy.peers.operation.yacyRelease; import net.yacy.peers.operation.yacyRelease;
import net.yacy.search.Switchboard; import net.yacy.search.Switchboard;
import de.anomic.server.serverObjects; import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch; import de.anomic.server.serverSwitch;
@ -50,7 +49,7 @@ public class Steering {
final String requestIP = post.get(HeaderFramework.CONNECTION_PROP_CLIENTIP, "127.0.0.1"); final String requestIP = post.get(HeaderFramework.CONNECTION_PROP_CLIENTIP, "127.0.0.1");
// handle access rights // handle access rights
if (!sb.verifyAuthentication(header, false)) { if (!sb.verifyAuthentication(header)) {
Log.logInfo("STEERING", "log-in attempt for steering from " + requestIP); Log.logInfo("STEERING", "log-in attempt for steering from " + requestIP);
prop.put("AUTHENTICATE", "admin log-in"); // force log-in prop.put("AUTHENTICATE", "admin log-in"); // force log-in
return prop; return prop;

@ -64,14 +64,14 @@ public class Supporter {
// access control // access control
final boolean publicPage = sb.getConfigBool("publicSurftips", true); final boolean publicPage = sb.getConfigBool("publicSurftips", true);
final boolean authorizedAccess = sb.verifyAuthentication(header, false); final boolean authorizedAccess = sb.verifyAuthentication(header);
if ((publicPage) || (authorizedAccess)) { if ((publicPage) || (authorizedAccess)) {
// read voting // read voting
String hash; String hash;
if ((post != null) && ((hash = post.get("voteNegative", null)) != null)) { if ((post != null) && ((hash = post.get("voteNegative", null)) != null)) {
if (!sb.verifyAuthentication(header, false)) { if (!sb.verifyAuthentication(header)) {
prop.put("AUTHENTICATE", "admin log-in"); // force log-in prop.put("AUTHENTICATE", "admin log-in"); // force log-in
return prop; return prop;
} }
@ -85,7 +85,7 @@ public class Supporter {
} }
} }
if ((post != null) && ((hash = post.get("votePositive", null)) != null)) { if ((post != null) && ((hash = post.get("votePositive", null)) != null)) {
if (!sb.verifyAuthentication(header, false)) { if (!sb.verifyAuthentication(header)) {
prop.put("AUTHENTICATE", "admin log-in"); // force log-in prop.put("AUTHENTICATE", "admin log-in"); // force log-in
return prop; return prop;
} }

@ -63,7 +63,7 @@ public class Surftips {
// access control // access control
boolean publicPage = sb.getConfigBool("publicSurftips", true); boolean publicPage = sb.getConfigBool("publicSurftips", true);
final boolean authorizedAccess = sb.verifyAuthentication(header, false); final boolean authorizedAccess = sb.verifyAuthentication(header);
if ((post != null) && (post.containsKey("publicPage"))) { if ((post != null) && (post.containsKey("publicPage"))) {
if (!authorizedAccess) { if (!authorizedAccess) {
prop.put("AUTHENTICATE", "admin log-in"); // force log-in prop.put("AUTHENTICATE", "admin log-in"); // force log-in
@ -78,7 +78,7 @@ public class Surftips {
// read voting // read voting
String hash; String hash;
if ((post != null) && ((hash = post.get("voteNegative", null)) != null)) { if ((post != null) && ((hash = post.get("voteNegative", null)) != null)) {
if (!sb.verifyAuthentication(header, false)) { if (!sb.verifyAuthentication(header)) {
prop.put("AUTHENTICATE", "admin log-in"); // force log-in prop.put("AUTHENTICATE", "admin log-in"); // force log-in
return prop; return prop;
} }
@ -92,7 +92,7 @@ public class Surftips {
} }
} }
if ((post != null) && ((hash = post.get("votePositive", null)) != null)) { if ((post != null) && ((hash = post.get("votePositive", null)) != null)) {
if (!sb.verifyAuthentication(header, false)) { if (!sb.verifyAuthentication(header)) {
prop.put("AUTHENTICATE", "admin log-in"); // force log-in prop.put("AUTHENTICATE", "admin log-in"); // force log-in
return prop; return prop;
} }

@ -35,7 +35,6 @@ import net.yacy.kelondro.order.Base64Order;
import net.yacy.kelondro.order.Digest; import net.yacy.kelondro.order.Digest;
import net.yacy.search.Switchboard; import net.yacy.search.Switchboard;
import net.yacy.search.SwitchboardConstants; import net.yacy.search.SwitchboardConstants;
import de.anomic.data.UserDB; import de.anomic.data.UserDB;
import de.anomic.server.serverObjects; import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch; import de.anomic.server.serverSwitch;
@ -88,7 +87,7 @@ public class User{
prop.put("logged-in_limit_percent2", (100-percent)/3); prop.put("logged-in_limit_percent2", (100-percent)/3);
} }
//logged in via static Password //logged in via static Password
}else if(sb.verifyAuthentication(requestHeader, true)){ }else if(sb.verifyAuthentication(requestHeader)){
prop.put("logged-in", "2"); prop.put("logged-in", "2");
//identified via form-login //identified via form-login
//TODO: this does not work for a static admin, yet. //TODO: this does not work for a static admin, yet.

@ -92,7 +92,7 @@ public class ViewFile {
// get segment // get segment
Segment indexSegment = null; Segment indexSegment = null;
final boolean authorized = sb.verifyAuthentication(header, false); final boolean authorized = sb.verifyAuthentication(header);
if (post != null && post.containsKey("segment") && authorized) { if (post != null && post.containsKey("segment") && authorized) {
indexSegment = sb.indexSegments.segment(post.get("segment")); indexSegment = sb.indexSegments.segment(post.get("segment"));
} else { } else {
@ -145,7 +145,7 @@ public class ViewFile {
// this call forces the peer to download web pages // this call forces the peer to download web pages
// it is therefore protected by the admin password // it is therefore protected by the admin password
if (!sb.verifyAuthentication(header, false)) { if (!sb.verifyAuthentication(header)) {
prop.put("AUTHENTICATE", "admin log-in"); // force log-in prop.put("AUTHENTICATE", "admin log-in"); // force log-in
return prop; return prop;
} }

@ -67,7 +67,7 @@ public class ViewImage {
String urlString = post.get("url", ""); String urlString = post.get("url", "");
final String urlLicense = post.get("code", ""); final String urlLicense = post.get("code", "");
final boolean auth = (header.get(HeaderFramework.CONNECTION_PROP_CLIENTIP, "")).equals("localhost") || sb.verifyAuthentication(header, true); // handle access rights final boolean auth = (header.get(HeaderFramework.CONNECTION_PROP_CLIENTIP, "")).equals("localhost") || sb.verifyAuthentication(header); // handle access rights
DigestURI url = null; DigestURI url = null;
if ((urlString.length() > 0) && (auth)) try { if ((urlString.length() > 0) && (auth)) try {

@ -36,6 +36,7 @@ import java.util.Date;
import java.util.HashMap; import java.util.HashMap;
import java.util.Iterator; import java.util.Iterator;
import java.util.Locale; import java.util.Locale;
import java.util.Map;
import net.yacy.cora.document.UTF8; import net.yacy.cora.document.UTF8;
import net.yacy.cora.protocol.HeaderFramework; import net.yacy.cora.protocol.HeaderFramework;
@ -43,12 +44,10 @@ import net.yacy.cora.protocol.RequestHeader;
import net.yacy.kelondro.util.ByteBuffer; import net.yacy.kelondro.util.ByteBuffer;
import net.yacy.peers.NewsPool; import net.yacy.peers.NewsPool;
import net.yacy.search.Switchboard; import net.yacy.search.Switchboard;
import de.anomic.data.Diff; import de.anomic.data.Diff;
import de.anomic.data.wiki.WikiBoard; import de.anomic.data.wiki.WikiBoard;
import de.anomic.server.serverObjects; import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch; import de.anomic.server.serverSwitch;
import java.util.Map;
public class Wiki { public class Wiki {
private static final String ANONYMOUS = "anonymous"; private static final String ANONYMOUS = "anonymous";
@ -85,7 +84,7 @@ public class Wiki {
if (post != null && post.containsKey("access")) { if (post != null && post.containsKey("access")) {
// only the administrator may change the access right // only the administrator may change the access right
if (!sb.verifyAuthentication(header, true)) { if (!sb.verifyAuthentication(header)) {
// check access right for admin // check access right for admin
prop.put("AUTHENTICATE", "admin log-in"); // force log-in prop.put("AUTHENTICATE", "admin log-in"); // force log-in
return prop; return prop;
@ -104,7 +103,7 @@ public class Wiki {
if (post != null && post.containsKey("submit")) { if (post != null && post.containsKey("submit")) {
if ((access.equals("admin") && (!sb.verifyAuthentication(header, true)))) { if ((access.equals("admin") && (!sb.verifyAuthentication(header)))) {
// check access right for admin // check access right for admin
prop.put("AUTHENTICATE", "admin log-in"); // force log-in prop.put("AUTHENTICATE", "admin log-in"); // force log-in
return prop; return prop;
@ -128,7 +127,7 @@ public class Wiki {
} }
if (post != null && post.containsKey("edit")) { if (post != null && post.containsKey("edit")) {
if ((access.equals("admin") && (!sb.verifyAuthentication(header, true)))) { if ((access.equals("admin") && (!sb.verifyAuthentication(header)))) {
// check access right for admin // check access right for admin
prop.put("AUTHENTICATE", "admin log-in"); // force log-in prop.put("AUTHENTICATE", "admin log-in"); // force log-in
return prop; return prop;

@ -10,7 +10,7 @@ public class YMarks {
final Switchboard sb = (Switchboard) env; final Switchboard sb = (Switchboard) env;
final serverObjects prop = new serverObjects(); final serverObjects prop = new serverObjects();
final UserDB.Entry user = sb.userDB.getUser(header); final UserDB.Entry user = sb.userDB.getUser(header);
final boolean isAdmin = (sb.verifyAuthentication(header, false)); final boolean isAdmin = (sb.verifyAuthentication(header));
final boolean isAuthUser = user!= null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT); final boolean isAuthUser = user!= null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT);
if(isAdmin || isAuthUser) { if(isAdmin || isAuthUser) {

@ -8,7 +8,6 @@ import net.yacy.cora.protocol.HeaderFramework;
import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.protocol.RequestHeader;
import net.yacy.document.parser.html.CharacterCoding; import net.yacy.document.parser.html.CharacterCoding;
import net.yacy.search.Switchboard; import net.yacy.search.Switchboard;
import de.anomic.data.BookmarkHelper; import de.anomic.data.BookmarkHelper;
import de.anomic.data.BookmarksDB; import de.anomic.data.BookmarksDB;
import de.anomic.data.UserDB; import de.anomic.data.UserDB;
@ -42,14 +41,14 @@ public class get_bookmarks {
RSS(3), // .xml (.rss) RSS(3), // .xml (.rss)
RDF(4); // .xml RDF(4); // .xml
private int value; private final int value;
DisplayType(final int value) { DisplayType(final int value) {
this.value = value; this.value = value;
} }
int getValue() { int getValue() {
return value; return this.value;
} }
} }
@ -58,7 +57,7 @@ public class get_bookmarks {
prop.clear(); prop.clear();
sb = (Switchboard) env; sb = (Switchboard) env;
user = sb.userDB.getUser(header); user = sb.userDB.getUser(header);
isAdmin = (sb.verifyAuthentication(header, true) || user != null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT)); isAdmin = (sb.verifyAuthentication(header) || user != null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT));
// set user name // set user name
final String username; final String username;

@ -4,7 +4,6 @@ import java.util.Iterator;
import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.protocol.RequestHeader;
import net.yacy.search.Switchboard; import net.yacy.search.Switchboard;
import de.anomic.data.BookmarkHelper; import de.anomic.data.BookmarkHelper;
import de.anomic.data.BookmarksDB; import de.anomic.data.BookmarksDB;
import de.anomic.data.UserDB; import de.anomic.data.UserDB;
@ -23,7 +22,7 @@ public class get_folders {
prop.clear(); prop.clear();
sb = (Switchboard) env; sb = (Switchboard) env;
user = sb.userDB.getUser(header); user = sb.userDB.getUser(header);
isAdmin = (sb.verifyAuthentication(header, true) || user != null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT)); isAdmin = (sb.verifyAuthentication(header) || user != null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT));
// set user name // set user name
final String username; final String username;
@ -56,7 +55,7 @@ public class get_folders {
if (n == 0) n = 1; if (n == 0) n = 1;
int count = 0; int count = 0;
while (it.hasNext()) { while (it.hasNext()) {
String folder = it.next(); final String folder = it.next();
foldername = folder.split("/"); foldername = folder.split("/");
if (foldername.length == n+1) { if (foldername.length == n+1) {
prop.put("folders_"+count+"_foldername", foldername[n]); prop.put("folders_"+count+"_foldername", foldername[n]);

@ -6,7 +6,6 @@ import java.util.Set;
import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.protocol.RequestHeader;
import net.yacy.peers.NewsPool; import net.yacy.peers.NewsPool;
import net.yacy.search.Switchboard; import net.yacy.search.Switchboard;
import de.anomic.data.BookmarkHelper; import de.anomic.data.BookmarkHelper;
import de.anomic.data.BookmarksDB; import de.anomic.data.BookmarksDB;
import de.anomic.data.ListManager; import de.anomic.data.ListManager;
@ -25,7 +24,7 @@ public class add_p {
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
sb = (Switchboard) env; sb = (Switchboard) env;
isAdmin=sb.verifyAuthentication(header, true); isAdmin=sb.verifyAuthentication(header);
user = sb.userDB.getUser(header); user = sb.userDB.getUser(header);
// set user name // set user name
@ -45,7 +44,7 @@ public class add_p {
final String title=post.get("title",url); final String title=post.get("title",url);
final String description=post.get("description",""); final String description=post.get("description","");
String tagsString = post.get("tags",""); String tagsString = post.get("tags","");
String pathString = post.get("path","/unsorted"); final String pathString = post.get("path","/unsorted");
tagsString= tagsString + "," + pathString; tagsString= tagsString + "," + pathString;
final Set<String> tags = ListManager.string2set(BookmarkHelper.cleanTagsString(tagsString)); final Set<String> tags = ListManager.string2set(BookmarkHelper.cleanTagsString(tagsString));
final BookmarksDB.Bookmark bookmark = sb.bookmarksDB.createBookmark(url, username); final BookmarksDB.Bookmark bookmark = sb.bookmarksDB.createBookmark(url, username);

@ -6,7 +6,6 @@ import net.yacy.cora.date.ISO8601Formatter;
import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.protocol.RequestHeader;
import net.yacy.kelondro.order.Digest; import net.yacy.kelondro.order.Digest;
import net.yacy.search.Switchboard; import net.yacy.search.Switchboard;
import de.anomic.data.BookmarksDB; import de.anomic.data.BookmarksDB;
import de.anomic.server.serverObjects; import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch; import de.anomic.server.serverSwitch;
@ -15,7 +14,7 @@ public class all {
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
// return variable that accumulates replacements // return variable that accumulates replacements
final Switchboard switchboard = (Switchboard) env; final Switchboard switchboard = (Switchboard) env;
final boolean isAdmin=switchboard.verifyAuthentication(header, true); final boolean isAdmin=switchboard.verifyAuthentication(header);
final serverObjects prop = new serverObjects(); final serverObjects prop = new serverObjects();
Iterator<String> it; Iterator<String> it;

@ -5,7 +5,6 @@ import net.yacy.cora.document.ASCII;
import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.protocol.RequestHeader;
import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.data.meta.DigestURI;
import net.yacy.search.Switchboard; import net.yacy.search.Switchboard;
import de.anomic.server.serverObjects; import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch; import de.anomic.server.serverSwitch;
@ -14,7 +13,7 @@ public class delete_p {
// return variable that accumulates replacements // return variable that accumulates replacements
final Switchboard switchboard = (Switchboard) env; final Switchboard switchboard = (Switchboard) env;
final serverObjects prop = new serverObjects(); final serverObjects prop = new serverObjects();
final boolean isAdmin=switchboard.verifyAuthentication(header, true); final boolean isAdmin=switchboard.verifyAuthentication(header);
if(post!= null){ if(post!= null){
if(!isAdmin){ if(!isAdmin){
// force authentication if desired // force authentication if desired

@ -16,7 +16,7 @@ public class get {
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
// return variable that accumulates replacements // return variable that accumulates replacements
final Switchboard switchboard = (Switchboard) env; final Switchboard switchboard = (Switchboard) env;
final boolean isAdmin=switchboard.verifyAuthentication(header, true); final boolean isAdmin=switchboard.verifyAuthentication(header);
final serverObjects prop = new serverObjects(); final serverObjects prop = new serverObjects();
String tag = null; String tag = null;
final String date; final String date;

@ -1,9 +1,9 @@
import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.protocol.RequestHeader;
import net.yacy.search.Switchboard; import net.yacy.search.Switchboard;
import de.anomic.data.BookmarksDB.Bookmark;
import de.anomic.server.serverObjects; import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch; import de.anomic.server.serverSwitch;
import de.anomic.data.BookmarksDB.Bookmark;
public class addTag_p { public class addTag_p {
@ -12,7 +12,7 @@ public class addTag_p {
final Switchboard switchboard = (Switchboard) env; final Switchboard switchboard = (Switchboard) env;
final serverObjects prop = new serverObjects(); final serverObjects prop = new serverObjects();
boolean isAdmin = false; boolean isAdmin = false;
isAdmin = switchboard.verifyAuthentication(header, true); isAdmin = switchboard.verifyAuthentication(header);
prop.put("result", "0");//error prop.put("result", "0");//error
//rename tags //rename tags

@ -10,7 +10,7 @@ public class editTag_p {
final Switchboard switchboard = (Switchboard) env; final Switchboard switchboard = (Switchboard) env;
final serverObjects prop = new serverObjects(); final serverObjects prop = new serverObjects();
boolean isAdmin = false; boolean isAdmin = false;
isAdmin = switchboard.verifyAuthentication(header, true); isAdmin = switchboard.verifyAuthentication(header);
prop.put("result", "0");//error prop.put("result", "0");//error
//rename tags //rename tags

@ -3,7 +3,6 @@ import java.util.Iterator;
import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.protocol.RequestHeader;
import net.yacy.search.Switchboard; import net.yacy.search.Switchboard;
import de.anomic.data.BookmarksDB; import de.anomic.data.BookmarksDB;
import de.anomic.server.serverObjects; import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch; import de.anomic.server.serverSwitch;
@ -16,7 +15,7 @@ public class getTag {
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
// return variable that accumulates replacements // return variable that accumulates replacements
final Switchboard switchboard = (Switchboard) env; final Switchboard switchboard = (Switchboard) env;
final boolean isAdmin = switchboard.verifyAuthentication(header, true); final boolean isAdmin = switchboard.verifyAuthentication(header);
final serverObjects prop = new serverObjects(); final serverObjects prop = new serverObjects();
Iterator<BookmarksDB.Tag> it = null; Iterator<BookmarksDB.Tag> it = null;
String tagName = ""; String tagName = "";

@ -27,7 +27,7 @@ public class xbel {
prop.clear(); prop.clear();
switchboard = (Switchboard) env; switchboard = (Switchboard) env;
isAdmin=switchboard.verifyAuthentication(header, true); isAdmin=switchboard.verifyAuthentication(header);
if(post != null) { if(post != null) {
if(!isAdmin) { if(!isAdmin) {

@ -23,7 +23,7 @@ public class feed {
prop.put("item", "0"); prop.put("item", "0");
if ((post == null) || (env == null)) return prop; if ((post == null) || (env == null)) return prop;
final boolean authorized = sb.verifyAuthentication(header, false); final boolean authorized = sb.verifyAuthentication(header);
final String channelNames = post.get("set"); final String channelNames = post.get("set");
if (channelNames == null) return prop; if (channelNames == null) return prop;

@ -11,7 +11,6 @@ import net.yacy.search.Switchboard;
import net.yacy.search.SwitchboardConstants; import net.yacy.search.SwitchboardConstants;
import net.yacy.search.index.Segment; import net.yacy.search.index.Segment;
import net.yacy.search.index.Segments; import net.yacy.search.index.Segments;
import de.anomic.crawler.NoticedURL; import de.anomic.crawler.NoticedURL;
import de.anomic.crawler.retrieval.Request; import de.anomic.crawler.retrieval.Request;
import de.anomic.server.serverObjects; import de.anomic.server.serverObjects;
@ -34,9 +33,9 @@ public class queues_p {
//wikiCode wikiTransformer = new wikiCode(switchboard); //wikiCode wikiTransformer = new wikiCode(switchboard);
final serverObjects prop = new serverObjects(); final serverObjects prop = new serverObjects();
Segment segment = null; Segment segment = null;
boolean html = post != null && post.containsKey("html"); final boolean html = post != null && post.containsKey("html");
prop.setLocalized(html); prop.setLocalized(html);
if (post != null && post.containsKey("segment") && sb.verifyAuthentication(header, false)) { if (post != null && post.containsKey("segment") && sb.verifyAuthentication(header)) {
segment = sb.indexSegments.segment(post.get("segment")); segment = sb.indexSegments.segment(post.get("segment"));
} }
if (segment == null) segment = sb.indexSegments.segment(Segments.Process.PUBLIC); if (segment == null) segment = sb.indexSegments.segment(Segments.Process.PUBLIC);

@ -2,8 +2,6 @@
import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.protocol.RequestHeader;
import net.yacy.kelondro.io.ByteCount; import net.yacy.kelondro.io.ByteCount;
//import net.yacy.kelondro.io.ByteCountInputStream;
//import net.yacy.kelondro.io.ByteCountOutputStream;
import net.yacy.kelondro.util.MemoryControl; import net.yacy.kelondro.util.MemoryControl;
import net.yacy.kelondro.workflow.WorkflowProcessor; import net.yacy.kelondro.workflow.WorkflowProcessor;
import net.yacy.search.Switchboard; import net.yacy.search.Switchboard;
@ -21,9 +19,9 @@ public class status_p {
final Switchboard sb = (Switchboard) env; final Switchboard sb = (Switchboard) env;
final serverObjects prop = new serverObjects(); final serverObjects prop = new serverObjects();
Segment segment = null; Segment segment = null;
boolean html = post != null && post.containsKey("html"); final boolean html = post != null && post.containsKey("html");
prop.setLocalized(html); prop.setLocalized(html);
if (post != null && post.containsKey("segment") && sb.verifyAuthentication(header, false)) { if (post != null && post.containsKey("segment") && sb.verifyAuthentication(header)) {
segment = sb.indexSegments.segment(post.get("segment")); segment = sb.indexSegments.segment(post.get("segment"));
} }
if (segment == null) segment = sb.indexSegments.segment(Segments.Process.PUBLIC); if (segment == null) segment = sb.indexSegments.segment(Segments.Process.PUBLIC);

@ -46,7 +46,7 @@ public class termlist_p {
Segment segment = null; Segment segment = null;
final boolean delete = post != null && post.containsKey("delete"); final boolean delete = post != null && post.containsKey("delete");
final long mincount = post == null ? 10000 : post.getLong("mincount", 10000); final long mincount = post == null ? 10000 : post.getLong("mincount", 10000);
if (post != null && post.containsKey("segment") && sb.verifyAuthentication(header, false)) { if (post != null && post.containsKey("segment") && sb.verifyAuthentication(header)) {
segment = sb.indexSegments.segment(post.get("segment")); segment = sb.indexSegments.segment(post.get("segment"));
} }
if (segment == null) segment = sb.indexSegments.segment(Segments.Process.PUBLIC); if (segment == null) segment = sb.indexSegments.segment(Segments.Process.PUBLIC);

@ -38,7 +38,6 @@ import net.yacy.kelondro.logging.Log;
import net.yacy.search.Switchboard; import net.yacy.search.Switchboard;
import net.yacy.search.index.Segment; import net.yacy.search.index.Segment;
import net.yacy.search.index.Segments; import net.yacy.search.index.Segments;
import de.anomic.server.serverObjects; import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch; import de.anomic.server.serverSwitch;
@ -50,9 +49,9 @@ public class yacydoc {
final serverObjects prop = new serverObjects(); final serverObjects prop = new serverObjects();
final Segment segment; final Segment segment;
boolean html = post != null && post.containsKey("html"); final boolean html = post != null && post.containsKey("html");
prop.setLocalized(html); prop.setLocalized(html);
boolean authorized = sb.verifyAuthentication(header, false); final boolean authorized = sb.verifyAuthentication(header);
if (post != null && post.containsKey("segment") && authorized) { if (post != null && post.containsKey("segment") && authorized) {
segment = sb.indexSegments.segment(post.get("segment")); segment = sb.indexSegments.segment(post.get("segment"));
} else { } else {
@ -72,15 +71,15 @@ public class yacydoc {
if (post == null) return prop; if (post == null) return prop;
String urlstring = post.get("url", "").trim(); final String urlstring = post.get("url", "").trim();
String urlhash = post.get("urlhash", "").trim(); String urlhash = post.get("urlhash", "").trim();
if (urlstring.length() == 0 && urlhash.length() == 0) return prop; if (urlstring.length() == 0 && urlhash.length() == 0) return prop;
if (urlstring.length() > 0 && urlhash.length() == 0) { if (urlstring.length() > 0 && urlhash.length() == 0) {
try { try {
DigestURI url = new DigestURI(urlstring); final DigestURI url = new DigestURI(urlstring);
urlhash = ASCII.String(url.hash()); urlhash = ASCII.String(url.hash());
} catch (MalformedURLException e) { } catch (final MalformedURLException e) {
Log.logException(e); Log.logException(e);
} }
} }

@ -1,4 +1,5 @@
import java.io.IOException; import java.io.IOException;
import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.protocol.RequestHeader;
import net.yacy.document.Parser.Failure; import net.yacy.document.Parser.Failure;
import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.data.meta.DigestURI;
@ -21,7 +22,7 @@ public class add_ymark {
final serverObjects prop = new serverObjects(); final serverObjects prop = new serverObjects();
final UserDB.Entry user = sb.userDB.getUser(header); final UserDB.Entry user = sb.userDB.getUser(header);
final boolean isAdmin = (sb.verifyAuthentication(header, true)); final boolean isAdmin = (sb.verifyAuthentication(header));
final boolean isAuthUser = user!= null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT); final boolean isAuthUser = user!= null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT);
if(isAdmin || isAuthUser) { if(isAdmin || isAuthUser) {
@ -40,13 +41,13 @@ public class add_ymark {
try { try {
sb.tables.bookmarks.createBookmark(sb.loader, url, bmk_user, true, tags, folders); sb.tables.bookmarks.createBookmark(sb.loader, url, bmk_user, true, tags, folders);
prop.put("status", "1"); prop.put("status", "1");
} catch (IOException e) { } catch (final IOException e) {
// TODO Auto-generated catch block // TODO Auto-generated catch block
Log.logException(e); Log.logException(e);
} catch (Failure e) { } catch (final Failure e) {
// TODO Auto-generated catch block // TODO Auto-generated catch block
Log.logException(e); Log.logException(e);
} catch (RowSpaceExceededException e) { } catch (final RowSpaceExceededException e) {
// TODO Auto-generated catch block // TODO Auto-generated catch block
Log.logException(e); Log.logException(e);
} }
@ -54,7 +55,7 @@ public class add_ymark {
} else if(post.containsKey(YMarkEntry.BOOKMARK.URL.key())) { } else if(post.containsKey(YMarkEntry.BOOKMARK.URL.key())) {
String url = post.get(YMarkEntry.BOOKMARK.URL.key(),YMarkEntry.BOOKMARK.URL.deflt()); String url = post.get(YMarkEntry.BOOKMARK.URL.key(),YMarkEntry.BOOKMARK.URL.deflt());
boolean hasProtocol = false; boolean hasProtocol = false;
for (YMarkTables.PROTOCOLS p : YMarkTables.PROTOCOLS.values()) { for (final YMarkTables.PROTOCOLS p : YMarkTables.PROTOCOLS.values()) {
if(url.toLowerCase().startsWith(p.protocol())) { if(url.toLowerCase().startsWith(p.protocol())) {
hasProtocol = true; hasProtocol = true;
break; break;
@ -75,9 +76,9 @@ public class add_ymark {
try { try {
sb.tables.bookmarks.addBookmark(bmk_user, bmk, false, false); sb.tables.bookmarks.addBookmark(bmk_user, bmk, false, false);
} catch (IOException e) { } catch (final IOException e) {
Log.logException(e); Log.logException(e);
} catch (RowSpaceExceededException e) { } catch (final RowSpaceExceededException e) {
} }
prop.put("status", "1"); prop.put("status", "1");
} else { } else {

@ -21,7 +21,7 @@ public class delete_ymark {
final serverObjects prop = new serverObjects(); final serverObjects prop = new serverObjects();
final UserDB.Entry user = sb.userDB.getUser(header); final UserDB.Entry user = sb.userDB.getUser(header);
final boolean isAdmin = (sb.verifyAuthentication(header, true)); final boolean isAdmin = (sb.verifyAuthentication(header));
final boolean isAuthUser = user!= null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT); final boolean isAuthUser = user!= null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT);
if(isAdmin || isAuthUser) { if(isAdmin || isAuthUser) {
@ -38,9 +38,9 @@ public class delete_ymark {
} }
sb.tables.bookmarks.deleteBookmark(bmk_user, urlHash); sb.tables.bookmarks.deleteBookmark(bmk_user, urlHash);
prop.put("result", "1"); prop.put("result", "1");
} catch (IOException e) { } catch (final IOException e) {
Log.logException(e); Log.logException(e);
} catch (RowSpaceExceededException e) { } catch (final RowSpaceExceededException e) {
Log.logException(e); Log.logException(e);
} }
} else { } else {

@ -26,7 +26,7 @@ public class get_metadata {
final Switchboard sb = (Switchboard) env; final Switchboard sb = (Switchboard) env;
prop = new serverObjects(); prop = new serverObjects();
final UserDB.Entry user = sb.userDB.getUser(header); final UserDB.Entry user = sb.userDB.getUser(header);
final boolean isAdmin = (sb.verifyAuthentication(header, true)); final boolean isAdmin = (sb.verifyAuthentication(header));
final boolean isAuthUser = user!= null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT); final boolean isAuthUser = user!= null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT);
if(isAdmin || isAuthUser) { if(isAdmin || isAuthUser) {
@ -36,7 +36,7 @@ public class get_metadata {
String url = post.get(YMarkEntry.BOOKMARK.URL.key(),YMarkEntry.BOOKMARK.URL.deflt()); String url = post.get(YMarkEntry.BOOKMARK.URL.key(),YMarkEntry.BOOKMARK.URL.deflt());
boolean hasProtocol = false; boolean hasProtocol = false;
for (YMarkTables.PROTOCOLS p : YMarkTables.PROTOCOLS.values()) { for (final YMarkTables.PROTOCOLS p : YMarkTables.PROTOCOLS.values()) {
if(url.toLowerCase().startsWith(p.protocol())) { if(url.toLowerCase().startsWith(p.protocol())) {
hasProtocol = true; hasProtocol = true;
break; break;
@ -47,7 +47,7 @@ public class get_metadata {
} }
try { try {
YMarkMetadata meta = new YMarkMetadata(new DigestURI(url), sb.indexSegments); final YMarkMetadata meta = new YMarkMetadata(new DigestURI(url), sb.indexSegments);
final Document document = meta.loadDocument(sb.loader); final Document document = meta.loadDocument(sb.loader);
final EnumMap<YMarkMetadata.METADATA, String> metadata = meta.loadMetadata(); final EnumMap<YMarkMetadata.METADATA, String> metadata = meta.loadMetadata();
@ -70,13 +70,13 @@ public class get_metadata {
} }
prop.put("crawlstart", count); prop.put("crawlstart", count);
} catch (MalformedURLException e1) { } catch (final MalformedURLException e1) {
// TODO Auto-generated catch block // TODO Auto-generated catch block
e1.printStackTrace(); e1.printStackTrace();
} catch (IOException e) { } catch (final IOException e) {
// TODO Auto-generated catch block // TODO Auto-generated catch block
e.printStackTrace(); e.printStackTrace();
} catch (Failure e) { } catch (final Failure e) {
// TODO Auto-generated catch block // TODO Auto-generated catch block
e.printStackTrace(); e.printStackTrace();
} }

@ -33,7 +33,7 @@ public class get_tags {
prop = new serverObjects(); prop = new serverObjects();
final UserDB.Entry user = sb.userDB.getUser(header); final UserDB.Entry user = sb.userDB.getUser(header);
final boolean isAdmin = (sb.verifyAuthentication(header, true)); final boolean isAdmin = (sb.verifyAuthentication(header));
final boolean isAuthUser = user!= null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT); final boolean isAuthUser = user!= null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT);
if(isAdmin || isAuthUser) { if(isAdmin || isAuthUser) {
@ -49,13 +49,13 @@ public class get_tags {
final String[] tagArray = YMarkUtil.cleanTagsString(post.get(TAG)).split(YMarkUtil.TAGS_SEPARATOR); final String[] tagArray = YMarkUtil.cleanTagsString(post.get(TAG)).split(YMarkUtil.TAGS_SEPARATOR);
try { try {
tags = new TreeSet<YMarkTag>(sb.tables.bookmarks.getTags(sb.tables.bookmarks.getBookmarksByTag(bmk_user, tagArray)).values()); tags = new TreeSet<YMarkTag>(sb.tables.bookmarks.getTags(sb.tables.bookmarks.getBookmarksByTag(bmk_user, tagArray)).values());
} catch (IOException e) { } catch (final IOException e) {
return prop; return prop;
} }
} else { } else {
try { try {
tags = new TreeSet<YMarkTag>(sb.tables.bookmarks.getTags(bmk_user).values()); tags = new TreeSet<YMarkTag>(sb.tables.bookmarks.getTags(bmk_user).values());
} catch (IOException e) { } catch (final IOException e) {
return prop; return prop;
} }
} }

@ -37,7 +37,7 @@ public class get_treeview {
final Switchboard sb = (Switchboard) env; final Switchboard sb = (Switchboard) env;
prop = new serverObjects(); prop = new serverObjects();
final UserDB.Entry user = sb.userDB.getUser(header); final UserDB.Entry user = sb.userDB.getUser(header);
final boolean isAdmin = (sb.verifyAuthentication(header, true)); final boolean isAdmin = (sb.verifyAuthentication(header));
final boolean isAuthUser = user!= null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT); final boolean isAuthUser = user!= null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT);
@ -93,13 +93,13 @@ public class get_treeview {
try { try {
// it = sb.tables.bookmarks.folders.getFolders(bmk_user, root); // it = sb.tables.bookmarks.folders.getFolders(bmk_user, root);
it = sb.tables.bookmarks.getFolders(bmk_user, root).iterator(); it = sb.tables.bookmarks.getFolders(bmk_user, root).iterator();
} catch (IOException e) { } catch (final IOException e) {
Log.logException(e); Log.logException(e);
} }
int n = root.split(YMarkUtil.FOLDERS_SEPARATOR).length; int n = root.split(YMarkUtil.FOLDERS_SEPARATOR).length;
if (n == 0) n = 1; if (n == 0) n = 1;
while (it.hasNext()) { while (it.hasNext()) {
String folder = it.next(); final String folder = it.next();
foldername = folder.split(YMarkUtil.FOLDERS_SEPARATOR); foldername = folder.split(YMarkUtil.FOLDERS_SEPARATOR);
if (foldername.length == n+1) { if (foldername.length == n+1) {
prop.put("folders_"+count+"_foldername", foldername[n]); prop.put("folders_"+count+"_foldername", foldername[n]);
@ -146,7 +146,7 @@ public class get_treeview {
prop.put("folders_"+count+"_comma", ""); prop.put("folders_"+count+"_comma", "");
count++; count++;
prop.put("folders", count); prop.put("folders", count);
} catch (IOException e) { } catch (final IOException e) {
Log.logException(e); Log.logException(e);
} }
} else if(displayBmk && isBookmark) { } else if(displayBmk && isBookmark) {
@ -201,9 +201,9 @@ public class get_treeview {
count++; count++;
prop.put("folders", count); prop.put("folders", count);
} }
} catch (IOException e) { } catch (final IOException e) {
Log.logException(e); Log.logException(e);
} catch (RowSpaceExceededException e) { } catch (final RowSpaceExceededException e) {
Log.logException(e); Log.logException(e);
} }
} else if (isAutoTagger || isMetadata || isURLdb || isCrawlStart) { } else if (isAutoTagger || isMetadata || isURLdb || isCrawlStart) {
@ -240,11 +240,11 @@ public class get_treeview {
prop.put("folders", count); prop.put("folders", count);
} }
} catch (MalformedURLException e) { } catch (final MalformedURLException e) {
Log.logException(e); Log.logException(e);
} catch (IOException e) { } catch (final IOException e) {
Log.logException(e); Log.logException(e);
} catch (Failure e) { } catch (final Failure e) {
Log.logException(e); Log.logException(e);
} }
} }

@ -27,7 +27,7 @@ public class get_xbel {
final HashSet<String> alias = new HashSet<String>(); final HashSet<String> alias = new HashSet<String>();
final StringBuilder buffer = new StringBuilder(250); final StringBuilder buffer = new StringBuilder(250);
final UserDB.Entry user = sb.userDB.getUser(header); final UserDB.Entry user = sb.userDB.getUser(header);
final boolean isAdmin = (sb.verifyAuthentication(header, true)); final boolean isAdmin = (sb.verifyAuthentication(header));
final boolean isAuthUser = user!= null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT); final boolean isAuthUser = user!= null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT);
final String bmk_user; final String bmk_user;
@ -61,7 +61,7 @@ public class get_xbel {
try { try {
fit = sb.tables.bookmarks.getFolders(bmk_user, root).iterator(); fit = sb.tables.bookmarks.getFolders(bmk_user, root).iterator();
} catch (IOException e) { } catch (final IOException e) {
Log.logException(e); Log.logException(e);
} }
@ -87,7 +87,7 @@ public class get_xbel {
} }
try { try {
bit = sb.tables.bookmarks.getBookmarksByFolder(bmk_user, folder); bit = sb.tables.bookmarks.getBookmarksByFolder(bmk_user, folder);
} catch (IOException e) { } catch (final IOException e) {
// TODO: better error handling (avoid NPE) // TODO: better error handling (avoid NPE)
bit = null; bit = null;
} }

@ -14,8 +14,8 @@ import de.anomic.data.ymark.YMarkCrawlStart;
import de.anomic.data.ymark.YMarkDate; import de.anomic.data.ymark.YMarkDate;
import de.anomic.data.ymark.YMarkEntry; import de.anomic.data.ymark.YMarkEntry;
import de.anomic.data.ymark.YMarkTables; import de.anomic.data.ymark.YMarkTables;
import de.anomic.data.ymark.YMarkUtil;
import de.anomic.data.ymark.YMarkTables.TABLES; import de.anomic.data.ymark.YMarkTables.TABLES;
import de.anomic.data.ymark.YMarkUtil;
import de.anomic.server.serverObjects; import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch; import de.anomic.server.serverSwitch;
@ -39,7 +39,7 @@ public class get_ymark {
String query; String query;
final UserDB.Entry user = sb.userDB.getUser(header); final UserDB.Entry user = sb.userDB.getUser(header);
final boolean isAdmin = (sb.verifyAuthentication(header, true)); final boolean isAdmin = (sb.verifyAuthentication(header));
final boolean isAuthUser = user!= null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT); final boolean isAuthUser = user!= null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT);
Iterator<Tables.Row> bookmarks = null; Iterator<Tables.Row> bookmarks = null;
@ -83,7 +83,7 @@ public class get_ymark {
} }
total = result.size(); total = result.size();
bookmarks = result.iterator(); bookmarks = result.iterator();
} catch (IOException e) { } catch (final IOException e) {
Log.logException(e); Log.logException(e);
} }
prop.put("page", page); prop.put("page", page);
@ -116,7 +116,7 @@ public class get_ymark {
// put JSON // put JSON
prop.put("json_"+count+"_id", count); prop.put("json_"+count+"_id", count);
prop.put("json_"+count+"_hash", UTF8.String(bmk_row.getPK())); prop.put("json_"+count+"_hash", UTF8.String(bmk_row.getPK()));
for (YMarkEntry.BOOKMARK bmk : YMarkEntry.BOOKMARK.values()) { for (final YMarkEntry.BOOKMARK bmk : YMarkEntry.BOOKMARK.values()) {
if(bmk == YMarkEntry.BOOKMARK.PUBLIC) if(bmk == YMarkEntry.BOOKMARK.PUBLIC)
prop.put("json_"+count+"_"+bmk.key(), bmk_row.get(bmk.key(),bmk.deflt()).equals("false") ? 1 : 0); prop.put("json_"+count+"_"+bmk.key(), bmk_row.get(bmk.key(),bmk.deflt()).equals("false") ? 1 : 0);
else if(bmk == YMarkEntry.BOOKMARK.TAGS) else if(bmk == YMarkEntry.BOOKMARK.TAGS)
@ -154,7 +154,7 @@ public class get_ymark {
// put XML // put XML
prop.putXML("xml_"+count+"_id", UTF8.String(bmk_row.getPK())); prop.putXML("xml_"+count+"_id", UTF8.String(bmk_row.getPK()));
for (YMarkEntry.BOOKMARK bmk : YMarkEntry.BOOKMARK.values()) { for (final YMarkEntry.BOOKMARK bmk : YMarkEntry.BOOKMARK.values()) {
prop.putXML("xml_"+count+"_"+bmk.key(), bmk_row.get(bmk.key(),bmk.deflt())); prop.putXML("xml_"+count+"_"+bmk.key(), bmk_row.get(bmk.key(),bmk.deflt()));
} }

@ -38,7 +38,7 @@ public class import_ymark {
final Switchboard sb = (Switchboard) env; final Switchboard sb = (Switchboard) env;
final serverObjects prop = new serverObjects(); final serverObjects prop = new serverObjects();
final UserDB.Entry user = sb.userDB.getUser(header); final UserDB.Entry user = sb.userDB.getUser(header);
final boolean isAdmin = (sb.verifyAuthentication(header, true)); final boolean isAdmin = (sb.verifyAuthentication(header));
final boolean isAuthUser = user!= null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT); final boolean isAuthUser = user!= null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT);
final int queueSize = 200; final int queueSize = 200;

@ -30,7 +30,7 @@ public class manage_tags {
String replace; String replace;
final UserDB.Entry user = sb.userDB.getUser(header); final UserDB.Entry user = sb.userDB.getUser(header);
final boolean isAdmin = (sb.verifyAuthentication(header, true)); final boolean isAdmin = (sb.verifyAuthentication(header));
final boolean isAuthUser = user!= null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT); final boolean isAuthUser = user!= null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT);
if(isAdmin || isAuthUser) { if(isAdmin || isAuthUser) {
@ -74,9 +74,9 @@ public class manage_tags {
} }
sb.tables.bookmarks.replaceTags(row_iter, bmk_user, tags, replace); sb.tables.bookmarks.replaceTags(row_iter, bmk_user, tags, replace);
prop.put("status", 1); prop.put("status", 1);
} catch (IOException e) { } catch (final IOException e) {
Log.logException(e); Log.logException(e);
} catch (RowSpaceExceededException e) { } catch (final RowSpaceExceededException e) {
Log.logException(e); Log.logException(e);
} }
} else { } else {

@ -9,7 +9,6 @@ import java.util.Scanner;
import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.protocol.RequestHeader;
import net.yacy.search.Switchboard; import net.yacy.search.Switchboard;
import de.anomic.server.serverObjects; import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch; import de.anomic.server.serverSwitch;
@ -17,7 +16,7 @@ public class ynetSearch {
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
final Switchboard switchboard = (Switchboard) env; final Switchboard switchboard = (Switchboard) env;
final boolean isAdmin=switchboard.verifyAuthentication(header, true); final boolean isAdmin=switchboard.verifyAuthentication(header);
final serverObjects prop = new serverObjects(); final serverObjects prop = new serverObjects();
if(post != null){ if(post != null){

@ -51,7 +51,7 @@ public class index {
} }
// access control // access control
final boolean authorizedAccess = sb.verifyAuthentication(header, false); final boolean authorizedAccess = sb.verifyAuthentication(header);
if ((post != null) && (post.containsKey("publicPage"))) { if ((post != null) && (post.containsKey("publicPage"))) {
if (!authorizedAccess) { if (!authorizedAccess) {
prop.put("AUTHENTICATE", "admin log-in"); // force log-in prop.put("AUTHENTICATE", "admin log-in"); // force log-in

@ -87,7 +87,7 @@ public class yacysearch {
final Switchboard sb = (Switchboard) env; final Switchboard sb = (Switchboard) env;
sb.localSearchLastAccess = System.currentTimeMillis(); sb.localSearchLastAccess = System.currentTimeMillis();
final boolean searchAllowed = sb.getConfigBool("publicSearchpage", true) || sb.verifyAuthentication(header, false); final boolean searchAllowed = sb.getConfigBool("publicSearchpage", true) || sb.verifyAuthentication(header);
boolean authenticated = sb.adminAuthenticated(header) >= 2; boolean authenticated = sb.adminAuthenticated(header) >= 2;
if (!authenticated) { if (!authenticated) {
@ -484,7 +484,7 @@ public class yacysearch {
// if a minus-button was hit, remove a special reference first // if a minus-button was hit, remove a special reference first
if (post != null && post.containsKey("deleteref")) { if (post != null && post.containsKey("deleteref")) {
try { try {
if (!sb.verifyAuthentication(header, false)) { if (!sb.verifyAuthentication(header)) {
prop.put("AUTHENTICATE", "admin log-in"); // force log-in prop.put("AUTHENTICATE", "admin log-in"); // force log-in
return prop; return prop;
} }
@ -511,7 +511,7 @@ public class yacysearch {
// if a plus-button was hit, create new voting message // if a plus-button was hit, create new voting message
if (post != null && post.containsKey("recommendref")) { if (post != null && post.containsKey("recommendref")) {
if (!sb.verifyAuthentication(header, false)) { if (!sb.verifyAuthentication(header)) {
prop.put("AUTHENTICATE", "admin log-in"); // force log-in prop.put("AUTHENTICATE", "admin log-in"); // force log-in
return prop; return prop;
} }
@ -541,7 +541,7 @@ public class yacysearch {
// if a bookmarks-button was hit, create new bookmark entry // if a bookmarks-button was hit, create new bookmark entry
if (post != null && post.containsKey("bookmarkref")) { if (post != null && post.containsKey("bookmarkref")) {
if (!sb.verifyAuthentication(header, false)) { if (!sb.verifyAuthentication(header)) {
prop.put("AUTHENTICATE", "admin log-in"); // force log-in prop.put("AUTHENTICATE", "admin log-in"); // force log-in
return prop; return prop;
} }

@ -71,7 +71,7 @@ public class yacysearchitem {
final String eventID = post.get("eventID", ""); final String eventID = post.get("eventID", "");
final boolean authenticated = sb.adminAuthenticated(header) >= 2; final boolean authenticated = sb.adminAuthenticated(header) >= 2;
final int item = post.getInt("item", -1); final int item = post.getInt("item", -1);
final boolean auth = (header.get(HeaderFramework.CONNECTION_PROP_CLIENTIP, "")).equals("localhost") || sb.verifyAuthentication(header, true); final boolean auth = (header.get(HeaderFramework.CONNECTION_PROP_CLIENTIP, "")).equals("localhost") || sb.verifyAuthentication(header);
final RequestHeader.FileType fileType = header.fileType(); final RequestHeader.FileType fileType = header.fileType();
// default settings for blank item // default settings for blank item
@ -141,7 +141,7 @@ public class yacysearchitem {
prop.putHTML("content_link", result.urlstring()); prop.putHTML("content_link", result.urlstring());
prop.putHTML("content_showPictures_link", result.urlstring()); prop.putHTML("content_showPictures_link", result.urlstring());
prop.putHTML("content_target", target); prop.putHTML("content_target", target);
if (faviconURL != null && fileType == FileType.HTML) sb.loader.loadIfNotExistBackground(faviconURL.toNormalform(true, false), 1024 * 1024 * 10); if (faviconURL != null && fileType == FileType.HTML) sb.loader.loadIfNotExistBackground(faviconURL, 1024 * 1024 * 10);
prop.putHTML("content_faviconCode", sb.licensedURLs.aquireLicense(faviconURL)); // acquire license for favicon url loading prop.putHTML("content_faviconCode", sb.licensedURLs.aquireLicense(faviconURL)); // acquire license for favicon url loading
prop.put("content_urlhash", resulthashString); prop.put("content_urlhash", resulthashString);
prop.put("content_ranking", result.ranking); prop.put("content_ranking", result.ranking);
@ -216,7 +216,7 @@ public class yacysearchitem {
prop.put("content_item", "0"); prop.put("content_item", "0");
} else { } else {
final String license = sb.licensedURLs.aquireLicense(ms.href); final String license = sb.licensedURLs.aquireLicense(ms.href);
sb.loader.loadIfNotExistBackground(ms.href.toNormalform(true, false), 1024 * 1024 * 10); sb.loader.loadIfNotExistBackground(ms.href, 1024 * 1024 * 10);
prop.putHTML("content_item_hrefCache", (auth) ? "/ViewImage.png?url=" + ms.href.toNormalform(true, false) : ms.href.toNormalform(true, false)); prop.putHTML("content_item_hrefCache", (auth) ? "/ViewImage.png?url=" + ms.href.toNormalform(true, false) : ms.href.toNormalform(true, false));
prop.putHTML("content_item_href", ms.href.toNormalform(true, false)); prop.putHTML("content_item_href", ms.href.toNormalform(true, false));
prop.putHTML("content_item_target", target); prop.putHTML("content_item_target", target);

@ -1140,6 +1140,11 @@ public class MultiProtocolURI implements Serializable, Comparable<MultiProtocolU
final Locale locale = Domains.getLocale(this.hostAddress); final Locale locale = Domains.getLocale(this.hostAddress);
if (locale != null && locale.getCountry() != null && locale.getCountry().length() > 0) return locale; if (locale != null && locale.getCountry() != null && locale.getCountry().length() > 0) return locale;
} }
/*
if (this.hostAddress != null) {
return Domains.getLocale(this.hostAddress);
}
*/
return Domains.getLocale(this.host); return Domains.getLocale(this.host);
} }

@ -67,8 +67,8 @@ public class Domains {
private static final String PRESENT = ""; private static final String PRESENT = "";
private static final String LOCAL_PATTERNS = "10\\..*,127\\..*,172\\.(1[6-9]|2[0-9]|3[0-1])\\..*,169\\.254\\..*,192\\.168\\..*,localhost"; private static final String LOCAL_PATTERNS = "10\\..*,127\\..*,172\\.(1[6-9]|2[0-9]|3[0-1])\\..*,169\\.254\\..*,192\\.168\\..*,localhost";
private static final int MAX_NAME_CACHE_HIT_SIZE = 20000; private static final int MAX_NAME_CACHE_HIT_SIZE = 100000;
private static final int MAX_NAME_CACHE_MISS_SIZE = 20000; private static final int MAX_NAME_CACHE_MISS_SIZE = 100000;
private static final int CONCURRENCY_LEVEL = Runtime.getRuntime().availableProcessors() + 1; private static final int CONCURRENCY_LEVEL = Runtime.getRuntime().availableProcessors() + 1;
// a dns cache // a dns cache
@ -76,7 +76,7 @@ public class Domains {
private static final ARC<String, String> NAME_CACHE_MISS = new ConcurrentARC<String, String>(MAX_NAME_CACHE_MISS_SIZE, CONCURRENCY_LEVEL); private static final ARC<String, String> NAME_CACHE_MISS = new ConcurrentARC<String, String>(MAX_NAME_CACHE_MISS_SIZE, CONCURRENCY_LEVEL);
private static final ConcurrentHashMap<String, Object> LOOKUP_SYNC = new ConcurrentHashMap<String, Object>(100, 0.75f, Runtime.getRuntime().availableProcessors() * 2); private static final ConcurrentHashMap<String, Object> LOOKUP_SYNC = new ConcurrentHashMap<String, Object>(100, 0.75f, Runtime.getRuntime().availableProcessors() * 2);
private static List<Pattern> nameCacheNoCachingPatterns = Collections.synchronizedList(new LinkedList<Pattern>()); private static List<Pattern> nameCacheNoCachingPatterns = Collections.synchronizedList(new LinkedList<Pattern>());
private static final List<Pattern> LOCALHOST_PATTERNS = makePatterns(LOCAL_PATTERNS); private static final List<Pattern> INTRANET_PATTERNS = makePatterns(LOCAL_PATTERNS);
public static long cacheHit_Hit = 0, cacheHit_Miss = 0, cacheHit_Insert = 0; // for statistics only; do not write public static long cacheHit_Hit = 0, cacheHit_Miss = 0, cacheHit_Insert = 0; // for statistics only; do not write
public static long cacheMiss_Hit = 0, cacheMiss_Miss = 0, cacheMiss_Insert = 0; // for statistics only; do not write public static long cacheMiss_Hit = 0, cacheMiss_Miss = 0, cacheMiss_Insert = 0; // for statistics only; do not write
@ -809,7 +809,7 @@ public class Domains {
if (localHostAddresses.isEmpty()) return list; // give up if (localHostAddresses.isEmpty()) return list; // give up
for (final InetAddress a: localHostAddresses) { for (final InetAddress a: localHostAddresses) {
if (((0Xff & a.getAddress()[0]) == 127) || if (((0Xff & a.getAddress()[0]) == 127) ||
(!matchesList(a.getHostAddress(), LOCALHOST_PATTERNS))) continue; (!matchesList(a.getHostAddress(), INTRANET_PATTERNS))) continue;
list.add(a); list.add(a);
} }
return list; return list;
@ -894,7 +894,7 @@ public class Domains {
// FIXME IPv4 only // FIXME IPv4 only
// check local ip addresses // check local ip addresses
if (matchesList(host, LOCALHOST_PATTERNS)) return true; if (matchesList(host, INTRANET_PATTERNS)) return true;
if (host.startsWith("0:0:0:0:0:0:0:1")) return true; if (host.startsWith("0:0:0:0:0:0:0:1")) return true;
// check if there are other local IP addresses that are not in // check if there are other local IP addresses that are not in

@ -28,9 +28,11 @@ import java.util.AbstractMap;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collection; import java.util.Collection;
import java.util.Comparator; import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet; import java.util.HashSet;
import java.util.Iterator; import java.util.Iterator;
import java.util.Map; import java.util.Map;
import java.util.Random;
import java.util.Set; import java.util.Set;
@ -72,7 +74,7 @@ public final class ConcurrentARC<K, V> extends AbstractMap<K, V> implements Map<
* @param comparator a comparator for the key object which may be of type byte[] * @param comparator a comparator for the key object which may be of type byte[]
*/ */
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
public ConcurrentARC(final int cacheSize, final int partitions, Comparator<? super K> comparator) { public ConcurrentARC(final int cacheSize, final int partitions, final Comparator<? super K> comparator) {
int m = 1; int m = 1;
while (m < partitions) m = m * 2; while (m < partitions) m = m * 2;
int partitionSize = cacheSize / m; int partitionSize = cacheSize / m;
@ -98,7 +100,7 @@ public final class ConcurrentARC<K, V> extends AbstractMap<K, V> implements Map<
* @param s * @param s
* @param v * @param v
*/ */
public void insertIfAbsent(K s, V v) { public void insertIfAbsent(final K s, final V v) {
this.arc[getPartition(s)].insertIfAbsent(s, v); this.arc[getPartition(s)].insertIfAbsent(s, v);
} }
@ -109,7 +111,7 @@ public final class ConcurrentARC<K, V> extends AbstractMap<K, V> implements Map<
* @param v * @param v
* @return the value before inserting the new value * @return the value before inserting the new value
*/ */
public V putIfAbsent(K s, V v) { public V putIfAbsent(final K s, final V v) {
return this.arc[getPartition(s)].putIfAbsent(s, v); return this.arc[getPartition(s)].putIfAbsent(s, v);
} }
@ -139,9 +141,10 @@ public final class ConcurrentARC<K, V> extends AbstractMap<K, V> implements Map<
* @param value * @param value
* @return the keys that have the given value * @return the keys that have the given value
*/ */
public Collection<K> getKeys(V value) { public Collection<K> getKeys(final V value) {
ArrayList<K> keys = new ArrayList<K>(); final ArrayList<K> keys = new ArrayList<K>();
for (int i = 0; i < this.arc.length; i++) keys.addAll(this.arc[i].getKeys(value)); for (final ARC<K, V> element : this.arc)
keys.addAll(element.getKeys(value));
return keys; return keys;
} }
@ -172,7 +175,7 @@ public final class ConcurrentARC<K, V> extends AbstractMap<K, V> implements Map<
*/ */
@Override @Override
public final void clear() { public final void clear() {
for (ARC<K, V> a: this.arc) a.clear(); for (final ARC<K, V> a: this.arc) a.clear();
} }
/** /**
@ -182,7 +185,7 @@ public final class ConcurrentARC<K, V> extends AbstractMap<K, V> implements Map<
@Override @Override
public final int size() { public final int size() {
int s = 0; int s = 0;
for (ARC<K, V> a: this.arc) s += a.size(); for (final ARC<K, V> a: this.arc) s += a.size();
return s; return s;
} }
@ -202,9 +205,9 @@ public final class ConcurrentARC<K, V> extends AbstractMap<K, V> implements Map<
*/ */
@Override @Override
public Set<java.util.Map.Entry<K, V>> entrySet() { public Set<java.util.Map.Entry<K, V>> entrySet() {
Set<Map.Entry<K, V>> m = new HashSet<Map.Entry<K, V>>(); final Set<Map.Entry<K, V>> m = new HashSet<Map.Entry<K, V>>();
for (ARC<K, V> a: this.arc) { for (final ARC<K, V> a: this.arc) {
for (Map.Entry<K, V> entry: a.entrySet()) m.add(entry); for (final Map.Entry<K, V> entry: a.entrySet()) m.add(entry);
} }
return m; return m;
} }
@ -228,13 +231,47 @@ public final class ConcurrentARC<K, V> extends AbstractMap<K, V> implements Map<
private int getPartition(final Object x) { private int getPartition(final Object x) {
if (x instanceof byte[]) { if (x instanceof byte[]) {
int h = 0; int h = 0;
for (byte c: (byte[])x) h = 31 * h + (c & 0xFF); for (final byte c: (byte[])x) h = 31 * h + (c & 0xFF);
int p = h & mask; final int p = h & this.mask;
//String o = UTF8.String((byte[]) x); try { if (o.equals(latestObject)) throw new RuntimeException("ConcurrentARC: p = " + p + ", objectb = " + o); } catch (Exception e) { Log.logException(e); } latestObject = o; //String o = UTF8.String((byte[]) x); try { if (o.equals(latestObject)) throw new RuntimeException("ConcurrentARC: p = " + p + ", objectb = " + o); } catch (Exception e) { Log.logException(e); } latestObject = o;
return p; return p;
} }
int p = x.hashCode() & mask; final int p = x.hashCode() & this.mask;
//String o = x.toString(); try { if (o.equals(latestObject)) throw new RuntimeException("ConcurrentARC: p = " + p + ", objecto = " + o); } catch (Exception e) { Log.logException(e); } latestObject = o; //String o = x.toString(); try { if (o.equals(latestObject)) throw new RuntimeException("ConcurrentARC: p = " + p + ", objecto = " + o); } catch (Exception e) { Log.logException(e); } latestObject = o;
return p; return p;
} }
public static void main(final String[] args) {
final Random r = new Random();
final int testsize = 10000;
final ARC<String, String> a = new ConcurrentARC<String, String>(testsize * 3, Runtime.getRuntime().availableProcessors());
final Map<String, String> b = new HashMap<String, String>();
String key, value;
for (int i = 0; i < testsize; i++) {
key = "k" + r.nextInt();
value = "v" + r.nextInt();
a.insertIfAbsent(key, value);
b.put(key, value);
}
// now put half of the entries AGAIN into the ARC
int h = testsize / 2;
for (final Map.Entry<String, String> entry: b.entrySet()) {
a.put(entry.getKey(), entry.getValue());
if (h-- <= 0) break;
}
// test correctness
for (final Map.Entry<String, String> entry: b.entrySet()) {
if (!a.containsKey(entry.getKey())) {
System.out.println("missing: " + entry.getKey());
continue;
}
if (!a.get(entry.getKey()).equals(entry.getValue())) {
System.out.println("wrong: a = " + entry.getKey() + "," + a.get(entry.getKey()) + "; v = " + entry.getValue());
}
}
System.out.println("finished test!");
}
} }

@ -26,8 +26,10 @@
package net.yacy.cora.storage; package net.yacy.cora.storage;
import java.util.Collections; import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashMap; import java.util.LinkedHashMap;
import java.util.Map; import java.util.Map;
import java.util.Random;
public final class HashARC<K, V> extends SimpleARC<K, V> implements Map<K, V>, Iterable<Map.Entry<K, V>>, ARC<K, V> { public final class HashARC<K, V> extends SimpleARC<K, V> implements Map<K, V>, Iterable<Map.Entry<K, V>>, ARC<K, V> {
@ -48,4 +50,37 @@ public final class HashARC<K, V> extends SimpleARC<K, V> implements Map<K, V>, I
} }
}); });
} }
public static void main(final String[] args) {
final Random r = new Random();
final int testsize = 10000;
final ARC<String, String> a = new HashARC<String, String>(testsize * 2);
final Map<String, String> b = new HashMap<String, String>();
String key, value;
for (int i = 0; i < testsize; i++) {
key = "k" + r.nextInt();
value = "v" + r.nextInt();
a.insertIfAbsent(key, value);
b.put(key, value);
}
// now put half of the entries AGAIN into the ARC
int h = testsize / 2;
for (final Map.Entry<String, String> entry: b.entrySet()) {
a.put(entry.getKey(), entry.getValue());
if (h-- <= 0) break;
}
// test correctness
for (final Map.Entry<String, String> entry: b.entrySet()) {
if (!a.containsKey(entry.getKey())) {
System.out.println("missing: " + entry.getKey());
continue;
}
if (!a.get(entry.getKey()).equals(entry.getValue())) {
System.out.println("wrong: a = " + entry.getKey() + "," + a.get(entry.getKey()) + "; v = " + entry.getValue());
}
}
System.out.println("finished test!");
}
} }

@ -58,10 +58,10 @@ abstract class SimpleARC<K, V> extends AbstractMap<K, V> implements Map<K, V>, I
public final synchronized void insert(final K s, final V v) { public final synchronized void insert(final K s, final V v) {
if (this.levelB.containsKey(s)) { if (this.levelB.containsKey(s)) {
this.levelB.put(s, v); this.levelB.put(s, v);
assert (this.levelB.size() <= cacheSize); // the cache should shrink automatically assert (this.levelB.size() <= this.cacheSize); // the cache should shrink automatically
} else { } else {
this.levelA.put(s, v); this.levelA.put(s, v);
assert (this.levelA.size() <= cacheSize); // the cache should shrink automatically assert (this.levelA.size() <= this.cacheSize); // the cache should shrink automatically
} }
} }
@ -71,7 +71,7 @@ abstract class SimpleARC<K, V> extends AbstractMap<K, V> implements Map<K, V>, I
* @param s * @param s
* @param v * @param v
*/ */
public void insertIfAbsent(K s, V v) { public void insertIfAbsent(final K s, final V v) {
if (this.levelB.containsKey(s)) { if (this.levelB.containsKey(s)) {
return; return;
} else if (this.levelA.containsKey(s)) { } else if (this.levelA.containsKey(s)) {
@ -85,7 +85,7 @@ abstract class SimpleARC<K, V> extends AbstractMap<K, V> implements Map<K, V>, I
return; return;
} else { } else {
this.levelA.put(s, v); this.levelA.put(s, v);
assert (this.levelA.size() <= cacheSize); // the cache should shrink automatically assert (this.levelA.size() <= this.cacheSize); // the cache should shrink automatically
} }
} }
} }
@ -98,14 +98,14 @@ abstract class SimpleARC<K, V> extends AbstractMap<K, V> implements Map<K, V>, I
* @param v * @param v
* @return the value before inserting the new value * @return the value before inserting the new value
*/ */
public V putIfAbsent(K s, V v) { public V putIfAbsent(final K s, final V v) {
synchronized (this) { synchronized (this) {
V o = this.levelB.get(s); V o = this.levelB.get(s);
if (o != null) return o; if (o != null) return o;
o = this.levelA.get(s); o = this.levelA.get(s);
if (o != null) return o; if (o != null) return o;
this.levelA.put(s, v); this.levelA.put(s, v);
assert (this.levelA.size() <= cacheSize); // the cache should shrink automatically assert (this.levelA.size() <= this.cacheSize); // the cache should shrink automatically
return null; return null;
} }
} }
@ -117,12 +117,12 @@ abstract class SimpleARC<K, V> extends AbstractMap<K, V> implements Map<K, V>, I
*/ */
public final synchronized V put(final K s, final V v) { public final synchronized V put(final K s, final V v) {
if (this.levelB.containsKey(s)) { if (this.levelB.containsKey(s)) {
V r = this.levelB.put(s, v); final V r = this.levelB.put(s, v);
assert (this.levelB.size() <= cacheSize); // the cache should shrink automatically assert (this.levelB.size() <= this.cacheSize); // the cache should shrink automatically
return r; return r;
} else { } else {
V r = this.levelA.put(s, v); final V r = this.levelA.put(s, v);
assert (this.levelA.size() <= cacheSize); // the cache should shrink automatically assert (this.levelA.size() <= this.cacheSize); // the cache should shrink automatically
return r; return r;
} }
} }
@ -148,7 +148,7 @@ abstract class SimpleARC<K, V> extends AbstractMap<K, V> implements Map<K, V>, I
// move value from A to B; since it was already removed from A, just put it to B // move value from A to B; since it was already removed from A, just put it to B
//System.out.println("ARC: moving A->B, size(A) = " + this.levelA.size() + ", size(B) = " + this.levelB.size()); //System.out.println("ARC: moving A->B, size(A) = " + this.levelA.size() + ", size(B) = " + this.levelB.size());
this.levelB.put((K) s, v); this.levelB.put((K) s, v);
assert (this.levelB.size() <= cacheSize); // the cache should shrink automatically assert (this.levelB.size() <= this.cacheSize); // the cache should shrink automatically
} }
return v; return v;
} }
@ -158,15 +158,15 @@ abstract class SimpleARC<K, V> extends AbstractMap<K, V> implements Map<K, V>, I
* @param value * @param value
* @return the keys that have the given value * @return the keys that have the given value
*/ */
public Collection<K> getKeys(V value) { public Collection<K> getKeys(final V value) {
ArrayList<K> keys = new ArrayList<K>(); final ArrayList<K> keys = new ArrayList<K>();
synchronized (this.levelB) { synchronized (this.levelB) {
for (Map.Entry<K, V> entry: this.levelB.entrySet()) { for (final Map.Entry<K, V> entry: this.levelB.entrySet()) {
if (value.equals(entry.getValue())) keys.add(entry.getKey()); if (value.equals(entry.getValue())) keys.add(entry.getKey());
} }
} }
synchronized (this) { synchronized (this) {
for (Map.Entry<K, V> entry: this.levelA.entrySet()) { for (final Map.Entry<K, V> entry: this.levelA.entrySet()) {
if (value.equals(entry.getValue())) keys.add(entry.getKey()); if (value.equals(entry.getValue())) keys.add(entry.getKey());
} }
} }
@ -231,9 +231,9 @@ abstract class SimpleARC<K, V> extends AbstractMap<K, V> implements Map<K, V>, I
*/ */
@Override @Override
public final synchronized Set<Map.Entry<K, V>> entrySet() { public final synchronized Set<Map.Entry<K, V>> entrySet() {
Set<Map.Entry<K, V>> m = new HashSet<Map.Entry<K, V>>(); final Set<Map.Entry<K, V>> m = new HashSet<Map.Entry<K, V>>();
for (Map.Entry<K, V> entry: this.levelA.entrySet()) m.add(entry); for (final Map.Entry<K, V> entry: this.levelA.entrySet()) m.add(entry);
for (Map.Entry<K, V> entry: this.levelB.entrySet()) m.add(entry); for (final Map.Entry<K, V> entry: this.levelB.entrySet()) m.add(entry);
return m; return m;
} }
@ -245,4 +245,5 @@ abstract class SimpleARC<K, V> extends AbstractMap<K, V> implements Map<K, V>, I
public final int hashCode() { public final int hashCode() {
return this.levelA.hashCode(); return this.levelA.hashCode();
} }
} }

@ -27,6 +27,7 @@ import java.io.File;
import java.io.FileInputStream; import java.io.FileInputStream;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.net.MalformedURLException;
import java.util.HashMap; import java.util.HashMap;
import java.util.Map; import java.util.Map;
import java.util.Map.Entry; import java.util.Map.Entry;
@ -59,7 +60,10 @@ public class OAIListFriendsLoader {
public static void init(final LoaderDispatcher loader, final Map<String, File> moreFriends) { public static void init(final LoaderDispatcher loader, final Map<String, File> moreFriends) {
listFriends.putAll(moreFriends); listFriends.putAll(moreFriends);
if (loader != null) for (final Map.Entry<String, File> oaiFriend: listFriends.entrySet()) { if (loader != null) for (final Map.Entry<String, File> oaiFriend: listFriends.entrySet()) {
loader.loadIfNotExistBackground(oaiFriend.getKey(), oaiFriend.getValue(), Integer.MAX_VALUE); try {
loader.loadIfNotExistBackground(new DigestURI(oaiFriend.getKey()), oaiFriend.getValue(), Integer.MAX_VALUE);
} catch (final MalformedURLException e) {
}
} }
} }

@ -77,7 +77,7 @@ public final class LoaderDispatcher {
private final FTPLoader ftpLoader; private final FTPLoader ftpLoader;
private final SMBLoader smbLoader; private final SMBLoader smbLoader;
private final FileLoader fileLoader; private final FileLoader fileLoader;
private final ConcurrentHashMap<String, Semaphore> loaderSteering; // a map that delivers a 'finish' semaphore for urls private final ConcurrentHashMap<DigestURI, Semaphore> loaderSteering; // a map that delivers a 'finish' semaphore for urls
private final Log log; private final Log log;
public LoaderDispatcher(final Switchboard sb) { public LoaderDispatcher(final Switchboard sb) {
@ -90,7 +90,7 @@ public final class LoaderDispatcher {
this.ftpLoader = new FTPLoader(sb, this.log); this.ftpLoader = new FTPLoader(sb, this.log);
this.smbLoader = new SMBLoader(sb, this.log); this.smbLoader = new SMBLoader(sb, this.log);
this.fileLoader = new FileLoader(sb, this.log); this.fileLoader = new FileLoader(sb, this.log);
this.loaderSteering = new ConcurrentHashMap<String, Semaphore>(); this.loaderSteering = new ConcurrentHashMap<DigestURI, Semaphore>();
} }
public boolean isSupportedProtocol(final String protocol) { public boolean isSupportedProtocol(final String protocol) {
@ -153,8 +153,7 @@ public final class LoaderDispatcher {
} }
public Response load(final Request request, final CacheStrategy cacheStrategy, final int maxFileSize, final boolean checkBlacklist) throws IOException { public Response load(final Request request, final CacheStrategy cacheStrategy, final int maxFileSize, final boolean checkBlacklist) throws IOException {
final String url = request.url().toNormalform(true, false); Semaphore check = this.loaderSteering.get(request.url());
Semaphore check = this.loaderSteering.get(url);
if (check != null) { if (check != null) {
// a loading process may be going on for that url // a loading process may be going on for that url
try { check.tryAcquire(5, TimeUnit.SECONDS);} catch (final InterruptedException e) {} try { check.tryAcquire(5, TimeUnit.SECONDS);} catch (final InterruptedException e) {}
@ -162,15 +161,15 @@ public final class LoaderDispatcher {
// which may be successful faster because of a cache hit // which may be successful faster because of a cache hit
} }
this.loaderSteering.put(url, new Semaphore(0)); this.loaderSteering.put(request.url(), new Semaphore(0));
try { try {
final Response response = loadInternal(request, cacheStrategy, maxFileSize, checkBlacklist); final Response response = loadInternal(request, cacheStrategy, maxFileSize, checkBlacklist);
check = this.loaderSteering.remove(url); check = this.loaderSteering.remove(request.url());
if (check != null) check.release(1000); if (check != null) check.release(1000);
return response; return response;
} catch (final IOException e) { } catch (final IOException e) {
// release the semaphore anyway // release the semaphore anyway
check = this.loaderSteering.remove(url); check = this.loaderSteering.remove(request.url());
if (check != null) check.release(1000); if (check != null) check.release(1000);
//Log.logException(e); //Log.logException(e);
throw new IOException(e); throw new IOException(e);
@ -386,22 +385,22 @@ public final class LoaderDispatcher {
} }
} }
public void loadIfNotExistBackground(final String url, final File cache, final int maxFileSize) { public void loadIfNotExistBackground(final DigestURI url, final File cache, final int maxFileSize) {
new Loader(url, cache, maxFileSize, CacheStrategy.IFEXIST).start(); new Loader(url, cache, maxFileSize, CacheStrategy.IFEXIST).start();
} }
public void loadIfNotExistBackground(final String url, final int maxFileSize) { public void loadIfNotExistBackground(final DigestURI url, final int maxFileSize) {
new Loader(url, null, maxFileSize, CacheStrategy.IFEXIST).start(); new Loader(url, null, maxFileSize, CacheStrategy.IFEXIST).start();
} }
private class Loader extends Thread { private class Loader extends Thread {
private final String url; private final DigestURI url;
private final File cache; private final File cache;
private final int maxFileSize; private final int maxFileSize;
private final CacheStrategy cacheStrategy; private final CacheStrategy cacheStrategy;
public Loader(final String url, final File cache, final int maxFileSize, final CacheStrategy cacheStrategy) { public Loader(final DigestURI url, final File cache, final int maxFileSize, final CacheStrategy cacheStrategy) {
this.url = url; this.url = url;
this.cache = cache; this.cache = cache;
this.maxFileSize = maxFileSize; this.maxFileSize = maxFileSize;
@ -412,7 +411,7 @@ public final class LoaderDispatcher {
if (this.cache != null && this.cache.exists()) return; if (this.cache != null && this.cache.exists()) return;
try { try {
// load from the net // load from the net
final Response response = load(request(new DigestURI(this.url), false, true), this.cacheStrategy, this.maxFileSize, true); final Response response = load(request(this.url, false, true), this.cacheStrategy, this.maxFileSize, true);
final byte[] b = response.getContent(); final byte[] b = response.getContent();
if (this.cache != null) FileUtils.copy(b, this.cache); if (this.cache != null) FileUtils.copy(b, this.cache);
} catch (final MalformedURLException e) {} catch (final IOException e) {} } catch (final MalformedURLException e) {} catch (final IOException e) {}

@ -2348,7 +2348,7 @@ public final class Switchboard extends serverSwitch {
return 1; return 1;
} }
public boolean verifyAuthentication(final RequestHeader header, final boolean strict) { public boolean verifyAuthentication(final RequestHeader header) {
// handle access rights // handle access rights
switch (adminAuthenticated(header)) { switch (adminAuthenticated(header)) {
case 0: // wrong password given case 0: // wrong password given
@ -2357,7 +2357,7 @@ public final class Switchboard extends serverSwitch {
case 1: // no password given case 1: // no password given
return false; return false;
case 2: // no password stored case 2: // no password stored
return !strict; return true;
case 3: // soft-authenticated for localhost only case 3: // soft-authenticated for localhost only
return true; return true;
case 4: // hard-authenticated, all ok case 4: // hard-authenticated, all ok

Loading…
Cancel
Save