- tested the ARC methods

- removed strict authentication (if password is empty; this was buggy and not useful; can be switched on if necessary globally and not for each interface method)
- increased speed of CrawlResults page (no dns lookup any more)
- increased speed of favicon display (removed dns lookup)

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@8104 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 13 years ago
parent bc5df0eef5
commit e22f8497c9

@ -34,20 +34,19 @@ import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import net.yacy.cora.document.UTF8;
import net.yacy.cora.protocol.HeaderFramework;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.peers.NewsPool;
import net.yacy.search.Switchboard;
import de.anomic.data.BlogBoard;
import de.anomic.data.UserDB;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import java.util.List;
import java.util.Map;
public class Blog {
@ -65,13 +64,13 @@ public class Blog {
final serverObjects prop = new serverObjects();
BlogBoard.BlogEntry page = null;
boolean hasRights = sb.verifyAuthentication(header, true);
boolean hasRights = sb.verifyAuthentication(header);
//final int display = (hasRights || post == null) ? 1 : post.getInt("display", 0);
//prop.put("display", display);
//prop.put("display", display);
prop.put("display", 1); // Fixed to 1
final boolean xml = (header.get(HeaderFramework.CONNECTION_PROP_PATH)).endsWith(".xml");
final String address = sb.peers.mySeed().getPublicAddress();
@ -249,7 +248,7 @@ public class Blog {
int start,
int num,
final boolean hasRights,
final boolean xml)
final boolean xml)
{
final Iterator<String> i = switchboard.blogDB.getBlogIterator(false);
@ -289,7 +288,7 @@ public class Blog {
prop.put("mode_preventries_start", prevstart);
prop.put("mode_preventries_num", num);
} else prop.put("mode_preventries", "0");
return prop;
}
@ -300,7 +299,7 @@ public class Blog {
final String address,
final int number,
final boolean hasRights,
final boolean xml)
final boolean xml)
{
prop.putHTML("mode_entries_" + number + "_subject", UTF8.String(entry.getSubject()));

@ -44,12 +44,11 @@ import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.util.FileUtils;
import net.yacy.peers.Network;
import net.yacy.search.Switchboard;
import de.anomic.data.BlogBoard;
import de.anomic.data.BlogBoard.BlogEntry;
import de.anomic.data.BlogBoardComments;
import de.anomic.data.MessageBoard;
import de.anomic.data.UserDB;
import de.anomic.data.BlogBoard.BlogEntry;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
@ -65,7 +64,7 @@ public class BlogComments {
public static serverObjects respond(final RequestHeader header, serverObjects post, final serverSwitch env) {
final Switchboard sb = (Switchboard) env;
final serverObjects prop = new serverObjects();
boolean hasRights = sb.verifyAuthentication(header, true);
boolean hasRights = sb.verifyAuthentication(header);
prop.put("mode_admin", hasRights ? "1" : "0");
@ -107,7 +106,7 @@ public class BlogComments {
final BlogBoard.BlogEntry page = sb.blogDB.readBlogEntry(pagename); //maybe "if(page == null)"
final boolean pageExists = sb.blogDB.contains(pagename);
// comments not allowed
prop.put("mode_allow", (page.getCommentMode() == 0) ? 0 : 1);
@ -217,12 +216,12 @@ public class BlogComments {
while (i.hasNext() && count < num) {
pageid = i.next();
if(start > 0) {
start--;
continue;
}
entry = sb.blogCommentDB.read(pageid);
if (commentMode == 2 && !hasRights && !entry.isAllowed()) {
@ -309,7 +308,7 @@ public class BlogComments {
.append("/")
.append(msgEntry.authorHash())
.append("\nMessage to: ")
.append(msgEntry.recipient())
.append(msgEntry.recipient())
.append("/")
.append(msgEntry.recipientHash())
.append("\nCategory: ")

@ -85,7 +85,7 @@ public class Bookmarks {
prop.clear();
sb = (Switchboard) env;
user = sb.userDB.getUser(header);
isAdmin = (sb.verifyAuthentication(header, true) || user!= null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT));
isAdmin = (sb.verifyAuthentication(header) || user!= null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT));
// set user name
if (user != null) {

@ -50,7 +50,7 @@ public class Collage {
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
final serverObjects prop = new serverObjects();
final Switchboard sb = (Switchboard) env;
final boolean authenticated = sb.verifyAuthentication(header, false);
final boolean authenticated = sb.verifyAuthentication(header);
final ResultImages.OriginEntry nextOrigin = ResultImages.next(!authenticated);
int posXMax = 800;
int posYMax = 500;

@ -41,17 +41,17 @@ public class ConfigParser {
final serverObjects prop = new serverObjects();
final Switchboard sb = (Switchboard) env;
if (post != null) {
if (!sb.verifyAuthentication(header, false)) {
if (!sb.verifyAuthentication(header)) {
// force log-in
prop.put("AUTHENTICATE", "admin log-in");
return prop;
}
if (post.containsKey("parserSettings")) {
post.remove("parserSettings");
for (final Parser parser: TextParser.parsers()) {
for (final String mimeType: parser.supportedMimeTypes()) {
TextParser.grantMime(mimeType, "on".equals(post.get("mimename_" + mimeType, "")));
@ -60,11 +60,11 @@ public class ConfigParser {
env.setConfig(SwitchboardConstants.PARSER_MIME_DENY, TextParser.getDenyMime());
}
}
int i = 0;
int i = 0;
for (final Parser parser: TextParser.parsers()) {
prop.put("parser_" + i + "_name", parser.getName());
int mimeIdx = 0;
for (final String mimeType: parser.supportedMimeTypes()) {
prop.put("parser_" + i + "_mime_" + mimeIdx + "_mimetype", mimeType);
@ -74,9 +74,9 @@ public class ConfigParser {
prop.put("parser_" + i + "_mime", mimeIdx);
i++;
}
prop.put("parser", i);
// return rewrite properties
return prop;
}

@ -83,7 +83,6 @@
<form action="#[feedbackpage]#" method="post" enctype="multipart/form-data" accept-charset="UTF-8">
<div>
<input type="hidden" name="process" value="#[tabletype]#" />
<input type="hidden" name="hashpart" value="#[hashpart]#" />
<input type="hidden" name="domain" value="#[domain]#" />
<input type="submit" name="deletedomain" value="delete all" />
</div>

@ -60,8 +60,8 @@ public class CrawlResults {
boolean showDate = env.getConfigBool("IndexMonitorDate", true);
boolean showWords = env.getConfigBool("IndexMonitorWords", true);
boolean showTitle = env.getConfigBool("IndexMonitorTitle", true);
boolean showCountry = env.getConfigBool("IndexMonitorCountry", true);
boolean showIP = env.getConfigBool("IndexMonitorIP", true);
boolean showCountry = env.getConfigBool("IndexMonitorCountry", false);
boolean showIP = env.getConfigBool("IndexMonitorIP", false);
boolean showURL = env.getConfigBool("IndexMonitorURL", true);
if (post == null) {
@ -92,7 +92,7 @@ public class CrawlResults {
post.containsKey("deleteentry")))) {
final String authorization = (header.get(RequestHeader.AUTHORIZATION, "xxxxxx"));
if (authorization.length() != 0) {
if (! sb.verifyAuthentication(header, true)){
if (! sb.verifyAuthentication(header)){
// force log-in (again, because wrong password was given)
prop.put("AUTHENTICATE", "admin log-in");
return prop;
@ -122,8 +122,8 @@ public class CrawlResults {
}
if (post.containsKey("deletedomain")) {
final String hashpart = post.get("hashpart", null);
final String domain = post.get("domain", null);
final String hashpart = domain == null ? null : DigestURI.hosthash6(domain);
if (hashpart != null) {
// delete all urls for this domain from database
try {
@ -289,7 +289,6 @@ public class CrawlResults {
prop.put("table_domains_" + cnt + "_feedbackpage", "CrawlResults.html");
prop.put("table_domains_" + cnt + "_tabletype", tabletype.getCode());
prop.put("table_domains_" + cnt + "_domain", domain);
prop.put("table_domains_" + cnt + "_hashpart", DigestURI.hosthash6(domain));
prop.put("table_domains_" + cnt + "_count", ResultURLs.domainCount(tabletype, domain));
dark = !dark;
cnt++;

@ -34,7 +34,6 @@ import net.yacy.cora.protocol.RequestHeader;
import net.yacy.kelondro.logging.Log;
import net.yacy.peers.operation.yacyRelease;
import net.yacy.search.Switchboard;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
@ -48,9 +47,9 @@ public class Steering {
prop.put("info", "0"); //no information submitted
final String requestIP = post.get(HeaderFramework.CONNECTION_PROP_CLIENTIP, "127.0.0.1");
// handle access rights
if (!sb.verifyAuthentication(header, false)) {
if (!sb.verifyAuthentication(header)) {
Log.logInfo("STEERING", "log-in attempt for steering from " + requestIP);
prop.put("AUTHENTICATE", "admin log-in"); // force log-in
return prop;
@ -60,7 +59,7 @@ public class Steering {
Log.logInfo("STEERING", "shutdown request from " + requestIP);
sb.terminate(10, "shutdown request from Steering; ip = " + requestIP);
prop.put("info", "3");
return prop;
}
@ -68,10 +67,10 @@ public class Steering {
Log.logInfo("STEERING", "restart request from " + requestIP);
yacyRelease.restart();
prop.put("info", "4");
return prop;
}
if (post.containsKey("update")) {
Log.logInfo("STEERING", "update request from " + requestIP);
final boolean devenvironment = new File(sb.getAppPath(), ".svn").exists();
@ -82,7 +81,7 @@ public class Steering {
}
prop.put("info", "5");
prop.putHTML("info_release", releaseFileName);
return prop;
}
return prop;

@ -64,14 +64,14 @@ public class Supporter {
// access control
final boolean publicPage = sb.getConfigBool("publicSurftips", true);
final boolean authorizedAccess = sb.verifyAuthentication(header, false);
final boolean authorizedAccess = sb.verifyAuthentication(header);
if ((publicPage) || (authorizedAccess)) {
// read voting
String hash;
if ((post != null) && ((hash = post.get("voteNegative", null)) != null)) {
if (!sb.verifyAuthentication(header, false)) {
if (!sb.verifyAuthentication(header)) {
prop.put("AUTHENTICATE", "admin log-in"); // force log-in
return prop;
}
@ -85,7 +85,7 @@ public class Supporter {
}
}
if ((post != null) && ((hash = post.get("votePositive", null)) != null)) {
if (!sb.verifyAuthentication(header, false)) {
if (!sb.verifyAuthentication(header)) {
prop.put("AUTHENTICATE", "admin log-in"); // force log-in
return prop;
}

@ -63,7 +63,7 @@ public class Surftips {
// access control
boolean publicPage = sb.getConfigBool("publicSurftips", true);
final boolean authorizedAccess = sb.verifyAuthentication(header, false);
final boolean authorizedAccess = sb.verifyAuthentication(header);
if ((post != null) && (post.containsKey("publicPage"))) {
if (!authorizedAccess) {
prop.put("AUTHENTICATE", "admin log-in"); // force log-in
@ -78,7 +78,7 @@ public class Surftips {
// read voting
String hash;
if ((post != null) && ((hash = post.get("voteNegative", null)) != null)) {
if (!sb.verifyAuthentication(header, false)) {
if (!sb.verifyAuthentication(header)) {
prop.put("AUTHENTICATE", "admin log-in"); // force log-in
return prop;
}
@ -92,7 +92,7 @@ public class Surftips {
}
}
if ((post != null) && ((hash = post.get("votePositive", null)) != null)) {
if (!sb.verifyAuthentication(header, false)) {
if (!sb.verifyAuthentication(header)) {
prop.put("AUTHENTICATE", "admin log-in"); // force log-in
return prop;
}

@ -1,4 +1,4 @@
//User.java
//User.java
//-----------------------
//part of the AnomicHTTPD caching proxy
//(C) by Michael Peter Christen; mc@yacy.net
@ -35,14 +35,13 @@ import net.yacy.kelondro.order.Base64Order;
import net.yacy.kelondro.order.Digest;
import net.yacy.search.Switchboard;
import net.yacy.search.SwitchboardConstants;
import de.anomic.data.UserDB;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.server.servletProperties;
public class User{
public static servletProperties respond(final RequestHeader requestHeader, final serverObjects post, final serverSwitch env) {
final servletProperties prop = new servletProperties();
final Switchboard sb = Switchboard.getSwitchboard();
@ -70,7 +69,7 @@ public class User{
}
}
}
//identified via userDB
if(entry != null){
prop.put("logged-in", "1");
@ -88,7 +87,7 @@ public class User{
prop.put("logged-in_limit_percent2", (100-percent)/3);
}
//logged in via static Password
}else if(sb.verifyAuthentication(requestHeader, true)){
}else if(sb.verifyAuthentication(requestHeader)){
prop.put("logged-in", "2");
//identified via form-login
//TODO: this does not work for a static admin, yet.
@ -98,7 +97,7 @@ public class User{
final String username=post.get("username");
final String password=post.get("password");
prop.put("logged-in_username", username);
entry=sb.userDB.passwordAuth(username, password);
final boolean staticAdmin = sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, "").equals(
Digest.encodeMD5Hex(
@ -111,12 +110,12 @@ public class User{
cookie=sb.userDB.getCookie(entry);
else if(staticAdmin)
cookie=sb.userDB.getAdminCookie();
if(entry != null || staticAdmin){
final ResponseHeader outgoingHeader=new ResponseHeader();
outgoingHeader.setCookie("login", cookie);
prop.setOutgoingHeader(outgoingHeader);
prop.put("logged-in", "1");
prop.put("logged-in_identified-by", "1");
prop.putHTML("logged-in_username", username);
@ -125,7 +124,7 @@ public class User{
}
}
}
if(post!= null && entry != null){
if(post.containsKey("changepass")){
prop.put("status", "1"); //password

@ -92,7 +92,7 @@ public class ViewFile {
// get segment
Segment indexSegment = null;
final boolean authorized = sb.verifyAuthentication(header, false);
final boolean authorized = sb.verifyAuthentication(header);
if (post != null && post.containsKey("segment") && authorized) {
indexSegment = sb.indexSegments.segment(post.get("segment"));
} else {
@ -145,7 +145,7 @@ public class ViewFile {
// this call forces the peer to download web pages
// it is therefore protected by the admin password
if (!sb.verifyAuthentication(header, false)) {
if (!sb.verifyAuthentication(header)) {
prop.put("AUTHENTICATE", "admin log-in"); // force log-in
return prop;
}

@ -67,7 +67,7 @@ public class ViewImage {
String urlString = post.get("url", "");
final String urlLicense = post.get("code", "");
final boolean auth = (header.get(HeaderFramework.CONNECTION_PROP_CLIENTIP, "")).equals("localhost") || sb.verifyAuthentication(header, true); // handle access rights
final boolean auth = (header.get(HeaderFramework.CONNECTION_PROP_CLIENTIP, "")).equals("localhost") || sb.verifyAuthentication(header); // handle access rights
DigestURI url = null;
if ((urlString.length() > 0) && (auth)) try {

@ -1,4 +1,4 @@
// Wiki.java
// Wiki.java
// -----------------------
// part of the AnomicHTTPD caching proxy
// (C) by Michael Peter Christen; mc@yacy.net
@ -36,6 +36,7 @@ import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Locale;
import java.util.Map;
import net.yacy.cora.document.UTF8;
import net.yacy.cora.protocol.HeaderFramework;
@ -43,12 +44,10 @@ import net.yacy.cora.protocol.RequestHeader;
import net.yacy.kelondro.util.ByteBuffer;
import net.yacy.peers.NewsPool;
import net.yacy.search.Switchboard;
import de.anomic.data.Diff;
import de.anomic.data.wiki.WikiBoard;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import java.util.Map;
public class Wiki {
private static final String ANONYMOUS = "anonymous";
@ -71,7 +70,7 @@ public class Wiki {
}
prop.put("topmenu", sb.getConfigBool("publicTopmenu", true) ? 1 : 0);
String access = sb.getConfig("WikiAccess", "admin");
final String pagename = get(post, "page", "start");
final String ip = get(post, HeaderFramework.CONNECTION_PROP_CLIENTIP, "127.0.0.1");
@ -82,15 +81,15 @@ public class Wiki {
author = (sb.peers.mySeed() == null) ? ANONYMOUS : sb.peers.mySeed().get("Name", ANONYMOUS);
}
}
if (post != null && post.containsKey("access")) {
// only the administrator may change the access right
if (!sb.verifyAuthentication(header, true)) {
if (!sb.verifyAuthentication(header)) {
// check access right for admin
prop.put("AUTHENTICATE", "admin log-in"); // force log-in
return prop;
}
access = post.get("access", "admin");
sb.setConfig("WikiAccess", access);
}
@ -101,15 +100,15 @@ public class Wiki {
}
WikiBoard.Entry page = sb.wikiDB.read(pagename);
if (post != null && post.containsKey("submit")) {
if ((access.equals("admin") && (!sb.verifyAuthentication(header, true)))) {
if ((access.equals("admin") && (!sb.verifyAuthentication(header)))) {
// check access right for admin
prop.put("AUTHENTICATE", "admin log-in"); // force log-in
return prop;
}
// store a new page
byte[] content;
content = UTF8.getBytes(post.get("content", ""));
@ -126,14 +125,14 @@ public class Wiki {
prop.putHTML("LOCATION", "/Wiki.html?page=" + pagename);
prop.put("LOCATION", prop.get("LOCATION"));
}
if (post != null && post.containsKey("edit")) {
if ((access.equals("admin") && (!sb.verifyAuthentication(header, true)))) {
if ((access.equals("admin") && (!sb.verifyAuthentication(header)))) {
// check access right for admin
prop.put("AUTHENTICATE", "admin log-in"); // force log-in
return prop;
}
prop.put("mode", "1"); //edit
prop.putHTML("mode_author", author);
prop.putHTML("mode_page-code", UTF8.String(page.page()));
@ -177,7 +176,7 @@ public class Wiki {
prop.put("mode", "4");
prop.putHTML("mode_page", pagename);
prop.putHTML("mode_error_page", pagename);
try {
final Iterator<byte[]> it = sb.wikiDB.keysBkp(true);
WikiBoard.Entry entry;
@ -201,26 +200,26 @@ public class Wiki {
count++;
}
count--; // don't show current version
if (!oldselected) { // select latest old entry
prop.put("mode_error_versions_" + (count - 1) + "_oldselected", "1");
}
if (!newselected) { // select latest new entry (== current)
prop.put("mode_error_curselected", "1");
}
if (count == 0) {
prop.put("mode_error", "2"); // no entries found
} else {
prop.put("mode_error_versions", count);
}
entry = sb.wikiDB.read(pagename);
if (entry != null) {
prop.put("mode_error_curdate", WikiBoard.dateString(entry.date()));
prop.put("mode_error_curfdate", dateString(entry.date()));
}
if (nentry == null) {
nentry = entry;
}
@ -264,7 +263,7 @@ public class Wiki {
/**
* get key from post, use dflt if (not present or post == null)
*
*
* @param post
* @param string
* @param string2

@ -10,7 +10,7 @@ public class YMarks {
final Switchboard sb = (Switchboard) env;
final serverObjects prop = new serverObjects();
final UserDB.Entry user = sb.userDB.getUser(header);
final boolean isAdmin = (sb.verifyAuthentication(header, false));
final boolean isAdmin = (sb.verifyAuthentication(header));
final boolean isAuthUser = user!= null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT);
if(isAdmin || isAuthUser) {

@ -8,15 +8,14 @@ import net.yacy.cora.protocol.HeaderFramework;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.document.parser.html.CharacterCoding;
import net.yacy.search.Switchboard;
import de.anomic.data.BookmarkHelper;
import de.anomic.data.BookmarksDB;
import de.anomic.data.UserDB;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
public class get_bookmarks {
public class get_bookmarks {
private static final serverObjects prop = new serverObjects();
private static Switchboard sb = null;
private static UserDB.Entry user = null;
@ -29,7 +28,7 @@ public class get_bookmarks {
private final static int SORT_SIZE = 2;
private final static int SHOW_ALL = -1;
*/
private final static int MAXRESULTS = 10000;
// file types and display types
@ -42,14 +41,14 @@ public class get_bookmarks {
RSS(3), // .xml (.rss)
RDF(4); // .xml
private int value;
private final int value;
DisplayType(final int value) {
this.value = value;
}
int getValue() {
return value;
return this.value;
}
}
@ -58,7 +57,7 @@ public class get_bookmarks {
prop.clear();
sb = (Switchboard) env;
user = sb.userDB.getUser(header);
isAdmin = (sb.verifyAuthentication(header, true) || user != null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT));
isAdmin = (sb.verifyAuthentication(header) || user != null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT));
// set user name
final String username;
@ -66,7 +65,7 @@ public class get_bookmarks {
else if(isAdmin) username="admin";
else username = "unknown";
prop.putHTML("display_user", username);
// set peer address
prop.put("display_address", sb.peers.mySeed().getPublicAddress());
prop.put("display_peer", sb.peers.mySeed().getName());
@ -78,7 +77,7 @@ public class get_bookmarks {
// String sortname = "date";
final String qtype;
final String query;
// check for GET parameters
if (post != null){
itemsPerPage = (post.containsKey("rp")) ? post.getInt("rp", MAXRESULTS) : MAXRESULTS;
@ -108,13 +107,13 @@ public class get_bookmarks {
itemsPerPage = MAXRESULTS;
display = DisplayType.XML.getValue();
}
int count = 0;
int total = 0;
int start = 0;
final Iterator<String> it;
if (display == DisplayType.XBEL.getValue()) {
String root = "/";
if ("tags".equals(qtype) && !"".equals(query)) {
@ -200,14 +199,14 @@ public class get_bookmarks {
prop.put("display_page", page);
prop.put("display_total", total);
}
// return rewrite properties
return prop;
}
private static int recurseFolders(final Iterator<String> it, String root, int count, final boolean next, final String prev){
String fn="";
String fn="";
if (next) fn = it.next();
else fn = prev;
@ -220,7 +219,7 @@ public class get_bookmarks {
}
return count;
}
if (fn.startsWith(("/".equals(root) ? root : root + "/"))) {
prop.put("display_xbel_"+count+"_elements", "<folder id=\""+BookmarkHelper.tagHash(fn)+"\">");
count++;
@ -233,7 +232,7 @@ public class get_bookmarks {
if (it.hasNext()) {
count = recurseFolders(it, fn, count, true, fn);
}
} else {
} else {
if (count > 0) {
prop.put("display_xbel_"+count+"_elements", "</folder>");
count++;
@ -243,14 +242,14 @@ public class get_bookmarks {
root = "/";
}
count = recurseFolders(it, root, count, false, fn);
}
}
return count;
}
private static int print_XBEL(final Iterator<String> bit, int count) {
BookmarksDB.Bookmark bookmark = null;
Date date;
while(bit.hasNext()){
while(bit.hasNext()){
bookmark = sb.bookmarksDB.getBookmark(bit.next());
date = new Date(bookmark.getTimeStamp());
prop.put("display_xbel_"+count+"_elements", "<bookmark id=\"" + bookmark.getUrlHash()
@ -283,5 +282,5 @@ public class get_bookmarks {
count++;
}
return count;
}
}
}

@ -4,41 +4,40 @@ import java.util.Iterator;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.search.Switchboard;
import de.anomic.data.BookmarkHelper;
import de.anomic.data.BookmarksDB;
import de.anomic.data.UserDB;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
public class get_folders {
public class get_folders {
private static final serverObjects prop = new serverObjects();
private static Switchboard sb = null;
private static UserDB.Entry user = null;
private static boolean isAdmin = false;
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
prop.clear();
sb = (Switchboard) env;
user = sb.userDB.getUser(header);
isAdmin = (sb.verifyAuthentication(header, true) || user != null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT));
user = sb.userDB.getUser(header);
isAdmin = (sb.verifyAuthentication(header) || user != null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT));
// set user name
final String username;
if(user != null) username=user.getUserName();
else if(isAdmin) username="admin";
else username = "unknown";
prop.putHTML("display_user", username);
// set peer address
// set peer address
prop.put("display_address", sb.peers.mySeed().getPublicAddress());
prop.put("display_peer", sb.peers.mySeed().getName());
String root = "/";
String root = "/";
String[] foldername = null;
// check for GET parameters
if (post != null){
if (post.containsKey("root")) {
@ -47,16 +46,16 @@ public class get_folders {
else root = "/" + post.get("root");
}
}
Iterator<String> it = null;
// loop through folderList
it = BookmarkHelper.getFolderList(root, sb.bookmarksDB.getTagIterator(isAdmin));
it = BookmarkHelper.getFolderList(root, sb.bookmarksDB.getTagIterator(isAdmin));
int n = root.split("/").length;
if (n == 0) n = 1;
int count = 0;
while (it.hasNext()) {
String folder = it.next();
while (it.hasNext()) {
final String folder = it.next();
foldername = folder.split("/");
if (foldername.length == n+1) {
prop.put("folders_"+count+"_foldername", foldername[n]);
@ -69,14 +68,14 @@ public class get_folders {
count++;
}
}
// loop through bookmarkList
it = sb.bookmarksDB.getBookmarksIterator(root, isAdmin);
it = sb.bookmarksDB.getBookmarksIterator(root, isAdmin);
BookmarksDB.Bookmark bm;
while (it.hasNext()) {
bm = sb.bookmarksDB.getBookmark(it.next());
// TODO: get rid of bmtype
if (post.containsKey("bmtype")) {
if (post.containsKey("bmtype")) {
if (post.get("bmtype").equals("title")) {
prop.put("folders_"+count+"_foldername", bm.getTitle());
} else if (post.get("bmtype").equals("href")) {
@ -84,22 +83,22 @@ public class get_folders {
} else {
prop.put("folders_"+count+"_foldername", bm.getUrl());
}
}
}
prop.put("folders_"+count+"_expanded", "false");
prop.put("folders_"+count+"_url", bm.getUrl());
prop.put("folders_"+count+"_type", "file");
prop.put("folders_"+count+"_hash", bm.getUrlHash());
prop.put("folders_"+count+"_hasChildren", "false");
prop.put("folders_"+count+"_comma", ",");
count++;
}
count++;
}
count--;
prop.put("folders_"+count+"_comma", "");
count++;
prop.put("folders", count);
// return rewrite properties
return prop;
}
}
}

@ -6,7 +6,6 @@ import java.util.Set;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.peers.NewsPool;
import net.yacy.search.Switchboard;
import de.anomic.data.BookmarkHelper;
import de.anomic.data.BookmarksDB;
import de.anomic.data.ListManager;
@ -16,23 +15,23 @@ import de.anomic.server.serverSwitch;
public class add_p {
private static final serverObjects prop = new serverObjects();
private static Switchboard sb = null;
private static UserDB.Entry user = null;
private static boolean isAdmin = false;
private static boolean isAdmin = false;
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
sb = (Switchboard) env;
isAdmin=sb.verifyAuthentication(header, true);
sb = (Switchboard) env;
isAdmin=sb.verifyAuthentication(header);
user = sb.userDB.getUser(header);
// set user name
String username="";
String username="";
if(user != null) username=user.getUserName();
else if(isAdmin) username="admin";
if (post != null) {
if (!isAdmin) {
// force authentication if desired
@ -45,7 +44,7 @@ public class add_p {
final String title=post.get("title",url);
final String description=post.get("description","");
String tagsString = post.get("tags","");
String pathString = post.get("path","/unsorted");
final String pathString = post.get("path","/unsorted");
tagsString= tagsString + "," + pathString;
final Set<String> tags = ListManager.string2set(BookmarkHelper.cleanTagsString(tagsString));
final BookmarksDB.Bookmark bookmark = sb.bookmarksDB.createBookmark(url, username);

@ -6,7 +6,6 @@ import net.yacy.cora.date.ISO8601Formatter;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.kelondro.order.Digest;
import net.yacy.search.Switchboard;
import de.anomic.data.BookmarksDB;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
@ -15,19 +14,19 @@ public class all {
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
// return variable that accumulates replacements
final Switchboard switchboard = (Switchboard) env;
final boolean isAdmin=switchboard.verifyAuthentication(header, true);
final boolean isAdmin=switchboard.verifyAuthentication(header);
final serverObjects prop = new serverObjects();
Iterator<String> it;
if(post != null && post.containsKey("tag")){
it=switchboard.bookmarksDB.getBookmarksIterator(post.get("tag"), isAdmin);
}else{
it=switchboard.bookmarksDB.getBookmarksIterator(isAdmin);
}
// if an extended xml should be used
final boolean extendedXML = (post != null && post.containsKey("extendedXML"));
int count=0;
BookmarksDB.Bookmark bookmark;
Date date;
@ -40,7 +39,7 @@ public class all {
date = new Date(bookmark.getTimeStamp());
prop.putXML("posts_"+count+"_time", ISO8601Formatter.FORMATTER.format(date));
prop.putXML("posts_"+count+"_tags", bookmark.getTagsString().replaceAll(","," "));
// additional XML tags
prop.put("posts_"+count+"_isExtended",extendedXML ? "1" : "0");
if (extendedXML) {
@ -53,5 +52,5 @@ public class all {
// return rewrite properties
return prop;
}
}

@ -5,7 +5,6 @@ import net.yacy.cora.document.ASCII;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.kelondro.data.meta.DigestURI;
import net.yacy.search.Switchboard;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
@ -14,7 +13,7 @@ public class delete_p {
// return variable that accumulates replacements
final Switchboard switchboard = (Switchboard) env;
final serverObjects prop = new serverObjects();
final boolean isAdmin=switchboard.verifyAuthentication(header, true);
final boolean isAdmin=switchboard.verifyAuthentication(header);
if(post!= null){
if(!isAdmin){
// force authentication if desired
@ -22,7 +21,7 @@ public class delete_p {
prop.put("AUTHENTICATE","admin log-in");
}
return prop;
}
}
try {
if (post.containsKey("url") && switchboard.bookmarksDB.removeBookmark(ASCII.String((new DigestURI(post.get("url", "nourl"))).hash()))) {
prop.put("result", "1");
@ -36,10 +35,10 @@ public class delete_p {
}
}else{
prop.put("result", "0");
}
}
// return rewrite properties
return prop;
}
}
}

@ -16,12 +16,12 @@ public class get {
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
// return variable that accumulates replacements
final Switchboard switchboard = (Switchboard) env;
final boolean isAdmin=switchboard.verifyAuthentication(header, true);
final boolean isAdmin=switchboard.verifyAuthentication(header);
final serverObjects prop = new serverObjects();
String tag = null;
final String date;
//String url=""; //urlfilter not yet implemented
if (post != null && post.containsKey("tag")) {
tag = post.get("tag");
}
@ -30,19 +30,19 @@ public class get {
} else {
date = ISO8601Formatter.FORMATTER.format();
}
// if an extended xml should be used
final boolean extendedXML = (post != null && post.containsKey("extendedXML"));
int count=0;
Date parsedDate = null;
Date parsedDate = null;
try {
parsedDate = ISO8601Formatter.FORMATTER.parse(date);
} catch (final ParseException e) {
parsedDate = new Date();
}
final List<String> bookmark_hashes = switchboard.bookmarksDB.getDate(Long.toString(parsedDate.getTime())).getBookmarkList();
BookmarksDB.Bookmark bookmark = null;
for (final String bookmark_hash : bookmark_hashes){
@ -56,7 +56,7 @@ public class get {
prop.put("posts_"+count+"_md5", Digest.encodeMD5Hex(bookmark.getUrl()));
prop.put("posts_"+count+"_time", date);
prop.putHTML("posts_"+count+"_tags", bookmark.getTagsString().replaceAll(","," "));
// additional XML tags
prop.put("posts_"+count+"_isExtended",extendedXML ? "1" : "0");
if (extendedXML) {
@ -70,5 +70,5 @@ public class get {
// return rewrite properties
return prop;
}
}

@ -1,9 +1,9 @@
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.search.Switchboard;
import de.anomic.data.BookmarksDB.Bookmark;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.data.BookmarksDB.Bookmark;
public class addTag_p {
@ -12,22 +12,22 @@ public class addTag_p {
final Switchboard switchboard = (Switchboard) env;
final serverObjects prop = new serverObjects();
boolean isAdmin = false;
isAdmin = switchboard.verifyAuthentication(header, true);
isAdmin = switchboard.verifyAuthentication(header);
prop.put("result", "0");//error
//rename tags
if(post != null && isAdmin) {
if (post.containsKey("selectTag") && post.containsKey("addTag")) {
switchboard.bookmarksDB.addTag(post.get("selectTag"), post.get("addTag"));
prop.put("result", "1");//success
prop.put("result", "1");//success
} else if (post.containsKey("urlhash") && post.containsKey("addTag")) {
final Bookmark bm = switchboard.bookmarksDB.getBookmark(post.get("urlhash"));
bm.addTag(post.get("addTag"));
prop.put("result", "1");//success
prop.put("result", "1");//success
}
}
}
// return rewrite properties
return prop;
}
}

@ -6,12 +6,12 @@ import de.anomic.server.serverSwitch;
public class editTag_p {
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
final Switchboard switchboard = (Switchboard) env;
final Switchboard switchboard = (Switchboard) env;
final serverObjects prop = new serverObjects();
boolean isAdmin = false;
isAdmin = switchboard.verifyAuthentication(header, true);
isAdmin = switchboard.verifyAuthentication(header);
prop.put("result", "0");//error
//rename tags
if(post != null && isAdmin && post.containsKey("old") && post.containsKey("new")){
@ -21,5 +21,5 @@ public class editTag_p {
// return rewrite properties
return prop;
}
}

@ -3,7 +3,6 @@ import java.util.Iterator;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.search.Switchboard;
import de.anomic.data.BookmarksDB;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
@ -12,18 +11,18 @@ public class getTag {
final static int SORT_ALPHA = 1;
final static int SORT_SIZE = 2;
final static int SHOW_ALL = -1;
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
// return variable that accumulates replacements
final Switchboard switchboard = (Switchboard) env;
final boolean isAdmin = switchboard.verifyAuthentication(header, true);
final boolean isAdmin = switchboard.verifyAuthentication(header);
final serverObjects prop = new serverObjects();
Iterator<BookmarksDB.Tag> it = null;
String tagName = "";
int top = SHOW_ALL;
int comp = SORT_ALPHA;
if (post != null) {
if (!isAdmin) {
// force authentication if desired
@ -43,16 +42,16 @@ public class getTag {
}
if (post != null && post.containsKey("tag")) {
tagName=post.get("tag");
tagName=post.get("tag");
if (!tagName.isEmpty()) {
it = switchboard.bookmarksDB.getTagIterator(tagName, isAdmin, comp, top);
}
it = switchboard.bookmarksDB.getTagIterator(tagName, isAdmin, comp, top);
}
} else {
it = switchboard.bookmarksDB.getTagIterator(isAdmin, comp, top);
}
}
// Iterator<bookmarksDB.Tag> it = switchboard.bookmarksDB.getTagIterator(isAdmin);
int count = 0;
if (it != null) {
BookmarksDB.Tag tag;
@ -70,5 +69,5 @@ public class getTag {
// return rewrite properties
return prop;
}
}

@ -18,16 +18,16 @@ public class xbel {
private static Switchboard switchboard = null;
private static boolean isAdmin = false;
private static int R = 1; // TODO: solve the recursion problem an remove global variable
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
int count = 0;;
String root = "/";
int style = 0;
prop.clear();
switchboard = (Switchboard) env;
isAdmin=switchboard.verifyAuthentication(header, true);
switchboard = (Switchboard) env;
isAdmin=switchboard.verifyAuthentication(header);
if(post != null) {
if(!isAdmin) {
@ -36,7 +36,7 @@ public class xbel {
}
}
if(post.containsKey("tag")) {
final String tagName=post.get("tag");
final String tagName=post.get("tag");
prop.putHTML("folder", tagName);
if (!tagName.equals("")) {
final Iterator<String> bit=switchboard.bookmarksDB.getBookmarksIterator(tagName, isAdmin);
@ -47,98 +47,98 @@ public class xbel {
}
if(post.containsKey("folder")) {
final String folderName=post.get("folder");
if (folderName.length() > 0 && folderName.charAt(0) == '/') { root = folderName; }
if (folderName.length() > 0 && folderName.charAt(0) == '/') { root = folderName; }
else { root = "/" + folderName; }
}
if(post.containsKey("style") && !post.get("style").equals("")) {
style = 1;
prop.putHTML("style_href", post.get("style"));
prop.putHTML("style_type", post.get("style").replaceAll("^.*\\.", ""));
}
style = 1;
prop.putHTML("style_href", post.get("style"));
prop.putHTML("style_type", post.get("style").replaceAll("^.*\\.", ""));
}
}
prop.put("style", style);
R = root.replaceAll("[^/]","").length() - 1;
count = recurseFolders(BookmarkHelper.getFolderList(root, switchboard.bookmarksDB.getTagIterator(isAdmin)),root,0,true,"");
prop.put("xbel", count);
return prop; // return from serverObjects respond()
return prop; // return from serverObjects respond()
}
private static int recurseFolders(final Iterator<String> it, String root, int count, final boolean next, final String prev){
String fn="";
if(next) fn = it.next();
String fn="";
if(next) fn = it.next();
else fn = prev;
if(fn.equals("\uffff")) {
int i = prev.replaceAll("[^/]","").length() - R;
int i = prev.replaceAll("[^/]","").length() - R;
while(i>0){
prop.put("xbel_"+count+"_elements", "</folder>");
count++;
i--;
}
}
return count;
}
if(fn.startsWith((root.equals("/") ? root : root+"/"))){
prop.put("xbel_"+count+"_elements", "<folder id=\""+BookmarkHelper.tagHash(fn)+"\">");
count++;
final String title = fn; // just to make sure fn stays untouched
prop.put("xbel_"+count+"_elements", "<title>" + CharacterCoding.unicode2xml(title.replaceAll("(/.[^/]*)*/", ""), true) + "</title>");
count++;
final String title = fn; // just to make sure fn stays untouched
prop.put("xbel_"+count+"_elements", "<title>" + CharacterCoding.unicode2xml(title.replaceAll("(/.[^/]*)*/", ""), true) + "</title>");
count++;
final Iterator<String> bit=switchboard.bookmarksDB.getBookmarksIterator(fn, isAdmin);
count = print_XBEL(bit, count);
if(it.hasNext()){
count = recurseFolders(it, fn, count, true, fn);
}
} else {
} else {
if (count > 0) {
prop.put("xbel_"+count+"_elements", "</folder>");
count++;
}
root = root.replaceAll("(/.[^/]*$)", "");
if(root.equals("")) root = "/";
root = root.replaceAll("(/.[^/]*$)", "");
if(root.equals("")) root = "/";
count = recurseFolders(it, root, count, false, fn);
}
}
return count;
}
private static int print_XBEL(final Iterator<String> bit, int count) {
BookmarksDB.Bookmark bookmark;
Date date;
while(bit.hasNext()){
while(bit.hasNext()){
bookmark=switchboard.bookmarksDB.getBookmark(bit.next());
date=new Date(bookmark.getTimeStamp());
prop.put("xbel_"+count+"_elements", "<bookmark id=\"" + bookmark.getUrlHash()
+ "\" href=\"" + CharacterCoding.unicode2xml(bookmark.getUrl(), true)
+ "\" added=\"" + CharacterCoding.unicode2xml(ISO8601Formatter.FORMATTER.format(date), true)+"\">");
count++;
count++;
prop.put("xbel_"+count+"_elements", "<title>");
count++;
prop.putXML("xbel_"+count+"_elements", bookmark.getTitle());
count++;
prop.putXML("xbel_"+count+"_elements", bookmark.getTitle());
count++;
prop.put("xbel_"+count+"_elements", "</title>");
count++;
prop.put("xbel_"+count+"_elements", "<info>");
prop.put("xbel_"+count+"_elements", "<info>");
count++;
prop.put("xbel_"+count+"_elements", "<metadata owner=\"Mozilla\" ShortcutURL=\""
+ CharacterCoding.unicode2xml(bookmark.getTagsString().replaceAll("/.*,", "").toLowerCase(), true)
+ "\"/>");
+ "\"/>");
count++;
prop.put("xbel_"+count+"_elements", "<metadata owner=\"YaCy\" public=\""+Boolean.toString(bookmark.getPublic())+"\"/>");
prop.put("xbel_"+count+"_elements", "<metadata owner=\"YaCy\" public=\""+Boolean.toString(bookmark.getPublic())+"\"/>");
count++;
prop.put("xbel_"+count+"_elements", "</info>");
prop.put("xbel_"+count+"_elements", "</info>");
count++;
prop.put("xbel_"+count+"_elements", "<desc>");
count++;
prop.putXML("xbel_"+count+"_elements", bookmark.getDescription());
count++;
prop.putXML("xbel_"+count+"_elements", bookmark.getDescription());
count++;
prop.put("xbel_"+count+"_elements", "</desc>");
count++;
prop.put("xbel_"+count+"_elements", "</bookmark>");
count++;
prop.put("xbel_"+count+"_elements", "</bookmark>");
count++;
}
return count;
}
}
}

@ -23,7 +23,7 @@ public class feed {
prop.put("item", "0");
if ((post == null) || (env == null)) return prop;
final boolean authorized = sb.verifyAuthentication(header, false);
final boolean authorized = sb.verifyAuthentication(header);
final String channelNames = post.get("set");
if (channelNames == null) return prop;

@ -11,46 +11,45 @@ import net.yacy.search.Switchboard;
import net.yacy.search.SwitchboardConstants;
import net.yacy.search.index.Segment;
import net.yacy.search.index.Segments;
import de.anomic.crawler.NoticedURL;
import de.anomic.crawler.retrieval.Request;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
public class queues_p {
public static final String STATE_RUNNING = "running";
public static final String STATE_PAUSED = "paused";
private static SimpleDateFormat dayFormatter = new SimpleDateFormat("yyyy/MM/dd", Locale.US);
private static String daydate(final Date date) {
if (date == null) return "";
return dayFormatter.format(date);
}
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
// return variable that accumulates replacements
final Switchboard sb = (Switchboard) env;
//wikiCode wikiTransformer = new wikiCode(switchboard);
final serverObjects prop = new serverObjects();
Segment segment = null;
boolean html = post != null && post.containsKey("html");
final boolean html = post != null && post.containsKey("html");
prop.setLocalized(html);
if (post != null && post.containsKey("segment") && sb.verifyAuthentication(header, false)) {
if (post != null && post.containsKey("segment") && sb.verifyAuthentication(header)) {
segment = sb.indexSegments.segment(post.get("segment"));
}
if (segment == null) segment = sb.indexSegments.segment(Segments.Process.PUBLIC);
prop.put("rejected", "0");
//int showRejectedCount = 10;
Seed initiator;
// index size
prop.putNum("urlpublictextSize", segment.urlMetadata().size());
prop.putNum("rwipublictextSize", segment.termIndex().sizesMax());
// loader queue
prop.putNum("loaderSize", sb.crawlQueues.workerSize());
prop.putNum("loaderSize", sb.crawlQueues.workerSize());
prop.putNum("loaderMax", sb.getConfigLong(SwitchboardConstants.CRAWLER_THREADS_ACTIVE_MAX, 10));
if (sb.crawlQueues.workerSize() == 0) {
prop.put("list-loader", "0");
@ -68,7 +67,7 @@ public class queues_p {
}
prop.put("list-loader", count);
}
//local crawl queue
prop.putNum("localCrawlSize", sb.getThread(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL).getJobCount());
prop.put("localCrawlState", sb.crawlJobIsPaused(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL) ? STATE_PAUSED : STATE_RUNNING);
@ -94,8 +93,8 @@ public class queues_p {
// return rewrite properties
return prop;
}
public static final void addNTable(final Switchboard sb, final serverObjects prop, final String tableName, final List<Request> crawlerList) {
int showNum = 0;

@ -2,8 +2,6 @@
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.kelondro.io.ByteCount;
//import net.yacy.kelondro.io.ByteCountInputStream;
//import net.yacy.kelondro.io.ByteCountOutputStream;
import net.yacy.kelondro.util.MemoryControl;
import net.yacy.kelondro.workflow.WorkflowProcessor;
import net.yacy.search.Switchboard;
@ -14,20 +12,20 @@ import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
public class status_p {
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
// return variable that accumulates replacements
final Switchboard sb = (Switchboard) env;
final serverObjects prop = new serverObjects();
Segment segment = null;
boolean html = post != null && post.containsKey("html");
final boolean html = post != null && post.containsKey("html");
prop.setLocalized(html);
if (post != null && post.containsKey("segment") && sb.verifyAuthentication(header, false)) {
if (post != null && post.containsKey("segment") && sb.verifyAuthentication(header)) {
segment = sb.indexSegments.segment(post.get("segment"));
}
if (segment == null) segment = sb.indexSegments.segment(Segments.Process.PUBLIC);
prop.put("rejected", "0");
sb.updateMySeed();
final int cacheMaxSize = (int) sb.getConfigLong(SwitchboardConstants.WORDCACHE_MAX_COUNT, 10000);
@ -35,14 +33,14 @@ public class status_p {
prop.putNum("qpm", sb.peers.mySeed().getQPM());
prop.putNum("wordCacheSize", segment.termIndex().getBufferSize());
prop.putNum("wordCacheMaxSize", cacheMaxSize);
// crawl queues
prop.putNum("localCrawlSize", sb.getThread(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL).getJobCount());
prop.putNum("limitCrawlSize", sb.crawlQueues.limitCrawlJobSize());
prop.putNum("remoteCrawlSize", sb.getThread(SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL).getJobCount());
prop.putNum("loaderSize", sb.crawlQueues.workerSize());
prop.putNum("loaderSize", sb.crawlQueues.workerSize());
prop.putNum("loaderMax", sb.getConfigLong(SwitchboardConstants.CRAWLER_THREADS_ACTIVE_MAX, 10));
// memory usage and system attributes
prop.putNum("freeMemory", MemoryControl.free());
prop.putNum("totalMemory", MemoryControl.total());
@ -57,5 +55,5 @@ public class status_p {
// return rewrite properties
return prop;
}
}

@ -46,7 +46,7 @@ public class termlist_p {
Segment segment = null;
final boolean delete = post != null && post.containsKey("delete");
final long mincount = post == null ? 10000 : post.getLong("mincount", 10000);
if (post != null && post.containsKey("segment") && sb.verifyAuthentication(header, false)) {
if (post != null && post.containsKey("segment") && sb.verifyAuthentication(header)) {
segment = sb.indexSegments.segment(post.get("segment"));
}
if (segment == null) segment = sb.indexSegments.segment(Segments.Process.PUBLIC);

@ -10,7 +10,7 @@
// $LastChangedBy: orbiter $
//
// LICENSE
//
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
@ -38,27 +38,26 @@ import net.yacy.kelondro.logging.Log;
import net.yacy.search.Switchboard;
import net.yacy.search.index.Segment;
import net.yacy.search.index.Segments;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
public class yacydoc {
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
// return variable that accumulates replacements
final Switchboard sb = (Switchboard) env;
final serverObjects prop = new serverObjects();
final Segment segment;
boolean html = post != null && post.containsKey("html");
final boolean html = post != null && post.containsKey("html");
prop.setLocalized(html);
boolean authorized = sb.verifyAuthentication(header, false);
final boolean authorized = sb.verifyAuthentication(header);
if (post != null && post.containsKey("segment") && authorized) {
segment = sb.indexSegments.segment(post.get("segment"));
} else {
segment = sb.indexSegments.segment(Segments.Process.PUBLIC);
}
prop.put("dc_title", "");
prop.put("dc_creator", "");
prop.put("dc_description", "");
@ -71,21 +70,21 @@ public class yacydoc {
prop.put("dc_language", "");
if (post == null) return prop;
String urlstring = post.get("url", "").trim();
final String urlstring = post.get("url", "").trim();
String urlhash = post.get("urlhash", "").trim();
if (urlstring.length() == 0 && urlhash.length() == 0) return prop;
if (urlstring.length() > 0 && urlhash.length() == 0) {
try {
DigestURI url = new DigestURI(urlstring);
final DigestURI url = new DigestURI(urlstring);
urlhash = ASCII.String(url.hash());
} catch (MalformedURLException e) {
} catch (final MalformedURLException e) {
Log.logException(e);
}
}
if (urlhash == null || urlhash.length() == 0) return prop;
final URIMetadataRow entry = segment.urlMetadata().load(urlhash.getBytes());
if (entry == null) return prop;
@ -94,7 +93,7 @@ public class yacydoc {
return prop;
}
final URIMetadataRow le = (entry.referrerHash() == null || entry.referrerHash().length != Word.commonHashLength) ? null : segment.urlMetadata().load(entry.referrerHash());
prop.putXML("dc_title", metadata.dc_title());
prop.putXML("dc_creator", metadata.dc_creator());
prop.putXML("dc_description", ""); // this is the fulltext part in the surrogate
@ -114,7 +113,7 @@ public class yacydoc {
prop.putXML("yacy_referrer_url", (le == null) ? "" : le.metadata().url().toNormalform(false, true));
prop.put("yacy_size", entry.size());
prop.put("yacy_words",entry.wordCount());
// return rewrite properties
return prop;
}

@ -1,4 +1,5 @@
import java.io.IOException;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.document.Parser.Failure;
import net.yacy.kelondro.data.meta.DigestURI;
@ -15,15 +16,15 @@ import de.anomic.server.serverSwitch;
public class add_ymark {
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
final Switchboard sb = (Switchboard) env;
final serverObjects prop = new serverObjects();
final UserDB.Entry user = sb.userDB.getUser(header);
final boolean isAdmin = (sb.verifyAuthentication(header, true));
final boolean isAdmin = (sb.verifyAuthentication(header));
final boolean isAuthUser = user!= null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT);
if(isAdmin || isAuthUser) {
final String bmk_user = (isAuthUser ? user.getUserName() : YMarkTables.USER_ADMIN);
@ -31,30 +32,30 @@ public class add_ymark {
prop.put("redirect_url", post.get("redirect"));
prop.put("redirect", "1");
}
if(post.containsKey("urlHash")) {
final String urlHash = post.get("urlHash",YMarkUtil.EMPTY_STRING);
final DigestURI url = sb.indexSegments.segment(Segments.Process.PUBLIC).urlMetadata().load(urlHash.getBytes()).metadata().url();
final DigestURI url = sb.indexSegments.segment(Segments.Process.PUBLIC).urlMetadata().load(urlHash.getBytes()).metadata().url();
final String folders = post.get(YMarkEntry.BOOKMARK.FOLDERS.key(),YMarkEntry.FOLDERS_UNSORTED);
final String tags = post.get(YMarkEntry.BOOKMARK.TAGS.key(),YMarkUtil.EMPTY_STRING);
try {
sb.tables.bookmarks.createBookmark(sb.loader, url, bmk_user, true, tags, folders);
prop.put("status", "1");
} catch (IOException e) {
} catch (final IOException e) {
// TODO Auto-generated catch block
Log.logException(e);
} catch (Failure e) {
} catch (final Failure e) {
// TODO Auto-generated catch block
Log.logException(e);
} catch (RowSpaceExceededException e) {
} catch (final RowSpaceExceededException e) {
// TODO Auto-generated catch block
Log.logException(e);
}
} else if(post.containsKey(YMarkEntry.BOOKMARK.URL.key())) {
String url = post.get(YMarkEntry.BOOKMARK.URL.key(),YMarkEntry.BOOKMARK.URL.deflt());
boolean hasProtocol = false;
for (YMarkTables.PROTOCOLS p : YMarkTables.PROTOCOLS.values()) {
for (final YMarkTables.PROTOCOLS p : YMarkTables.PROTOCOLS.values()) {
if(url.toLowerCase().startsWith(p.protocol())) {
hasProtocol = true;
break;
@ -63,22 +64,22 @@ public class add_ymark {
if (!hasProtocol) {
url=YMarkTables.PROTOCOLS.HTTP.protocol(url);
}
final YMarkEntry bmk = new YMarkEntry();
final YMarkEntry bmk = new YMarkEntry();
bmk.put(YMarkEntry.BOOKMARK.URL.key(), url);
bmk.put(YMarkEntry.BOOKMARK.TITLE.key(), post.get(YMarkEntry.BOOKMARK.TITLE.key(),YMarkEntry.BOOKMARK.TITLE.deflt()));
bmk.put(YMarkEntry.BOOKMARK.DESC.key(), post.get(YMarkEntry.BOOKMARK.DESC.key(),YMarkEntry.BOOKMARK.DESC.deflt()));
bmk.put(YMarkEntry.BOOKMARK.PUBLIC.key(), post.get(YMarkEntry.BOOKMARK.PUBLIC.key(),YMarkEntry.BOOKMARK.PUBLIC.deflt()));
bmk.put(YMarkEntry.BOOKMARK.TAGS.key(), YMarkUtil.cleanTagsString(post.get(YMarkEntry.BOOKMARK.TAGS.key(),YMarkEntry.BOOKMARK.TAGS.deflt())));
bmk.put(YMarkEntry.BOOKMARK.FOLDERS.key(), YMarkUtil.cleanFoldersString(post.get(YMarkEntry.BOOKMARK.FOLDERS.key(),YMarkEntry.FOLDERS_UNSORTED)));
try {
sb.tables.bookmarks.addBookmark(bmk_user, bmk, false, false);
} catch (IOException e) {
} catch (final IOException e) {
Log.logException(e);
} catch (RowSpaceExceededException e) {
}
} catch (final RowSpaceExceededException e) {
}
prop.put("status", "1");
} else {
prop.put("status", "0");

@ -13,18 +13,18 @@ import de.anomic.server.serverSwitch;
public class delete_ymark {
private static Switchboard sb = null;
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
sb = (Switchboard) env;
final serverObjects prop = new serverObjects();
final UserDB.Entry user = sb.userDB.getUser(header);
final boolean isAdmin = (sb.verifyAuthentication(header, true));
final boolean isAdmin = (sb.verifyAuthentication(header));
final boolean isAuthUser = user!= null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT);
if(isAdmin || isAuthUser) {
if(isAdmin || isAuthUser) {
final String bmk_user = (isAuthUser ? user.getUserName() : YMarkTables.USER_ADMIN);
byte[] urlHash = null;
try {
@ -38,14 +38,14 @@ public class delete_ymark {
}
sb.tables.bookmarks.deleteBookmark(bmk_user, urlHash);
prop.put("result", "1");
} catch (IOException e) {
} catch (final IOException e) {
Log.logException(e);
} catch (RowSpaceExceededException e) {
} catch (final RowSpaceExceededException e) {
Log.logException(e);
}
} else {
prop.put(YMarkTables.USER_AUTHENTICATE,YMarkTables.USER_AUTHENTICATE_MSG);
}
}
// return rewrite properties
return prop;
}

@ -18,25 +18,25 @@ import de.anomic.data.ymark.YMarkUtil;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
public class get_metadata {
public class get_metadata {
static serverObjects prop;
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
final Switchboard sb = (Switchboard) env;
prop = new serverObjects();
prop = new serverObjects();
final UserDB.Entry user = sb.userDB.getUser(header);
final boolean isAdmin = (sb.verifyAuthentication(header, true));
final boolean isAuthUser = user!= null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT);
final boolean isAdmin = (sb.verifyAuthentication(header));
final boolean isAuthUser = user!= null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT);
if(isAdmin || isAuthUser) {
final String bmk_user = (isAuthUser ? user.getUserName() : YMarkTables.USER_ADMIN);
final String bmk_user = (isAuthUser ? user.getUserName() : YMarkTables.USER_ADMIN);
String url = post.get(YMarkEntry.BOOKMARK.URL.key(),YMarkEntry.BOOKMARK.URL.deflt());
boolean hasProtocol = false;
for (YMarkTables.PROTOCOLS p : YMarkTables.PROTOCOLS.values()) {
for (final YMarkTables.PROTOCOLS p : YMarkTables.PROTOCOLS.values()) {
if(url.toLowerCase().startsWith(p.protocol())) {
hasProtocol = true;
break;
@ -45,17 +45,17 @@ public class get_metadata {
if (!hasProtocol) {
url=YMarkTables.PROTOCOLS.HTTP.protocol(url);
}
try {
YMarkMetadata meta = new YMarkMetadata(new DigestURI(url), sb.indexSegments);
try {
final YMarkMetadata meta = new YMarkMetadata(new DigestURI(url), sb.indexSegments);
final Document document = meta.loadDocument(sb.loader);
final EnumMap<YMarkMetadata.METADATA, String> metadata = meta.loadMetadata();
prop.putXML("title", metadata.get(YMarkMetadata.METADATA.TITLE));
prop.putXML("desc", metadata.get(YMarkMetadata.METADATA.DESCRIPTION));
prop.putXML("desc", metadata.get(YMarkMetadata.METADATA.DESCRIPTION));
prop.put("keywords", putTags(document.dc_subject(','), "keywords"));
prop.put("autotags", putTags(YMarkAutoTagger.autoTag(document, 5, sb.tables.bookmarks.getTags(bmk_user)), "autotags"));
final YMarkCrawlStart crawlStart = new YMarkCrawlStart(sb.tables, url);
int count = 0;
if(!crawlStart.isEmpty()) {
@ -70,23 +70,23 @@ public class get_metadata {
}
prop.put("crawlstart", count);
} catch (MalformedURLException e1) {
} catch (final MalformedURLException e1) {
// TODO Auto-generated catch block
e1.printStackTrace();
} catch (IOException e) {
} catch (final IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (Failure e) {
} catch (final Failure e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
} else {
prop.put(YMarkTables.USER_AUTHENTICATE,YMarkTables.USER_AUTHENTICATE_MSG);
}
}
// return rewrite properties
return prop;
}
public static int putTags(final String tagString, final String var) {
final String list[] = tagString.split(YMarkUtil.TAGS_SEPARATOR);
int count = 0;

@ -17,25 +17,25 @@ import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
public class get_tags {
final static String TAG = "tag";
final static String TOP = "top";
final static String SORT = "sort";
final static String SIZE = "size";
final static String ALPHA = "alpha";
private static Switchboard sb = null;
private static serverObjects prop = null;
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
sb = (Switchboard) env;
prop = new serverObjects();
final UserDB.Entry user = sb.userDB.getUser(header);
final boolean isAdmin = (sb.verifyAuthentication(header, true));
final boolean isAdmin = (sb.verifyAuthentication(header));
final boolean isAuthUser = user!= null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT);
if(isAdmin || isAuthUser) {
final String bmk_user = (isAuthUser ? user.getUserName() : YMarkTables.USER_ADMIN);
Integer top = Integer.MAX_VALUE;
@ -49,13 +49,13 @@ public class get_tags {
final String[] tagArray = YMarkUtil.cleanTagsString(post.get(TAG)).split(YMarkUtil.TAGS_SEPARATOR);
try {
tags = new TreeSet<YMarkTag>(sb.tables.bookmarks.getTags(sb.tables.bookmarks.getBookmarksByTag(bmk_user, tagArray)).values());
} catch (IOException e) {
} catch (final IOException e) {
return prop;
}
} else {
try {
tags = new TreeSet<YMarkTag>(sb.tables.bookmarks.getTags(bmk_user).values());
} catch (IOException e) {
} catch (final IOException e) {
return prop;
}
}
@ -63,17 +63,17 @@ public class get_tags {
if (post != null && post.containsKey(TOP)) {
top = post.getInt(TOP, Integer.MAX_VALUE);
}
if (post != null && post.containsKey(SORT)) {
if (SIZE.equals(post.get(SORT))) {
sortAlpha = false;
}
}
if(sortAlpha) {
final TreeMap<CollationKey, YMarkTag> sort = new TreeMap<CollationKey, YMarkTag>();
final Collator collator = Collator.getInstance();
collator.setStrength(Collator.SECONDARY);
final Collator collator = Collator.getInstance();
collator.setStrength(Collator.SECONDARY);
tit = tags.iterator();
while(tit.hasNext() && count < top) {
t = tit.next();

@ -26,25 +26,25 @@ import de.anomic.data.ymark.YMarkUtil;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
public class get_treeview {
public class get_treeview {
public static final String ROOT = "root";
public static final String SOURCE = "source";
static serverObjects prop;
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
final Switchboard sb = (Switchboard) env;
prop = new serverObjects();
prop = new serverObjects();
final UserDB.Entry user = sb.userDB.getUser(header);
final boolean isAdmin = (sb.verifyAuthentication(header, true));
final boolean isAdmin = (sb.verifyAuthentication(header));
final boolean isAuthUser = user!= null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT);
if(isAdmin || isAuthUser) {
final String bmk_user = (isAuthUser ? user.getUserName() : YMarkTables.USER_ADMIN);
String root = YMarkTables.FOLDERS_ROOT;
String root = YMarkTables.FOLDERS_ROOT;
String[] foldername = null;
boolean isFolder = true;
boolean isBookmark = false;
@ -58,7 +58,7 @@ public class get_treeview {
if(post.containsKey("display") && post.get("display").equals("bmk")) {
displayBmk = true;
}
if (post.containsKey(ROOT)) {
if (post.get(ROOT).equals(SOURCE) || post.get(ROOT).equals(YMarkTables.FOLDERS_ROOT)) {
root = "";
@ -82,24 +82,24 @@ public class get_treeview {
}
}
}
Iterator<String> it = null;
Iterator<Tables.Row> bit = null;
Tables.Row bmk_row = null;
int count = 0;
if(isFolder) {
// loop through folderList
try {
// loop through folderList
try {
// it = sb.tables.bookmarks.folders.getFolders(bmk_user, root);
it = sb.tables.bookmarks.getFolders(bmk_user, root).iterator();
} catch (IOException e) {
} catch (final IOException e) {
Log.logException(e);
}
int n = root.split(YMarkUtil.FOLDERS_SEPARATOR).length;
if (n == 0) n = 1;
while (it.hasNext()) {
String folder = it.next();
while (it.hasNext()) {
final String folder = it.next();
foldername = folder.split(YMarkUtil.FOLDERS_SEPARATOR);
if (foldername.length == n+1) {
prop.put("folders_"+count+"_foldername", foldername[n]);
@ -118,17 +118,17 @@ public class get_treeview {
bit = sb.tables.bookmarks.getBookmarksByFolder(bmk_user, root);
while (bit.hasNext()) {
bmk_row = bit.next();
if(bmk_row != null) {
if(bmk_row != null) {
final String url = UTF8.String(bmk_row.get(YMarkEntry.BOOKMARK.URL.key()));
final String title = bmk_row.get(YMarkEntry.BOOKMARK.TITLE.key(), YMarkEntry.BOOKMARK.TITLE.deflt());
// TODO: get_treeview - get rid of bmtype
if (post.containsKey("bmtype")) {
if (post.containsKey("bmtype")) {
if (post.get("bmtype").equals("title")) {
prop.putJSON("folders_"+count+"_foldername", title);
} else if (post.get("bmtype").equals("href")) {
prop.putJSON("folders_"+count+"_foldername", "<a href='"+url+"' target='_blank'>"+title+"</a>");
}
}
} else {
prop.putJSON("folders_"+count+"_foldername", url);
}
@ -138,15 +138,15 @@ public class get_treeview {
prop.put("folders_"+count+"_hash", "b:"+new String(bmk_row.getPK()));
prop.put("folders_"+count+"_hasChildren", "true");
prop.put("folders_"+count+"_comma", ",");
count++;
count++;
}
}
}
}
count--;
prop.put("folders_"+count+"_comma", "");
count++;
prop.put("folders", count);
} catch (IOException e) {
} catch (final IOException e) {
Log.logException(e);
}
} else if(displayBmk && isBookmark) {
@ -170,12 +170,12 @@ public class get_treeview {
final String value = UTF8.String(bmk_row.get(key));
if (key.equals("url"))
url = value;
prop.put("folders_"+count+"_foldername","<small><b>"+key+":</b> " + value + "</small>");
prop.put("folders_"+count+"_foldername","<small><b>"+key+":</b> " + value + "</small>");
if(YMarkEntry.BOOKMARK.contains(key))
putProp(count, YMarkEntry.BOOKMARK.get(key).type());
else
putProp(count, "meta");
count++;
count++;
}
}
prop.put("folders_"+count+"_foldername","<small><b>MetaData</b></small>");
@ -198,12 +198,12 @@ public class get_treeview {
prop.put("folders_"+count+"_hash", "w:"+url);
prop.put("folders_"+count+"_hasChildren", "true");
prop.put("folders_"+count+"_comma", "");
count++;
count++;
prop.put("folders", count);
}
} catch (IOException e) {
} catch (final IOException e) {
Log.logException(e);
} catch (RowSpaceExceededException e) {
} catch (final RowSpaceExceededException e) {
Log.logException(e);
}
} else if (isAutoTagger || isMetadata || isURLdb || isCrawlStart) {
@ -220,7 +220,7 @@ public class get_treeview {
count++;
prop.put("folders_"+count+"_foldername","<small><b>without preference: </b>"+YMarkAutoTagger.autoTag(document, 4, new TreeMap<String, YMarkTag>())+"</small>");
putProp(count, "meta");
count++;
count++;
prop.put("folders", count);
} else if(isMetadata) {
count = putMeta(count, meta.loadMetadata());
@ -240,17 +240,17 @@ public class get_treeview {
prop.put("folders", count);
}
} catch (MalformedURLException e) {
} catch (final MalformedURLException e) {
Log.logException(e);
} catch (IOException e) {
} catch (final IOException e) {
Log.logException(e);
} catch (Failure e) {
} catch (final Failure e) {
Log.logException(e);
}
}
}
} else {
prop.put(YMarkTables.USER_AUTHENTICATE,YMarkTables.USER_AUTHENTICATE_MSG);
}
}
// return rewrite properties
return prop;
}

@ -23,20 +23,20 @@ public class get_xbel {
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
final Switchboard sb = (Switchboard) env;
final serverObjects prop = new serverObjects();
final serverObjects prop = new serverObjects();
final HashSet<String> alias = new HashSet<String>();
final StringBuilder buffer = new StringBuilder(250);
final UserDB.Entry user = sb.userDB.getUser(header);
final boolean isAdmin = (sb.verifyAuthentication(header, true));
final boolean isAdmin = (sb.verifyAuthentication(header));
final boolean isAuthUser = user!= null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT);
final String bmk_user;
if(isAdmin || isAuthUser) {
bmk_user = (isAuthUser ? user.getUserName() : YMarkTables.USER_ADMIN);
String root = YMarkTables.FOLDERS_ROOT;
String root = YMarkTables.FOLDERS_ROOT;
String[] foldername = null;
// TODO: better handling of query
if (post != null){
if (post.containsKey(ROOT)) {
@ -45,27 +45,27 @@ public class get_xbel {
} else if (post.get(ROOT).startsWith(YMarkTables.FOLDERS_ROOT)) {
root = post.get(ROOT);
} else {
root = "";
root = "";
}
}
} else {
root = "";
}
final int root_depth = root.split(YMarkUtil.FOLDERS_SEPARATOR).length - 1;
// Log.logInfo(YMarkTables.BOOKMARKS_LOG, "root: "+root+" root_depth: "+root_depth);
Iterator<String> fit = null;
Iterator<Tables.Row> bit = null;
int count = 0;
int count = 0;
int n = root_depth;
try {
fit = sb.tables.bookmarks.getFolders(bmk_user, root).iterator();
} catch (IOException e) {
} catch (final IOException e) {
Log.logException(e);
}
while (fit.hasNext()) {
while (fit.hasNext()) {
final String folder = fit.next();
foldername = folder.split(YMarkUtil.FOLDERS_SEPARATOR);
final int len = foldername.length -1;
@ -76,73 +76,73 @@ public class get_xbel {
count++;
}
}
if (len >= n) {
if (len >= n) {
n = len;
if(n > root_depth) {
// Log.logInfo(YMarkTables.BOOKMARKS_LOG, "<folder>: "+folder+" n: "+n);
prop.put("xbel_"+count+"_elements", "<folder id=\"f:"+UTF8.String(YMarkUtil.getKeyId(foldername[n]))+"\">");
count++;
prop.put("xbel_"+count+"_elements", "<title>" + CharacterCoding.unicode2xml(foldername[n], true) + "</title>");
count++;
prop.put("xbel_"+count+"_elements", "<title>" + CharacterCoding.unicode2xml(foldername[n], true) + "</title>");
count++;
}
try {
bit = sb.tables.bookmarks.getBookmarksByFolder(bmk_user, folder);
} catch (IOException e) {
} catch (final IOException e) {
// TODO: better error handling (avoid NPE)
bit = null;
}
Tables.Row bmk_row = null;
String urlHash;
final YMarkDate date = new YMarkDate();
while(bit.hasNext()){
while(bit.hasNext()){
bmk_row = bit.next();
urlHash = new String(bmk_row.getPK());
if(alias.contains(urlHash)) {
buffer.setLength(0);
buffer.append(YMarkXBELImporter.XBEL.ALIAS.startTag(true));
buffer.append(" ref=\"b:");
buffer.append(urlHash);
buffer.append("\"/>");
prop.put("xbel_"+count+"_elements", buffer.toString());
count++;
buffer.append("\"/>");
prop.put("xbel_"+count+"_elements", buffer.toString());
count++;
} else {
alias.add(urlHash);
if(bmk_row != null) {
buffer.setLength(0);
buffer.append(YMarkXBELImporter.XBEL.BOOKMARK.startTag(true));
buffer.append(" id=\"b:");
buffer.append(urlHash);
buffer.append(YMarkEntry.BOOKMARK.URL.xbel());
buffer.append(CharacterCoding.unicode2xml(bmk_row.get(YMarkEntry.BOOKMARK.URL.key(), YMarkEntry.BOOKMARK.URL.deflt()), true));
buffer.append(YMarkEntry.BOOKMARK.DATE_ADDED.xbel());
date.set(bmk_row.get(YMarkEntry.BOOKMARK.DATE_ADDED.key()));
buffer.append(CharacterCoding.unicode2xml(date.toISO8601(), true));
buffer.append(YMarkEntry.BOOKMARK.DATE_MODIFIED.xbel());
date.set(bmk_row.get(YMarkEntry.BOOKMARK.DATE_MODIFIED.key()));
buffer.append(CharacterCoding.unicode2xml(date.toISO8601(), true));
buffer.append(YMarkEntry.BOOKMARK.DATE_VISITED.xbel());
date.set(bmk_row.get(YMarkEntry.BOOKMARK.DATE_VISITED.key()));
buffer.append(CharacterCoding.unicode2xml(date.toISO8601(), true));
buffer.append(YMarkEntry.BOOKMARK.TAGS.xbel());
buffer.append(bmk_row.get(YMarkEntry.BOOKMARK.TAGS.key(), YMarkEntry.BOOKMARK.TAGS.deflt()));
buffer.append(YMarkEntry.BOOKMARK.PUBLIC.xbel());
buffer.append(bmk_row.get(YMarkEntry.BOOKMARK.PUBLIC.key(), YMarkEntry.BOOKMARK.PUBLIC.deflt()));
buffer.append(YMarkEntry.BOOKMARK.VISITS.xbel());
buffer.append(bmk_row.get(YMarkEntry.BOOKMARK.VISITS.key(), YMarkEntry.BOOKMARK.VISITS.deflt()));
buffer.append("\"\n>");
prop.put("xbel_"+count+"_elements", buffer.toString());
count++;
count++;
buffer.setLength(0);
buffer.append(YMarkXBELImporter.XBEL.TITLE.startTag(false));
buffer.append(CharacterCoding.unicode2xml(bmk_row.get(YMarkEntry.BOOKMARK.TITLE.key(), YMarkEntry.BOOKMARK.TITLE.deflt()), true));
@ -156,9 +156,9 @@ public class get_xbel {
buffer.append(YMarkXBELImporter.XBEL.DESC.endTag(false));
prop.put("xbel_"+count+"_elements", buffer.toString());
count++;
prop.put("xbel_"+count+"_elements", YMarkXBELImporter.XBEL.BOOKMARK.endTag(false));
count++;
prop.put("xbel_"+count+"_elements", YMarkXBELImporter.XBEL.BOOKMARK.endTag(false));
count++;
}
}
}
@ -173,13 +173,13 @@ public class get_xbel {
prop.put("root", root);
prop.put("user", bmk_user.substring(0,1).toUpperCase() + bmk_user.substring(1));
prop.put("xbel", count);
} else {
prop.put(YMarkTables.USER_AUTHENTICATE,YMarkTables.USER_AUTHENTICATE_MSG);
}
}
// return rewrite properties
return prop;
}
}

@ -14,22 +14,22 @@ import de.anomic.data.ymark.YMarkCrawlStart;
import de.anomic.data.ymark.YMarkDate;
import de.anomic.data.ymark.YMarkEntry;
import de.anomic.data.ymark.YMarkTables;
import de.anomic.data.ymark.YMarkUtil;
import de.anomic.data.ymark.YMarkTables.TABLES;
import de.anomic.data.ymark.YMarkUtil;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
public class get_ymark {
private static Switchboard sb = null;
private static serverObjects prop = null;
final static String FOLDER_IMG = "<img src=\"/yacy/ui/img/treeview/folder-closed.gif\" />";
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
sb = (Switchboard) env;
prop = new serverObjects();
int rp; // items per page
int page; // page
int total;
@ -37,15 +37,15 @@ public class get_ymark {
String sortname;
String qtype;
String query;
final UserDB.Entry user = sb.userDB.getUser(header);
final boolean isAdmin = (sb.verifyAuthentication(header, true));
final boolean isAdmin = (sb.verifyAuthentication(header));
final boolean isAuthUser = user!= null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT);
Iterator<Tables.Row> bookmarks = null;
if(isAdmin || isAuthUser) {
final String bmk_user = (isAuthUser ? user.getUserName() : YMarkTables.USER_ADMIN);
query = ".*";
qtype = YMarkEntry.BOOKMARK.TITLE.key();
page = 1;
@ -53,7 +53,7 @@ public class get_ymark {
total = 0;
sortname = YMarkEntry.BOOKMARK.TITLE.key();
sortorder = "asc";
if(post != null) {
rp = (post.containsKey("rp")) ? post.getInt("rp", 10) : 10;
page = (post.containsKey("page")) ? post.getInt("page", 1): 1;
@ -61,7 +61,7 @@ public class get_ymark {
qtype = (post.containsKey("qtype")) ? post.get("qtype", YMarkEntry.BOOKMARK.TAGS.key()) : YMarkEntry.BOOKMARK.TAGS.key();
sortname = (post.containsKey("sortname")) ? post.get("sortname", YMarkEntry.BOOKMARK.TITLE.key()) : YMarkEntry.BOOKMARK.TITLE.key();
sortorder = (post.containsKey("sortorder")) ? post.get("sortorder", "asc") : "asc";
}
}
try {
final String bmk_table = TABLES.BOOKMARKS.tablename(bmk_user);
final Collection<Row> result;
@ -72,7 +72,7 @@ public class get_ymark {
result = sb.tables.bookmarks.orderBookmarksBy(sb.tables.bookmarks.getBookmarksByTag(bmk_user, tagArray), sortname, sortorder);
} else if(qtype.equals("_folder")) {
result = sb.tables.bookmarks.orderBookmarksBy(sb.tables.bookmarks.getBookmarksByFolder(bmk_user, query), sortname, sortorder);
} else {
} else {
result = sb.tables.bookmarks.orderBookmarksBy(sb.tables.iterator(bmk_table, qtype, Pattern.compile(query)), sortname, sortorder);
}
} else {
@ -83,20 +83,20 @@ public class get_ymark {
}
total = result.size();
bookmarks = result.iterator();
} catch (IOException e) {
} catch (final IOException e) {
Log.logException(e);
}
prop.put("page", page);
prop.put("total", total);
putProp(bookmarks, rp, page);
} else {
prop.put(YMarkTables.USER_AUTHENTICATE,YMarkTables.USER_AUTHENTICATE_MSG);
}
// return rewrite properties
return prop;
}
private static void putProp(final Iterator<Tables.Row> bit, final int rp, final int page) {
Tables.Row bmk_row;
int count = 0;
@ -112,11 +112,11 @@ public class get_ymark {
while (count < rp && bit.hasNext()) {
bmk_row = bit.next();
if (bmk_row != null) {
// put JSON
prop.put("json_"+count+"_id", count);
prop.put("json_"+count+"_hash", UTF8.String(bmk_row.getPK()));
for (YMarkEntry.BOOKMARK bmk : YMarkEntry.BOOKMARK.values()) {
for (final YMarkEntry.BOOKMARK bmk : YMarkEntry.BOOKMARK.values()) {
if(bmk == YMarkEntry.BOOKMARK.PUBLIC)
prop.put("json_"+count+"_"+bmk.key(), bmk_row.get(bmk.key(),bmk.deflt()).equals("false") ? 1 : 0);
else if(bmk == YMarkEntry.BOOKMARK.TAGS)
@ -128,7 +128,7 @@ public class get_ymark {
else
prop.putJSON("json_"+count+"_"+bmk.key(), bmk_row.get(bmk.key(),bmk.deflt()));
}
final YMarkCrawlStart crawlstart = new YMarkCrawlStart(sb.tables, bmk_row.get(YMarkEntry.BOOKMARK.URL.key(),YMarkEntry.BOOKMARK.URL.deflt()));
int crawl = 0;
if (!crawlstart.isEmpty()) {
@ -144,20 +144,20 @@ public class get_ymark {
prop.put("json_"+count+"_crawlstart_info", "Crawl is running ...");
}
prop.put("json_"+count+"_crawlstart", crawl);
prop.put("json_"+count+"_apicall_pk", crawlstart.getPK());
prop.put("json_"+count+"_date_recording", YMarkDate.ISO8601(crawlstart.date_recording()).replaceAll("T", "<br />"));
prop.put("json_"+count+"_date_next_exec", YMarkDate.ISO8601(crawlstart.date_next_exec()).replaceAll("T", "<br />"));
prop.put("json_"+count+"_date_last_exec", YMarkDate.ISO8601(crawlstart.date_last_exec()).replaceAll("T", "<br />"));
prop.put("json_"+count+"_date_last_exec", YMarkDate.ISO8601(crawlstart.date_last_exec()).replaceAll("T", "<br />"));
prop.put("json_"+count+"_comma", ",");
// put XML
prop.putXML("xml_"+count+"_id", UTF8.String(bmk_row.getPK()));
for (YMarkEntry.BOOKMARK bmk : YMarkEntry.BOOKMARK.values()) {
for (final YMarkEntry.BOOKMARK bmk : YMarkEntry.BOOKMARK.values()) {
prop.putXML("xml_"+count+"_"+bmk.key(), bmk_row.get(bmk.key(),bmk.deflt()));
}
count++;
}
}

@ -38,7 +38,7 @@ public class import_ymark {
final Switchboard sb = (Switchboard) env;
final serverObjects prop = new serverObjects();
final UserDB.Entry user = sb.userDB.getUser(header);
final boolean isAdmin = (sb.verifyAuthentication(header, true));
final boolean isAdmin = (sb.verifyAuthentication(header));
final boolean isAuthUser = user!= null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT);
final int queueSize = 200;

@ -16,10 +16,10 @@ import de.anomic.server.serverSwitch;
public class manage_tags {
private static Switchboard sb = null;
private static serverObjects prop = null;
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
sb = (Switchboard) env;
prop = new serverObjects();
@ -28,27 +28,27 @@ public class manage_tags {
String query;
String tags;
String replace;
final UserDB.Entry user = sb.userDB.getUser(header);
final boolean isAdmin = (sb.verifyAuthentication(header, true));
final boolean isAdmin = (sb.verifyAuthentication(header));
final boolean isAuthUser = user!= null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT);
if(isAdmin || isAuthUser) {
final String bmk_user = (isAuthUser ? user.getUserName() : YMarkTables.USER_ADMIN);
if(post != null) {
query = post.get("query", post.get("tags", YMarkUtil.EMPTY_STRING));
qtype = post.get("qtype", "_tags");
tags = YMarkUtil.cleanTagsString(post.get("tags", YMarkUtil.EMPTY_STRING));
replace = post.get("replace", YMarkUtil.EMPTY_STRING);
} else {
query = ".*";
qtype = YMarkUtil.EMPTY_STRING;
tags = YMarkUtil.EMPTY_STRING;
replace = YMarkUtil.EMPTY_STRING;
}
try {
final String bmk_table = TABLES.BOOKMARKS.tablename(bmk_user);
final Iterator<Row> row_iter;
@ -61,7 +61,7 @@ public class manage_tags {
row_iter = sb.tables.bookmarks.getBookmarksByTag(bmk_user, tagArray);
} else if(qtype.equals("_folder")) {
row_iter = sb.tables.bookmarks.getBookmarksByFolder(bmk_user, query);
} else {
} else {
row_iter = sb.tables.iterator(bmk_table, qtype, Pattern.compile(query));
}
} else {
@ -74,15 +74,15 @@ public class manage_tags {
}
sb.tables.bookmarks.replaceTags(row_iter, bmk_user, tags, replace);
prop.put("status", 1);
} catch (IOException e) {
} catch (final IOException e) {
Log.logException(e);
} catch (RowSpaceExceededException e) {
} catch (final RowSpaceExceededException e) {
Log.logException(e);
}
}
} else {
prop.put(YMarkTables.USER_AUTHENTICATE,YMarkTables.USER_AUTHENTICATE_MSG);
}
// return rewrite properties
return prop;
}
}
}

@ -9,18 +9,17 @@ import java.util.Scanner;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.search.Switchboard;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
public class ynetSearch {
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
final Switchboard switchboard = (Switchboard) env;
final boolean isAdmin=switchboard.verifyAuthentication(header, true);
final serverObjects prop = new serverObjects();
if(post != null){
final boolean isAdmin=switchboard.verifyAuthentication(header);
final serverObjects prop = new serverObjects();
if(post != null){
if(!isAdmin){
// force authentication if desired
if(post.containsKey("login")){
@ -28,7 +27,7 @@ public class ynetSearch {
}
return prop;
} else {
InputStream is = null;
InputStream is = null;
try {
String searchaddress = post.get("url");
if (!searchaddress.startsWith("http://")) {
@ -42,23 +41,23 @@ public class ynetSearch {
Map.Entry<String, String> k;
while(it.hasNext()) {
k = it.next();
s = s + "&" + k.getKey() + "=" + k.getValue();
s = s + "&" + k.getKey() + "=" + k.getValue();
}
// final String s = searchaddress+"&query="+post.get("search")+"&maximumRecords="+post.get("maximumRecords")+"&startRecord="+post.get("startRecord");
final URL url = new URL(s);
is = url.openStream();
final String httpout = new Scanner(is).useDelimiter( "\\Z" ).next();
// final String s = searchaddress+"&query="+post.get("search")+"&maximumRecords="+post.get("maximumRecords")+"&startRecord="+post.get("startRecord");
final URL url = new URL(s);
is = url.openStream();
final String httpout = new Scanner(is).useDelimiter( "\\Z" ).next();
prop.put("http", httpout);
}
catch ( final Exception e ) {
}
catch ( final Exception e ) {
prop.put("url", "error!");
}
finally {
if ( is != null )
try { is.close(); } catch ( final IOException e ) { }
}
finally {
if ( is != null )
try { is.close(); } catch ( final IOException e ) { }
}
}
}
}
return prop;
}
}

@ -51,7 +51,7 @@ public class index {
}
// access control
final boolean authorizedAccess = sb.verifyAuthentication(header, false);
final boolean authorizedAccess = sb.verifyAuthentication(header);
if ((post != null) && (post.containsKey("publicPage"))) {
if (!authorizedAccess) {
prop.put("AUTHENTICATE", "admin log-in"); // force log-in

@ -87,7 +87,7 @@ public class yacysearch {
final Switchboard sb = (Switchboard) env;
sb.localSearchLastAccess = System.currentTimeMillis();
final boolean searchAllowed = sb.getConfigBool("publicSearchpage", true) || sb.verifyAuthentication(header, false);
final boolean searchAllowed = sb.getConfigBool("publicSearchpage", true) || sb.verifyAuthentication(header);
boolean authenticated = sb.adminAuthenticated(header) >= 2;
if (!authenticated) {
@ -484,7 +484,7 @@ public class yacysearch {
// if a minus-button was hit, remove a special reference first
if (post != null && post.containsKey("deleteref")) {
try {
if (!sb.verifyAuthentication(header, false)) {
if (!sb.verifyAuthentication(header)) {
prop.put("AUTHENTICATE", "admin log-in"); // force log-in
return prop;
}
@ -511,7 +511,7 @@ public class yacysearch {
// if a plus-button was hit, create new voting message
if (post != null && post.containsKey("recommendref")) {
if (!sb.verifyAuthentication(header, false)) {
if (!sb.verifyAuthentication(header)) {
prop.put("AUTHENTICATE", "admin log-in"); // force log-in
return prop;
}
@ -541,7 +541,7 @@ public class yacysearch {
// if a bookmarks-button was hit, create new bookmark entry
if (post != null && post.containsKey("bookmarkref")) {
if (!sb.verifyAuthentication(header, false)) {
if (!sb.verifyAuthentication(header)) {
prop.put("AUTHENTICATE", "admin log-in"); // force log-in
return prop;
}

@ -71,7 +71,7 @@ public class yacysearchitem {
final String eventID = post.get("eventID", "");
final boolean authenticated = sb.adminAuthenticated(header) >= 2;
final int item = post.getInt("item", -1);
final boolean auth = (header.get(HeaderFramework.CONNECTION_PROP_CLIENTIP, "")).equals("localhost") || sb.verifyAuthentication(header, true);
final boolean auth = (header.get(HeaderFramework.CONNECTION_PROP_CLIENTIP, "")).equals("localhost") || sb.verifyAuthentication(header);
final RequestHeader.FileType fileType = header.fileType();
// default settings for blank item
@ -141,7 +141,7 @@ public class yacysearchitem {
prop.putHTML("content_link", result.urlstring());
prop.putHTML("content_showPictures_link", result.urlstring());
prop.putHTML("content_target", target);
if (faviconURL != null && fileType == FileType.HTML) sb.loader.loadIfNotExistBackground(faviconURL.toNormalform(true, false), 1024 * 1024 * 10);
if (faviconURL != null && fileType == FileType.HTML) sb.loader.loadIfNotExistBackground(faviconURL, 1024 * 1024 * 10);
prop.putHTML("content_faviconCode", sb.licensedURLs.aquireLicense(faviconURL)); // acquire license for favicon url loading
prop.put("content_urlhash", resulthashString);
prop.put("content_ranking", result.ranking);
@ -216,7 +216,7 @@ public class yacysearchitem {
prop.put("content_item", "0");
} else {
final String license = sb.licensedURLs.aquireLicense(ms.href);
sb.loader.loadIfNotExistBackground(ms.href.toNormalform(true, false), 1024 * 1024 * 10);
sb.loader.loadIfNotExistBackground(ms.href, 1024 * 1024 * 10);
prop.putHTML("content_item_hrefCache", (auth) ? "/ViewImage.png?url=" + ms.href.toNormalform(true, false) : ms.href.toNormalform(true, false));
prop.putHTML("content_item_href", ms.href.toNormalform(true, false));
prop.putHTML("content_item_target", target);

@ -1140,6 +1140,11 @@ public class MultiProtocolURI implements Serializable, Comparable<MultiProtocolU
final Locale locale = Domains.getLocale(this.hostAddress);
if (locale != null && locale.getCountry() != null && locale.getCountry().length() > 0) return locale;
}
/*
if (this.hostAddress != null) {
return Domains.getLocale(this.hostAddress);
}
*/
return Domains.getLocale(this.host);
}

@ -67,8 +67,8 @@ public class Domains {
private static final String PRESENT = "";
private static final String LOCAL_PATTERNS = "10\\..*,127\\..*,172\\.(1[6-9]|2[0-9]|3[0-1])\\..*,169\\.254\\..*,192\\.168\\..*,localhost";
private static final int MAX_NAME_CACHE_HIT_SIZE = 20000;
private static final int MAX_NAME_CACHE_MISS_SIZE = 20000;
private static final int MAX_NAME_CACHE_HIT_SIZE = 100000;
private static final int MAX_NAME_CACHE_MISS_SIZE = 100000;
private static final int CONCURRENCY_LEVEL = Runtime.getRuntime().availableProcessors() + 1;
// a dns cache
@ -76,7 +76,7 @@ public class Domains {
private static final ARC<String, String> NAME_CACHE_MISS = new ConcurrentARC<String, String>(MAX_NAME_CACHE_MISS_SIZE, CONCURRENCY_LEVEL);
private static final ConcurrentHashMap<String, Object> LOOKUP_SYNC = new ConcurrentHashMap<String, Object>(100, 0.75f, Runtime.getRuntime().availableProcessors() * 2);
private static List<Pattern> nameCacheNoCachingPatterns = Collections.synchronizedList(new LinkedList<Pattern>());
private static final List<Pattern> LOCALHOST_PATTERNS = makePatterns(LOCAL_PATTERNS);
private static final List<Pattern> INTRANET_PATTERNS = makePatterns(LOCAL_PATTERNS);
public static long cacheHit_Hit = 0, cacheHit_Miss = 0, cacheHit_Insert = 0; // for statistics only; do not write
public static long cacheMiss_Hit = 0, cacheMiss_Miss = 0, cacheMiss_Insert = 0; // for statistics only; do not write
@ -809,7 +809,7 @@ public class Domains {
if (localHostAddresses.isEmpty()) return list; // give up
for (final InetAddress a: localHostAddresses) {
if (((0Xff & a.getAddress()[0]) == 127) ||
(!matchesList(a.getHostAddress(), LOCALHOST_PATTERNS))) continue;
(!matchesList(a.getHostAddress(), INTRANET_PATTERNS))) continue;
list.add(a);
}
return list;
@ -894,7 +894,7 @@ public class Domains {
// FIXME IPv4 only
// check local ip addresses
if (matchesList(host, LOCALHOST_PATTERNS)) return true;
if (matchesList(host, INTRANET_PATTERNS)) return true;
if (host.startsWith("0:0:0:0:0:0:0:1")) return true;
// check if there are other local IP addresses that are not in

@ -11,12 +11,12 @@
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program in the file lgpl21.txt
* If not, see <http://www.gnu.org/licenses/>.
@ -28,9 +28,11 @@ import java.util.AbstractMap;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
import java.util.Random;
import java.util.Set;
@ -47,7 +49,7 @@ public final class ConcurrentARC<K, V> extends AbstractMap<K, V> implements Map<
private final int mask;
private final ARC<K, V> arc[];
/**
* create a concurrent ARC based on a HashARC. The type of the key elements must implement a hashing function
* @param cacheSize the number of maximum entries
@ -64,7 +66,7 @@ public final class ConcurrentARC<K, V> extends AbstractMap<K, V> implements Map<
m -= 1;
this.mask = m;
}
/**
* create a concurrent ARC based on a ComparableARC
* @param cacheSize the number of maximum entries
@ -72,7 +74,7 @@ public final class ConcurrentARC<K, V> extends AbstractMap<K, V> implements Map<
* @param comparator a comparator for the key object which may be of type byte[]
*/
@SuppressWarnings("unchecked")
public ConcurrentARC(final int cacheSize, final int partitions, Comparator<? super K> comparator) {
public ConcurrentARC(final int cacheSize, final int partitions, final Comparator<? super K> comparator) {
int m = 1;
while (m < partitions) m = m * 2;
int partitionSize = cacheSize / m;
@ -82,7 +84,7 @@ public final class ConcurrentARC<K, V> extends AbstractMap<K, V> implements Map<
m -= 1;
this.mask = m;
}
/**
* put a value to the cache.
* @param s
@ -98,7 +100,7 @@ public final class ConcurrentARC<K, V> extends AbstractMap<K, V> implements Map<
* @param s
* @param v
*/
public void insertIfAbsent(K s, V v) {
public void insertIfAbsent(final K s, final V v) {
this.arc[getPartition(s)].insertIfAbsent(s, v);
}
@ -109,10 +111,10 @@ public final class ConcurrentARC<K, V> extends AbstractMap<K, V> implements Map<
* @param v
* @return the value before inserting the new value
*/
public V putIfAbsent(K s, V v) {
public V putIfAbsent(final K s, final V v) {
return this.arc[getPartition(s)].putIfAbsent(s, v);
}
/**
* put a value to the cache.
* @param s
@ -122,7 +124,7 @@ public final class ConcurrentARC<K, V> extends AbstractMap<K, V> implements Map<
public final V put(final K s, final V v) {
return this.arc[getPartition(s)].put(s, v);
}
/**
* get a value from the cache.
* @param s
@ -133,18 +135,19 @@ public final class ConcurrentARC<K, V> extends AbstractMap<K, V> implements Map<
public final V get(final Object s) {
return this.arc[getPartition(s)].get((K) s);
}
/**
* check if the map contains the value
* @param value
* @return the keys that have the given value
*/
public Collection<K> getKeys(V value) {
ArrayList<K> keys = new ArrayList<K>();
for (int i = 0; i < this.arc.length; i++) keys.addAll(this.arc[i].getKeys(value));
public Collection<K> getKeys(final V value) {
final ArrayList<K> keys = new ArrayList<K>();
for (final ARC<K, V> element : this.arc)
keys.addAll(element.getKeys(value));
return keys;
}
/**
* check if the map contains the key
* @param s
@ -155,7 +158,7 @@ public final class ConcurrentARC<K, V> extends AbstractMap<K, V> implements Map<
public final boolean containsKey(final Object s) {
return this.arc[getPartition(s)].containsKey((K) s);
}
/**
* remove an entry from the cache
* @param s
@ -166,13 +169,13 @@ public final class ConcurrentARC<K, V> extends AbstractMap<K, V> implements Map<
public final V remove(final Object s) {
return this.arc[getPartition(s)].remove((K) s);
}
/**
* clear the cache
*/
@Override
public final void clear() {
for (ARC<K, V> a: this.arc) a.clear();
for (final ARC<K, V> a: this.arc) a.clear();
}
/**
@ -182,7 +185,7 @@ public final class ConcurrentARC<K, V> extends AbstractMap<K, V> implements Map<
@Override
public final int size() {
int s = 0;
for (ARC<K, V> a: this.arc) s += a.size();
for (final ARC<K, V> a: this.arc) s += a.size();
return s;
}
@ -202,13 +205,13 @@ public final class ConcurrentARC<K, V> extends AbstractMap<K, V> implements Map<
*/
@Override
public Set<java.util.Map.Entry<K, V>> entrySet() {
Set<Map.Entry<K, V>> m = new HashSet<Map.Entry<K, V>>();
for (ARC<K, V> a: this.arc) {
for (Map.Entry<K, V> entry: a.entrySet()) m.add(entry);
final Set<Map.Entry<K, V>> m = new HashSet<Map.Entry<K, V>>();
for (final ARC<K, V> a: this.arc) {
for (final Map.Entry<K, V> entry: a.entrySet()) m.add(entry);
}
return m;
}
/**
* a hash code for this ARC
* @return a hash code
@ -217,7 +220,7 @@ public final class ConcurrentARC<K, V> extends AbstractMap<K, V> implements Map<
public int hashCode() {
return this.arc.hashCode();
}
//private static String latestObject = "";
/**
* return in which partition the Object belongs
@ -228,13 +231,47 @@ public final class ConcurrentARC<K, V> extends AbstractMap<K, V> implements Map<
private int getPartition(final Object x) {
if (x instanceof byte[]) {
int h = 0;
for (byte c: (byte[])x) h = 31 * h + (c & 0xFF);
int p = h & mask;
for (final byte c: (byte[])x) h = 31 * h + (c & 0xFF);
final int p = h & this.mask;
//String o = UTF8.String((byte[]) x); try { if (o.equals(latestObject)) throw new RuntimeException("ConcurrentARC: p = " + p + ", objectb = " + o); } catch (Exception e) { Log.logException(e); } latestObject = o;
return p;
}
int p = x.hashCode() & mask;
final int p = x.hashCode() & this.mask;
//String o = x.toString(); try { if (o.equals(latestObject)) throw new RuntimeException("ConcurrentARC: p = " + p + ", objecto = " + o); } catch (Exception e) { Log.logException(e); } latestObject = o;
return p;
}
public static void main(final String[] args) {
final Random r = new Random();
final int testsize = 10000;
final ARC<String, String> a = new ConcurrentARC<String, String>(testsize * 3, Runtime.getRuntime().availableProcessors());
final Map<String, String> b = new HashMap<String, String>();
String key, value;
for (int i = 0; i < testsize; i++) {
key = "k" + r.nextInt();
value = "v" + r.nextInt();
a.insertIfAbsent(key, value);
b.put(key, value);
}
// now put half of the entries AGAIN into the ARC
int h = testsize / 2;
for (final Map.Entry<String, String> entry: b.entrySet()) {
a.put(entry.getKey(), entry.getValue());
if (h-- <= 0) break;
}
// test correctness
for (final Map.Entry<String, String> entry: b.entrySet()) {
if (!a.containsKey(entry.getKey())) {
System.out.println("missing: " + entry.getKey());
continue;
}
if (!a.get(entry.getKey()).equals(entry.getValue())) {
System.out.println("wrong: a = " + entry.getKey() + "," + a.get(entry.getKey()) + "; v = " + entry.getValue());
}
}
System.out.println("finished test!");
}
}

@ -12,12 +12,12 @@
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program in the file lgpl21.txt
* If not, see <http://www.gnu.org/licenses/>.
@ -26,13 +26,15 @@
package net.yacy.cora.storage;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Random;
public final class HashARC<K, V> extends SimpleARC<K, V> implements Map<K, V>, Iterable<Map.Entry<K, V>>, ARC<K, V> {
public final static boolean accessOrder = false; // if false, then a insertion-order is used
public HashARC(final int cacheSize) {
this.cacheSize = cacheSize / 2;
super.levelA = Collections.synchronizedMap(new LinkedHashMap<K, V>(cacheSize, 0.1f, accessOrder) {
@ -48,4 +50,37 @@ public final class HashARC<K, V> extends SimpleARC<K, V> implements Map<K, V>, I
}
});
}
public static void main(final String[] args) {
final Random r = new Random();
final int testsize = 10000;
final ARC<String, String> a = new HashARC<String, String>(testsize * 2);
final Map<String, String> b = new HashMap<String, String>();
String key, value;
for (int i = 0; i < testsize; i++) {
key = "k" + r.nextInt();
value = "v" + r.nextInt();
a.insertIfAbsent(key, value);
b.put(key, value);
}
// now put half of the entries AGAIN into the ARC
int h = testsize / 2;
for (final Map.Entry<String, String> entry: b.entrySet()) {
a.put(entry.getKey(), entry.getValue());
if (h-- <= 0) break;
}
// test correctness
for (final Map.Entry<String, String> entry: b.entrySet()) {
if (!a.containsKey(entry.getKey())) {
System.out.println("missing: " + entry.getKey());
continue;
}
if (!a.get(entry.getKey()).equals(entry.getValue())) {
System.out.println("wrong: a = " + entry.getKey() + "," + a.get(entry.getKey()) + "; v = " + entry.getValue());
}
}
System.out.println("finished test!");
}
}

@ -12,12 +12,12 @@
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program in the file lgpl21.txt
* If not, see <http://www.gnu.org/licenses/>.
@ -41,7 +41,7 @@ import java.util.Set;
* or http://en.wikipedia.org/wiki/Adaptive_Replacement_Cache
* This version omits the ghost entry handling which is described in ARC, and keeps both cache levels
* at the same size.
*
*
* This class is defined abstract because it shall be used with either the HashARC or the ComparableARC classes
*/
@ -49,7 +49,7 @@ abstract class SimpleARC<K, V> extends AbstractMap<K, V> implements Map<K, V>, I
protected int cacheSize;
protected Map<K, V> levelA, levelB; // we can assume that these maps are synchronized
/**
* put a value to the cache.
* @param s
@ -58,10 +58,10 @@ abstract class SimpleARC<K, V> extends AbstractMap<K, V> implements Map<K, V>, I
public final synchronized void insert(final K s, final V v) {
if (this.levelB.containsKey(s)) {
this.levelB.put(s, v);
assert (this.levelB.size() <= cacheSize); // the cache should shrink automatically
assert (this.levelB.size() <= this.cacheSize); // the cache should shrink automatically
} else {
this.levelA.put(s, v);
assert (this.levelA.size() <= cacheSize); // the cache should shrink automatically
assert (this.levelA.size() <= this.cacheSize); // the cache should shrink automatically
}
}
@ -71,7 +71,7 @@ abstract class SimpleARC<K, V> extends AbstractMap<K, V> implements Map<K, V>, I
* @param s
* @param v
*/
public void insertIfAbsent(K s, V v) {
public void insertIfAbsent(final K s, final V v) {
if (this.levelB.containsKey(s)) {
return;
} else if (this.levelA.containsKey(s)) {
@ -85,12 +85,12 @@ abstract class SimpleARC<K, V> extends AbstractMap<K, V> implements Map<K, V>, I
return;
} else {
this.levelA.put(s, v);
assert (this.levelA.size() <= cacheSize); // the cache should shrink automatically
assert (this.levelA.size() <= this.cacheSize); // the cache should shrink automatically
}
}
}
}
/**
* put a value to the cache if there was not an entry before
* return a previous content value
@ -98,18 +98,18 @@ abstract class SimpleARC<K, V> extends AbstractMap<K, V> implements Map<K, V>, I
* @param v
* @return the value before inserting the new value
*/
public V putIfAbsent(K s, V v) {
public V putIfAbsent(final K s, final V v) {
synchronized (this) {
V o = this.levelB.get(s);
if (o != null) return o;
o = this.levelA.get(s);
if (o != null) return o;
this.levelA.put(s, v);
assert (this.levelA.size() <= cacheSize); // the cache should shrink automatically
assert (this.levelA.size() <= this.cacheSize); // the cache should shrink automatically
return null;
}
}
/**
* put a value to the cache.
* @param s
@ -117,16 +117,16 @@ abstract class SimpleARC<K, V> extends AbstractMap<K, V> implements Map<K, V>, I
*/
public final synchronized V put(final K s, final V v) {
if (this.levelB.containsKey(s)) {
V r = this.levelB.put(s, v);
assert (this.levelB.size() <= cacheSize); // the cache should shrink automatically
final V r = this.levelB.put(s, v);
assert (this.levelB.size() <= this.cacheSize); // the cache should shrink automatically
return r;
} else {
V r = this.levelA.put(s, v);
assert (this.levelA.size() <= cacheSize); // the cache should shrink automatically
final V r = this.levelA.put(s, v);
assert (this.levelA.size() <= this.cacheSize); // the cache should shrink automatically
return r;
}
}
/**
* get a value from the cache.
* @param s
@ -148,7 +148,7 @@ abstract class SimpleARC<K, V> extends AbstractMap<K, V> implements Map<K, V>, I
// move value from A to B; since it was already removed from A, just put it to B
//System.out.println("ARC: moving A->B, size(A) = " + this.levelA.size() + ", size(B) = " + this.levelB.size());
this.levelB.put((K) s, v);
assert (this.levelB.size() <= cacheSize); // the cache should shrink automatically
assert (this.levelB.size() <= this.cacheSize); // the cache should shrink automatically
}
return v;
}
@ -158,21 +158,21 @@ abstract class SimpleARC<K, V> extends AbstractMap<K, V> implements Map<K, V>, I
* @param value
* @return the keys that have the given value
*/
public Collection<K> getKeys(V value) {
ArrayList<K> keys = new ArrayList<K>();
public Collection<K> getKeys(final V value) {
final ArrayList<K> keys = new ArrayList<K>();
synchronized (this.levelB) {
for (Map.Entry<K, V> entry: this.levelB.entrySet()) {
for (final Map.Entry<K, V> entry: this.levelB.entrySet()) {
if (value.equals(entry.getValue())) keys.add(entry.getKey());
}
}
synchronized (this) {
for (Map.Entry<K, V> entry: this.levelA.entrySet()) {
for (final Map.Entry<K, V> entry: this.levelA.entrySet()) {
if (value.equals(entry.getValue())) keys.add(entry.getKey());
}
}
return keys;
}
/**
* check if the map contains the key
* @param s
@ -183,8 +183,8 @@ abstract class SimpleARC<K, V> extends AbstractMap<K, V> implements Map<K, V>, I
if (this.levelB.containsKey(s)) return true;
return this.levelA.containsKey(s);
}
/**
* remove an entry from the cache
* @param s
@ -196,7 +196,7 @@ abstract class SimpleARC<K, V> extends AbstractMap<K, V> implements Map<K, V>, I
if (r != null) return r;
return this.levelA.remove(s);
}
/**
* clear the cache
*/
@ -214,7 +214,7 @@ abstract class SimpleARC<K, V> extends AbstractMap<K, V> implements Map<K, V>, I
public final synchronized int size() {
return this.levelA.size() + this.levelB.size();
}
/**
* iterator implements the Iterable interface
*/
@ -231,12 +231,12 @@ abstract class SimpleARC<K, V> extends AbstractMap<K, V> implements Map<K, V>, I
*/
@Override
public final synchronized Set<Map.Entry<K, V>> entrySet() {
Set<Map.Entry<K, V>> m = new HashSet<Map.Entry<K, V>>();
for (Map.Entry<K, V> entry: this.levelA.entrySet()) m.add(entry);
for (Map.Entry<K, V> entry: this.levelB.entrySet()) m.add(entry);
final Set<Map.Entry<K, V>> m = new HashSet<Map.Entry<K, V>>();
for (final Map.Entry<K, V> entry: this.levelA.entrySet()) m.add(entry);
for (final Map.Entry<K, V> entry: this.levelB.entrySet()) m.add(entry);
return m;
}
/**
* a hash code for this ARC
* @return the hash code of one of the ARC partial hash tables
@ -245,4 +245,5 @@ abstract class SimpleARC<K, V> extends AbstractMap<K, V> implements Map<K, V>, I
public final int hashCode() {
return this.levelA.hashCode();
}
}

@ -27,6 +27,7 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.MalformedURLException;
import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;
@ -59,7 +60,10 @@ public class OAIListFriendsLoader {
public static void init(final LoaderDispatcher loader, final Map<String, File> moreFriends) {
listFriends.putAll(moreFriends);
if (loader != null) for (final Map.Entry<String, File> oaiFriend: listFriends.entrySet()) {
loader.loadIfNotExistBackground(oaiFriend.getKey(), oaiFriend.getValue(), Integer.MAX_VALUE);
try {
loader.loadIfNotExistBackground(new DigestURI(oaiFriend.getKey()), oaiFriend.getValue(), Integer.MAX_VALUE);
} catch (final MalformedURLException e) {
}
}
}

@ -77,7 +77,7 @@ public final class LoaderDispatcher {
private final FTPLoader ftpLoader;
private final SMBLoader smbLoader;
private final FileLoader fileLoader;
private final ConcurrentHashMap<String, Semaphore> loaderSteering; // a map that delivers a 'finish' semaphore for urls
private final ConcurrentHashMap<DigestURI, Semaphore> loaderSteering; // a map that delivers a 'finish' semaphore for urls
private final Log log;
public LoaderDispatcher(final Switchboard sb) {
@ -90,7 +90,7 @@ public final class LoaderDispatcher {
this.ftpLoader = new FTPLoader(sb, this.log);
this.smbLoader = new SMBLoader(sb, this.log);
this.fileLoader = new FileLoader(sb, this.log);
this.loaderSteering = new ConcurrentHashMap<String, Semaphore>();
this.loaderSteering = new ConcurrentHashMap<DigestURI, Semaphore>();
}
public boolean isSupportedProtocol(final String protocol) {
@ -153,8 +153,7 @@ public final class LoaderDispatcher {
}
public Response load(final Request request, final CacheStrategy cacheStrategy, final int maxFileSize, final boolean checkBlacklist) throws IOException {
final String url = request.url().toNormalform(true, false);
Semaphore check = this.loaderSteering.get(url);
Semaphore check = this.loaderSteering.get(request.url());
if (check != null) {
// a loading process may be going on for that url
try { check.tryAcquire(5, TimeUnit.SECONDS);} catch (final InterruptedException e) {}
@ -162,15 +161,15 @@ public final class LoaderDispatcher {
// which may be successful faster because of a cache hit
}
this.loaderSteering.put(url, new Semaphore(0));
this.loaderSteering.put(request.url(), new Semaphore(0));
try {
final Response response = loadInternal(request, cacheStrategy, maxFileSize, checkBlacklist);
check = this.loaderSteering.remove(url);
check = this.loaderSteering.remove(request.url());
if (check != null) check.release(1000);
return response;
} catch (final IOException e) {
// release the semaphore anyway
check = this.loaderSteering.remove(url);
check = this.loaderSteering.remove(request.url());
if (check != null) check.release(1000);
//Log.logException(e);
throw new IOException(e);
@ -386,22 +385,22 @@ public final class LoaderDispatcher {
}
}
public void loadIfNotExistBackground(final String url, final File cache, final int maxFileSize) {
public void loadIfNotExistBackground(final DigestURI url, final File cache, final int maxFileSize) {
new Loader(url, cache, maxFileSize, CacheStrategy.IFEXIST).start();
}
public void loadIfNotExistBackground(final String url, final int maxFileSize) {
public void loadIfNotExistBackground(final DigestURI url, final int maxFileSize) {
new Loader(url, null, maxFileSize, CacheStrategy.IFEXIST).start();
}
private class Loader extends Thread {
private final String url;
private final DigestURI url;
private final File cache;
private final int maxFileSize;
private final CacheStrategy cacheStrategy;
public Loader(final String url, final File cache, final int maxFileSize, final CacheStrategy cacheStrategy) {
public Loader(final DigestURI url, final File cache, final int maxFileSize, final CacheStrategy cacheStrategy) {
this.url = url;
this.cache = cache;
this.maxFileSize = maxFileSize;
@ -412,7 +411,7 @@ public final class LoaderDispatcher {
if (this.cache != null && this.cache.exists()) return;
try {
// load from the net
final Response response = load(request(new DigestURI(this.url), false, true), this.cacheStrategy, this.maxFileSize, true);
final Response response = load(request(this.url, false, true), this.cacheStrategy, this.maxFileSize, true);
final byte[] b = response.getContent();
if (this.cache != null) FileUtils.copy(b, this.cache);
} catch (final MalformedURLException e) {} catch (final IOException e) {}

@ -2348,7 +2348,7 @@ public final class Switchboard extends serverSwitch {
return 1;
}
public boolean verifyAuthentication(final RequestHeader header, final boolean strict) {
public boolean verifyAuthentication(final RequestHeader header) {
// handle access rights
switch (adminAuthenticated(header)) {
case 0: // wrong password given
@ -2357,7 +2357,7 @@ public final class Switchboard extends serverSwitch {
case 1: // no password given
return false;
case 2: // no password stored
return !strict;
return true;
case 3: // soft-authenticated for localhost only
return true;
case 4: // hard-authenticated, all ok

Loading…
Cancel
Save