refactoring - moved htroot/yacy classes

pull/533/head
Michael Peter Christen 2 years ago
parent 48fcf3b3b5
commit 6d388bb7bf

@ -1,4 +1,3 @@
package net.yacy.htroot;
// Wiki.java // Wiki.java
// ----------------------- // -----------------------
// part of the AnomicHTTPD caching proxy // part of the AnomicHTTPD caching proxy
@ -31,6 +30,8 @@ package net.yacy.htroot;
// javac -classpath .:../classes Wiki.java // javac -classpath .:../classes Wiki.java
// if the shell's current path is HTROOT // if the shell's current path is HTROOT
package net.yacy.htroot;
import java.io.IOException; import java.io.IOException;
import java.util.Arrays; import java.util.Arrays;
import java.util.Date; import java.util.Date;
@ -126,7 +127,7 @@ public class Wiki {
prop.putHTML(serverObjects.ACTION_LOCATION, "Wiki.html?page=" + pagename); prop.putHTML(serverObjects.ACTION_LOCATION, "Wiki.html?page=" + pagename);
prop.put(serverObjects.ACTION_LOCATION, prop.get(serverObjects.ACTION_LOCATION)); prop.put(serverObjects.ACTION_LOCATION, prop.get(serverObjects.ACTION_LOCATION));
} }
if (post != null && post.containsKey("edit")) { if (post != null && post.containsKey("edit")) {
if ((access.equals("admin") && (!sb.verifyAuthentication(header)))) { if ((access.equals("admin") && (!sb.verifyAuthentication(header)))) {
// check access right for admin // check access right for admin
@ -146,7 +147,7 @@ public class Wiki {
prop.putHTML("mode_pagename", pagename); prop.putHTML("mode_pagename", pagename);
prop.putHTML("mode_author", author); prop.putHTML("mode_author", author);
prop.put("mode_date", dateString(new Date())); prop.put("mode_date", dateString(new Date()));
/* We do not fill hostport parameter : relative links should stay relative as it is more reliable /* We do not fill hostport parameter : relative links should stay relative as it is more reliable
* when the peer is behind any kind of reverse Proxy */ * when the peer is behind any kind of reverse Proxy */
prop.putWiki("mode_page", post.get("content", "")); prop.putWiki("mode_page", post.get("content", ""));
prop.putHTML("mode_page-code", post.get("content", "")); prop.putHTML("mode_page-code", post.get("content", ""));
@ -238,7 +239,7 @@ public class Wiki {
prop.putHTML("mode_versioning_pagename", pagename); prop.putHTML("mode_versioning_pagename", pagename);
prop.putHTML("mode_versioning_author", oentry.author()); prop.putHTML("mode_versioning_author", oentry.author());
prop.put("mode_versioning_date", dateString(oentry.date())); prop.put("mode_versioning_date", dateString(oentry.date()));
/* We do not fill hostport parameter : relative links should stay relative as it is more reliable /* We do not fill hostport parameter : relative links should stay relative as it is more reliable
* when the peer is behind any kind of reverse Proxy */ * when the peer is behind any kind of reverse Proxy */
prop.putWiki("mode_versioning_page", oentry.page()); prop.putWiki("mode_versioning_page", oentry.page());
prop.putHTML("mode_versioning_page-code", UTF8.String(oentry.page())); prop.putHTML("mode_versioning_page-code", UTF8.String(oentry.page()));
@ -255,7 +256,7 @@ public class Wiki {
prop.putHTML("mode_pagename", pagename); prop.putHTML("mode_pagename", pagename);
prop.putHTML("mode_author", page.author()); prop.putHTML("mode_author", page.author());
prop.put("mode_date", dateString(page.date())); prop.put("mode_date", dateString(page.date()));
/* We do not fill hostport parameter : relative links should stay relative as it is more reliable /* We do not fill hostport parameter : relative links should stay relative as it is more reliable
* when the peer is behind any kind of reverse Proxy */ * when the peer is behind any kind of reverse Proxy */
prop.putWiki("mode_page", page.page()); prop.putWiki("mode_page", page.page());

@ -22,6 +22,8 @@
// along with this program; if not, write to the Free Software // along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package net.yacy.htroot.api;
import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.protocol.RequestHeader;
import net.yacy.search.Switchboard; import net.yacy.search.Switchboard;
import net.yacy.server.serverObjects; import net.yacy.server.serverObjects;
@ -34,7 +36,7 @@ public class trail_p {
final serverObjects prop = new serverObjects(); final serverObjects prop = new serverObjects();
int c = 0; int c = 0;
for (String t: sb.trail) { for (final String t: sb.trail) {
prop.put("trails_" + c++ + "_trail", t); // don't put in putHTML or putXML in, this is wrong! prop.put("trails_" + c++ + "_trail", t); // don't put in putHTML or putXML in, this is wrong!
} }
prop.put("trails", c); prop.put("trails", c);

@ -26,8 +26,10 @@
// You must compile this file with // You must compile this file with
// javac -classpath .:../classes crawlOrder.java // javac -classpath .:../classes crawlOrder.java
package net.yacy.htroot.yacy;
import java.io.IOException; import java.io.IOException;
import net.yacy.cora.document.encoding.ASCII; import net.yacy.cora.document.encoding.ASCII;
import net.yacy.cora.federate.solr.FailCategory; import net.yacy.cora.federate.solr.FailCategory;
import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.protocol.RequestHeader;

@ -27,6 +27,8 @@
// javac -classpath .:../../classes hello.java // javac -classpath .:../../classes hello.java
// if the shell's current path is HTROOT // if the shell's current path is HTROOT
package net.yacy.htroot.yacy;
import java.io.IOException; import java.io.IOException;
import java.net.InetAddress; import java.net.InetAddress;
import java.net.MalformedURLException; import java.net.MalformedURLException;
@ -38,8 +40,8 @@ import net.yacy.cora.document.id.MultiProtocolURL;
import net.yacy.cora.protocol.Domains; import net.yacy.cora.protocol.Domains;
import net.yacy.cora.protocol.HeaderFramework; import net.yacy.cora.protocol.HeaderFramework;
import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.protocol.RequestHeader;
import net.yacy.peers.Network;
import net.yacy.peers.DHTSelection; import net.yacy.peers.DHTSelection;
import net.yacy.peers.Network;
import net.yacy.peers.Protocol; import net.yacy.peers.Protocol;
import net.yacy.peers.Seed; import net.yacy.peers.Seed;
import net.yacy.peers.graphics.ProfilingGraph; import net.yacy.peers.graphics.ProfilingGraph;
@ -51,7 +53,7 @@ import net.yacy.server.serverObjects;
import net.yacy.server.serverSwitch; import net.yacy.server.serverSwitch;
public final class hello { public final class hello {
// example: // example:
// http://localhost:8090/yacy/hello.html?count=1&seed=p|{Hash=sCJ6Tq8T0N9x,Port=8090,PeerType=junior} // http://localhost:8090/yacy/hello.html?count=1&seed=p|{Hash=sCJ6Tq8T0N9x,Port=8090,PeerType=junior}
// http://localhost:8090/yacy/hello.html?count=10&seed=z|H4sIAAAAAAAAADWQW2vDMAyF_81eJork3GyGX-YxGigly2WFvZTQijbQJsHx1pWx_z7nMj1J4ug7B_2s6-GsP5q3G-G6vBz2e0iz8t6zfuBr7-5PUNanQfulhqyzTkuUCFXvmitrBJtq4ed3tkPTtRpXhIiRDAmq0uhHFIiQMduJ-NXYU9NCbrrP1vnjIdUqgk09uIK51V6rMBRIilAo2NajwzfhGcx8QUKsEIp5iCJo-eaTVUXPfPQ4k5dm4pp8NzaESsLzS-14QVNIMlA-ka2m1JuZJJWIBRwPo0GIIiYp4zCSkC5GQSLiJIah0p6X_rvlS-MTbWdhkCSBIni9jA_rfP3-Ae1Oye9dAQAA // http://localhost:8090/yacy/hello.html?count=10&seed=z|H4sIAAAAAAAAADWQW2vDMAyF_81eJork3GyGX-YxGigly2WFvZTQijbQJsHx1pWx_z7nMj1J4ug7B_2s6-GsP5q3G-G6vBz2e0iz8t6zfuBr7-5PUNanQfulhqyzTkuUCFXvmitrBJtq4ed3tkPTtRpXhIiRDAmq0uhHFIiQMduJ-NXYU9NCbrrP1vnjIdUqgk09uIK51V6rMBRIilAo2NajwzfhGcx8QUKsEIp5iCJo-eaTVUXPfPQ4k5dm4pp8NzaESsLzS-14QVNIMlA-ka2m1JuZJJWIBRwPo0GIIiYp4zCSkC5GQSLiJIah0p6X_rvlS-MTbWdhkCSBIni9jA_rfP3-Ae1Oye9dAQAA
@ -162,8 +164,8 @@ public final class hello {
int callbackRemain = Math.min(5, reportedips.size()); int callbackRemain = Math.min(5, reportedips.size());
final long callbackStart = System.currentTimeMillis(); final long callbackStart = System.currentTimeMillis();
if (callbackRemain > 0 && reportedips.size() > 0) { if (callbackRemain > 0 && reportedips.size() > 0) {
for (String reportedip: reportedips) { for (final String reportedip: reportedips) {
int partialtimeout = ((int) (callbackStart + totalTimeout - System.currentTimeMillis())) / callbackRemain; // bad hack until a concurrent version is implemented final int partialtimeout = ((int) (callbackStart + totalTimeout - System.currentTimeMillis())) / callbackRemain; // bad hack until a concurrent version is implemented
if (partialtimeout <= 0) break; if (partialtimeout <= 0) break;
//ConcurrentLog.info("**hello-DEBUG**", "reportedip = " + reportedip + " is handled"); //ConcurrentLog.info("**hello-DEBUG**", "reportedip = " + reportedip + " is handled");
if (Seed.isProperIP(reportedip)) { if (Seed.isProperIP(reportedip)) {
@ -174,7 +176,7 @@ public final class hello {
try { try {
MultiProtocolURL remoteBaseURL = remoteSeed.getPublicMultiprotocolURL(reportedip, preferHttps); MultiProtocolURL remoteBaseURL = remoteSeed.getPublicMultiprotocolURL(reportedip, preferHttps);
callback = Protocol.queryRWICount(remoteBaseURL, remoteSeed, partialtimeout); callback = Protocol.queryRWICount(remoteBaseURL, remoteSeed, partialtimeout);
if (callback[0] < 0 && remoteBaseURL.isHTTPS()) { if (callback[0] < 0 && remoteBaseURL.isHTTPS()) {
/* Failed using https : retry using http */ /* Failed using https : retry using http */
remoteBaseURL = remoteSeed.getPublicMultiprotocolURL(reportedip, false); remoteBaseURL = remoteSeed.getPublicMultiprotocolURL(reportedip, false);
callback = Protocol.queryRWICount(remoteBaseURL, remoteSeed, partialtimeout); callback = Protocol.queryRWICount(remoteBaseURL, remoteSeed, partialtimeout);
@ -185,9 +187,9 @@ public final class hello {
//ConcurrentLog.info("**hello-DEBUG**", "reportedip = " + reportedip + " returns callback " + (callback == null ? "NULL" : callback[0])); //ConcurrentLog.info("**hello-DEBUG**", "reportedip = " + reportedip + " returns callback " + (callback == null ? "NULL" : callback[0]));
time_backping = System.currentTimeMillis() - time; time_backping = System.currentTimeMillis() - time;
backping_method = "reportedip=" + reportedip; backping_method = "reportedip=" + reportedip;
if (callback[0] >= 0) { if (callback[0] >= 0) {
success = true; success = true;
break; break;
} }
if (--callbackRemain <= 0) break; // no more tries left / restrict to a limited number of ips if (--callbackRemain <= 0) break; // no more tries left / restrict to a limited number of ips
} }

@ -22,6 +22,8 @@
* If not, see <http://www.gnu.org/licenses/>. * If not, see <http://www.gnu.org/licenses/>.
*/ */
package net.yacy.htroot.yacy;
import java.util.Iterator; import java.util.Iterator;
import net.yacy.cora.document.encoding.ASCII; import net.yacy.cora.document.encoding.ASCII;

@ -25,21 +25,21 @@
// along with this program; if not, write to the Free Software // along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package net.yacy.htroot.yacy;
import java.io.File; import java.io.File;
import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.protocol.RequestHeader;
import net.yacy.cora.util.CommonPattern; import net.yacy.cora.util.CommonPattern;
import net.yacy.kelondro.util.FileUtils; import net.yacy.kelondro.util.FileUtils;
import net.yacy.peers.Seed;
import net.yacy.peers.Protocol; import net.yacy.peers.Protocol;
import net.yacy.peers.Seed;
import net.yacy.search.Switchboard; import net.yacy.search.Switchboard;
import net.yacy.search.SwitchboardConstants; import net.yacy.search.SwitchboardConstants;
import net.yacy.server.serverCore; import net.yacy.server.serverCore;
import net.yacy.server.serverObjects; import net.yacy.server.serverObjects;
import net.yacy.server.serverSwitch; import net.yacy.server.serverSwitch;
public final class list { public final class list {
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {

@ -27,12 +27,16 @@
// javac -classpath .:../../classes message.java // javac -classpath .:../../classes message.java
// if the shell's current path is HTROOT/yacy // if the shell's current path is HTROOT/yacy
package net.yacy.htroot.yacy;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.io.PrintWriter; import java.io.PrintWriter;
import java.net.InetAddress; import java.net.InetAddress;
import java.util.Date; import java.util.Date;
import com.google.common.io.Files;
import net.yacy.cora.date.GenericFormatter; import net.yacy.cora.date.GenericFormatter;
import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.document.encoding.UTF8;
import net.yacy.cora.protocol.Domains; import net.yacy.cora.protocol.Domains;
@ -49,8 +53,6 @@ import net.yacy.server.serverObjects;
import net.yacy.server.serverSwitch; import net.yacy.server.serverSwitch;
import net.yacy.utils.crypt; import net.yacy.utils.crypt;
import com.google.common.io.Files;
public final class message { public final class message {
@ -67,7 +69,7 @@ public final class message {
prop.put("messagesize", "0"); prop.put("messagesize", "0");
prop.put("attachmentsize", "0"); prop.put("attachmentsize", "0");
prop.put("response", "-1"); // request rejected prop.put("response", "-1"); // request rejected
if ((post == null) || (env == null)) return prop; if ((post == null) || (env == null)) return prop;
if (!Protocol.authentifyRequest(post, env)) return prop; if (!Protocol.authentifyRequest(post, env)) return prop;

@ -29,6 +29,8 @@
// javac -classpath .:../../Classes hello.java // javac -classpath .:../../Classes hello.java
// if the shell's current path is HTROOT // if the shell's current path is HTROOT
package net.yacy.htroot.yacy;
import java.io.File; import java.io.File;
import java.io.FileInputStream; import java.io.FileInputStream;
import java.io.IOException; import java.io.IOException;

@ -27,6 +27,8 @@
// javac -classpath .:../../classes query.java // javac -classpath .:../../classes query.java
// if the shell's current path is HTROOT // if the shell's current path is HTROOT
package net.yacy.htroot.yacy;
import java.io.IOException; import java.io.IOException;
import net.yacy.cora.date.GenericFormatter; import net.yacy.cora.date.GenericFormatter;

@ -28,6 +28,8 @@
// javac -classpath .:../../Classes search.java // javac -classpath .:../../Classes search.java
// if the shell's current path is htroot/yacy // if the shell's current path is htroot/yacy
package net.yacy.htroot.yacy;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
@ -119,7 +121,7 @@ public final class search {
final boolean strictContentDom = post.getBoolean("strictContentDom"); final boolean strictContentDom = post.getBoolean("strictContentDom");
final String filter = post.get("filter", ".*"); // a filter on the url final String filter = post.get("filter", ".*"); // a filter on the url
final int timezoneOffset = post.getInt("timezoneOffset", 0); final int timezoneOffset = post.getInt("timezoneOffset", 0);
QueryModifier modifier = new QueryModifier(timezoneOffset); final QueryModifier modifier = new QueryModifier(timezoneOffset);
modifier.sitehost = post.get("sitehost", ""); if (modifier.sitehost.isEmpty()) modifier.sitehost = null; modifier.sitehost = post.get("sitehost", ""); if (modifier.sitehost.isEmpty()) modifier.sitehost = null;
modifier.sitehash = post.get("sitehash", ""); if (modifier.sitehash.isEmpty()) modifier.sitehash = null; modifier.sitehash = post.get("sitehash", ""); if (modifier.sitehash.isEmpty()) modifier.sitehash = null;
modifier.author = post.get("author", ""); if (modifier.author.isEmpty()) modifier.author = null; modifier.author = post.get("author", ""); if (modifier.author.isEmpty()) modifier.author = null;
@ -225,7 +227,7 @@ public final class search {
if (query.isEmpty() && abstractSet != null) { if (query.isEmpty() && abstractSet != null) {
// this is _not_ a normal search, only a request for index abstracts // this is _not_ a normal search, only a request for index abstracts
final Segment indexSegment = sb.index; final Segment indexSegment = sb.index;
QueryGoal qg = new QueryGoal(abstractSet, new RowHandleSet(WordReferenceRow.urlEntryRow.primaryKeyLength, WordReferenceRow.urlEntryRow.objectOrder, 0)); final QueryGoal qg = new QueryGoal(abstractSet, new RowHandleSet(WordReferenceRow.urlEntryRow.primaryKeyLength, WordReferenceRow.urlEntryRow.objectOrder, 0));
theQuery = new QueryParams( theQuery = new QueryParams(
qg, qg,
modifier, modifier,
@ -287,10 +289,10 @@ public final class search {
} else { } else {
// retrieve index containers from search request // retrieve index containers from search request
RowHandleSet allHashes = new RowHandleSet(WordReferenceRow.urlEntryRow.primaryKeyLength, WordReferenceRow.urlEntryRow.objectOrder, 0); final RowHandleSet allHashes = new RowHandleSet(WordReferenceRow.urlEntryRow.primaryKeyLength, WordReferenceRow.urlEntryRow.objectOrder, 0);
try {allHashes.putAll(queryhashes);} catch (final SpaceExceededException e) {} try {allHashes.putAll(queryhashes);} catch (final SpaceExceededException e) {}
try {allHashes.putAll(excludehashes);} catch (final SpaceExceededException e) {} try {allHashes.putAll(excludehashes);} catch (final SpaceExceededException e) {}
QueryGoal qg = new QueryGoal(queryhashes, excludehashes); final QueryGoal qg = new QueryGoal(queryhashes, excludehashes);
theQuery = new QueryParams( theQuery = new QueryParams(
qg, qg,
modifier, modifier,
@ -325,7 +327,7 @@ public final class search {
Network.log.info("INIT HASH SEARCH (query-" + abstracts + "): " + QueryParams.anonymizedQueryHashes(theQuery.getQueryGoal().getIncludeHashes()) + " - " + theQuery.itemsPerPage() + " links"); Network.log.info("INIT HASH SEARCH (query-" + abstracts + "): " + QueryParams.anonymizedQueryHashes(theQuery.getQueryGoal().getIncludeHashes()) + " - " + theQuery.itemsPerPage() + " links");
EventChannel.channels(EventChannel.REMOTESEARCH).addMessage(new RSSMessage("Remote Search Request from " + ((remoteSeed == null) ? "unknown" : remoteSeed.getName()), QueryParams.anonymizedQueryHashes(theQuery.getQueryGoal().getIncludeHashes()), "")); EventChannel.channels(EventChannel.REMOTESEARCH).addMessage(new RSSMessage("Remote Search Request from " + ((remoteSeed == null) ? "unknown" : remoteSeed.getName()), QueryParams.anonymizedQueryHashes(theQuery.getQueryGoal().getIncludeHashes()), ""));
if (sb.getConfigBool(SwitchboardConstants.DECORATION_AUDIO, false)) Audio.Soundclip.remotesearch.play(-10.0f); if (sb.getConfigBool(SwitchboardConstants.DECORATION_AUDIO, false)) Audio.Soundclip.remotesearch.play(-10.0f);
// make event // make event
theSearch = SearchEventCache.getEvent(theQuery, sb.peers, sb.tables, null, abstracts.length() > 0, sb.loader, count, maxtime); theSearch = SearchEventCache.getEvent(theQuery, sb.peers, sb.tables, null, abstracts.length() > 0, sb.loader, count, maxtime);
if (theSearch.rwiProcess != null && theSearch.rwiProcess.isAlive()) try {theSearch.rwiProcess.join();} catch (final InterruptedException e) {} if (theSearch.rwiProcess != null && theSearch.rwiProcess.isAlive()) try {theSearch.rwiProcess.join();} catch (final InterruptedException e) {}
@ -384,7 +386,7 @@ public final class search {
final long timer = System.currentTimeMillis(); final long timer = System.currentTimeMillis();
//final ScoreMap<String> topicNavigator = sb.index.connectedRWI() ? theSearch.getTopics(5, 100) : new ConcurrentScoreMap<String>(); //final ScoreMap<String> topicNavigator = sb.index.connectedRWI() ? theSearch.getTopics(5, 100) : new ConcurrentScoreMap<String>();
final ScoreMap<String> topicNavigator = theSearch.getTopics(); // as there is currently no index interaction in getTopics(), we can use it by default final ScoreMap<String> topicNavigator = theSearch.getTopics(); // as there is currently no index interaction in getTopics(), we can use it by default
final StringBuilder refstr = new StringBuilder(6000); final StringBuilder refstr = new StringBuilder(6000);
final Iterator<String> navigatorIterator = topicNavigator.keys(false); final Iterator<String> navigatorIterator = topicNavigator.keys(false);
int i = 0; int i = 0;
@ -400,7 +402,7 @@ public final class search {
prop.put("indexabstract", indexabstract.toString()); prop.put("indexabstract", indexabstract.toString());
// prepare result // prepare result
int resultCount = theSearch == null ? 0 : theSearch.getResultCount(); // theSearch may be null if we searched only for abstracts final int resultCount = theSearch == null ? 0 : theSearch.getResultCount(); // theSearch may be null if we searched only for abstracts
if (resultCount == 0 || accu == null || accu.isEmpty()) { if (resultCount == 0 || accu == null || accu.isEmpty()) {
// no results // no results

@ -19,6 +19,8 @@
// along with this program; if not, write to the Free Software // along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package net.yacy.htroot.yacy;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
@ -45,38 +47,38 @@ import net.yacy.server.serverSwitch;
public final class seedlist { public final class seedlist {
private static final int LISTMAX = 1000; private static final int LISTMAX = 1000;
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
// return variable that accumulates replacements // return variable that accumulates replacements
final Switchboard sb = (Switchboard) env; final Switchboard sb = (Switchboard) env;
int maxcount = Math.min(LISTMAX, post == null ? Integer.MAX_VALUE : post.getInt("maxcount", Integer.MAX_VALUE)); final int maxcount = Math.min(LISTMAX, post == null ? Integer.MAX_VALUE : post.getInt("maxcount", Integer.MAX_VALUE));
float minversion = Math.min(LISTMAX, post == null ? 0.0f : post.getFloat("minversion", 0.0f)); final float minversion = Math.min(LISTMAX, post == null ? 0.0f : post.getFloat("minversion", 0.0f));
boolean nodeonly = post == null || !post.containsKey("node") ? false : post.getBoolean("node"); final boolean nodeonly = post == null || !post.containsKey("node") ? false : post.getBoolean("node");
boolean includeme = post == null || !post.containsKey("me") ? true : post.getBoolean("me"); final boolean includeme = post == null || !post.containsKey("me") ? true : post.getBoolean("me");
boolean addressonly = post == null || !post.containsKey("address") ? false : post.getBoolean("address"); final boolean addressonly = post == null || !post.containsKey("address") ? false : post.getBoolean("address");
String peername = post == null ? null : post.containsKey("my") ? sb.peers.myName() : post.get("peername"); final String peername = post == null ? null : post.containsKey("my") ? sb.peers.myName() : post.get("peername");
final ArrayList<Seed> v; final ArrayList<Seed> v;
if (post != null && post.containsKey("my")) { if (post != null && post.containsKey("my")) {
v = new ArrayList<Seed>(1); v = new ArrayList<Seed>(1);
v.add(sb.peers.mySeed()); v.add(sb.peers.mySeed());
} else if (post != null && post.containsKey("id")) { } else if (post != null && post.containsKey("id")) {
v = new ArrayList<Seed>(1); v = new ArrayList<Seed>(1);
Seed s = sb.peers.get(post.get("id")); final Seed s = sb.peers.get(post.get("id"));
if (s != null) v.add(s); if (s != null) v.add(s);
} else if (post != null && post.containsKey("name")) { } else if (post != null && post.containsKey("name")) {
v = new ArrayList<Seed>(1); v = new ArrayList<Seed>(1);
Seed s = sb.peers.lookupByName(post.get("name")); final Seed s = sb.peers.lookupByName(post.get("name"));
if (s != null) v.add(s); if (s != null) v.add(s);
} else { } else {
v= sb.peers.getSeedlist(maxcount, includeme, nodeonly, minversion); v= sb.peers.getSeedlist(maxcount, includeme, nodeonly, minversion);
} }
final serverObjects prop = new serverObjects(); final serverObjects prop = new serverObjects();
// write simple-encoded seed lines or json // write simple-encoded seed lines or json
String EXT = header.get(HeaderFramework.CONNECTION_PROP_EXT); final String EXT = header.get(HeaderFramework.CONNECTION_PROP_EXT);
boolean json = EXT != null && EXT.equals("json"); final boolean json = EXT != null && EXT.equals("json");
boolean xml = EXT != null && EXT.equals("xml"); final boolean xml = EXT != null && EXT.equals("xml");
if (json) { if (json) {
// check for JSONP // check for JSONP
if ( post != null && post.containsKey("callback") ) { if ( post != null && post.containsKey("callback") ) {
@ -89,17 +91,17 @@ public final class seedlist {
// construct json property lists // construct json property lists
int count = 0; int count = 0;
for (int i = 0; i < v.size(); i++) { for (int i = 0; i < v.size(); i++) {
Seed seed = v.get(i); final Seed seed = v.get(i);
if (peername != null && !peername.equals(seed.getName())) continue; if (peername != null && !peername.equals(seed.getName())) continue;
Set<String> ips = seed.getIPs(); final Set<String> ips = seed.getIPs();
if (ips == null || ips.size() == 0) continue; if (ips == null || ips.size() == 0) continue;
prop.putJSON("peers_" + count + "_map_0_k", Seed.HASH); prop.putJSON("peers_" + count + "_map_0_k", Seed.HASH);
prop.put("peers_" + count + "_map_0_v", JSONObject.quote(seed.hash)); prop.put("peers_" + count + "_map_0_v", JSONObject.quote(seed.hash));
prop.put("peers_" + count + "_map_0_c", 1); prop.put("peers_" + count + "_map_0_c", 1);
Map<String, String> map = seed.getMap(); final Map<String, String> map = seed.getMap();
int c = 1; int c = 1;
if (!addressonly) { if (!addressonly) {
for (Map.Entry<String, String> m: map.entrySet()) { for (final Map.Entry<String, String> m: map.entrySet()) {
prop.putJSON("peers_" + count + "_map_" + c + "_k", m.getKey()); prop.putJSON("peers_" + count + "_map_" + c + "_k", m.getKey());
prop.put("peers_" + count + "_map_" + c + "_v", JSONObject.quote(m.getValue())); prop.put("peers_" + count + "_map_" + c + "_v", JSONObject.quote(m.getValue()));
prop.put("peers_" + count + "_map_" + c + "_c", 1); prop.put("peers_" + count + "_map_" + c + "_c", 1);
@ -107,9 +109,9 @@ public final class seedlist {
} }
} }
// construct a list of ips // construct a list of ips
StringBuilder a = new StringBuilder(); final StringBuilder a = new StringBuilder();
a.append('['); a.append('[');
for (String ip: ips) a.append(JSONObject.quote(seed.getPublicAddress(ip))).append(','); for (final String ip: ips) a.append(JSONObject.quote(seed.getPublicAddress(ip))).append(',');
a.setCharAt(a.length()-1, ']'); a.setCharAt(a.length()-1, ']');
prop.putJSON("peers_" + count + "_map_" + c + "_k", "Address"); prop.putJSON("peers_" + count + "_map_" + c + "_k", "Address");
prop.put("peers_" + count + "_map_" + c + "_v", a.toString()); prop.put("peers_" + count + "_map_" + c + "_v", a.toString());
@ -124,22 +126,22 @@ public final class seedlist {
} else if (xml) { } else if (xml) {
int count = 0; int count = 0;
for (int i = 0; i < v.size(); i++) { for (int i = 0; i < v.size(); i++) {
Seed seed = v.get(i); final Seed seed = v.get(i);
if (peername != null && !peername.equals(seed.getName())) continue; if (peername != null && !peername.equals(seed.getName())) continue;
Set<String> ips = seed.getIPs(); final Set<String> ips = seed.getIPs();
if (ips == null || ips.size() == 0) continue; if (ips == null || ips.size() == 0) continue;
prop.putXML("peers_" + count + "_map_0_k", Seed.HASH); prop.putXML("peers_" + count + "_map_0_k", Seed.HASH);
prop.putXML("peers_" + count + "_map_0_v", seed.hash); prop.putXML("peers_" + count + "_map_0_v", seed.hash);
Map<String, String> map = seed.getMap(); final Map<String, String> map = seed.getMap();
int c = 1; int c = 1;
if (!addressonly) { if (!addressonly) {
for (Map.Entry<String, String> m: map.entrySet()) { for (final Map.Entry<String, String> m: map.entrySet()) {
prop.putXML("peers_" + count + "_map_" + c + "_k", m.getKey()); prop.putXML("peers_" + count + "_map_" + c + "_k", m.getKey());
prop.putXML("peers_" + count + "_map_" + c + "_v", m.getValue()); prop.putXML("peers_" + count + "_map_" + c + "_v", m.getValue());
c++; c++;
} }
} }
for (String ip: ips) { for (final String ip: ips) {
prop.putXML("peers_" + count + "_map_" + c + "_k", "Address"); prop.putXML("peers_" + count + "_map_" + c + "_k", "Address");
prop.putXML("peers_" + count + "_map_" + c + "_v", seed.getPublicAddress(ip)); prop.putXML("peers_" + count + "_map_" + c + "_v", seed.getPublicAddress(ip));
c++; c++;
@ -147,17 +149,17 @@ public final class seedlist {
prop.put("peers_" + count + "_map", c); prop.put("peers_" + count + "_map", c);
count++; count++;
} }
prop.put("peers_" + (count - 1) + "_c", 0); prop.put("peers_" + (count - 1) + "_c", 0);
prop.put("peers", count); prop.put("peers", count);
} else { } else {
final StringBuilder encoded = new StringBuilder(1024); final StringBuilder encoded = new StringBuilder(1024);
for (Seed seed: v) { for (final Seed seed: v) {
encoded.append(seed.genSeedStr(null)).append(serverCore.CRLF_STRING); encoded.append(seed.genSeedStr(null)).append(serverCore.CRLF_STRING);
} }
prop.put("encoded", encoded.toString()); prop.put("encoded", encoded.toString());
} }
// return rewrite properties // return rewrite properties
return prop; return prop;
} }

@ -26,6 +26,7 @@
// You must compile this file with // You must compile this file with
// javac -classpath .:../classes transferRWI.java // javac -classpath .:../classes transferRWI.java
package net.yacy.htroot.yacy;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashSet; import java.util.HashSet;
@ -126,8 +127,8 @@ public final class transferRWI {
result = "ok"; result = "ok";
final StringBuilder unknownURLs = new StringBuilder(6000); final StringBuilder unknownURLs = new StringBuilder(6000);
double load = Memory.getSystemLoadAverage(); final double load = Memory.getSystemLoadAverage();
float maxload = sb.getConfigFloat(SwitchboardConstants.INDEX_DIST_LOADPREREQ, 2.0f); final float maxload = sb.getConfigFloat(SwitchboardConstants.INDEX_DIST_LOADPREREQ, 2.0f);
if (load > maxload) { if (load > maxload) {
// too high local load. this is bad but we must reject this to protect ourself! // too high local load. this is bad but we must reject this to protect ourself!
sb.getLog().info("Rejecting RWIs from peer " + otherPeerName + ", system has too high load = " + load + ", maxload = " + maxload); sb.getLog().info("Rejecting RWIs from peer " + otherPeerName + ", system has too high load = " + load + ", maxload = " + maxload);
@ -187,7 +188,7 @@ public final class transferRWI {
int received = 0; int received = 0;
int blocked = 0; int blocked = 0;
int count = 0; int count = 0;
Set<String> testids = new HashSet<String>(); final Set<String> testids = new HashSet<String>();
while (it.hasNext()) { while (it.hasNext()) {
estring = it.next(); estring = it.next();
count++; count++;
@ -231,7 +232,7 @@ public final class transferRWI {
testids.add(ASCII.String(urlHash)); testids.add(ASCII.String(urlHash));
received++; received++;
} }
for (String id: testids) { for (final String id: testids) {
try { try {
if (!sb.index.fulltext().exists(id)) { if (!sb.index.fulltext().exists(id)) {
unknownURL.put(ASCII.getBytes(id)); unknownURL.put(ASCII.getBytes(id));

@ -26,6 +26,8 @@
// You must compile this file with // You must compile this file with
// javac -classpath .:../classes transferRWI.java // javac -classpath .:../classes transferRWI.java
package net.yacy.htroot.yacy;
import java.io.IOException; import java.io.IOException;
import java.text.ParseException; import java.text.ParseException;
import java.util.HashMap; import java.util.HashMap;
@ -91,7 +93,7 @@ public final class transferURL {
// read the urls from the other properties and store // read the urls from the other properties and store
String urls; String urls;
URIMetadataNode lEntry; URIMetadataNode lEntry;
Map<String, URIMetadataNode> lEm = new HashMap<String, URIMetadataNode>(); final Map<String, URIMetadataNode> lEm = new HashMap<String, URIMetadataNode>();
for (int i = 0; i < urlc; i++) { for (int i = 0; i < urlc; i++) {
// read new lurl-entry // read new lurl-entry
@ -143,9 +145,9 @@ public final class transferURL {
lEm.put(ASCII.String(lEntry.hash()), lEntry); lEm.put(ASCII.String(lEntry.hash()), lEntry);
} }
doublecheck = 0; doublecheck = 0;
for (String id : lEm.keySet()) { for (final String id : lEm.keySet()) {
if (sb.index.exists(id)) { if (sb.index.exists(id)) {
doublecheck++; doublecheck++;
} else { } else {
@ -170,7 +172,7 @@ public final class transferURL {
Network.log.info("Received " + received + " URLs from peer " + otherPeerName + " in " + (System.currentTimeMillis() - start) + " ms, blocked " + blocked + " URLs"); Network.log.info("Received " + received + " URLs from peer " + otherPeerName + " in " + (System.currentTimeMillis() - start) + " ms, blocked " + blocked + " URLs");
EventChannel.channels(EventChannel.DHTRECEIVE).addMessage(new RSSMessage("Received " + received + ", blocked " + blocked + " URLs from peer " + otherPeerName, "", otherPeer.hash)); EventChannel.channels(EventChannel.DHTRECEIVE).addMessage(new RSSMessage("Received " + received + ", blocked " + blocked + " URLs from peer " + otherPeerName, "", otherPeer.hash));
if (sb.getConfigBool(SwitchboardConstants.DECORATION_AUDIO, false)) Audio.Soundclip.dhtin.play(-10.0f); if (sb.getConfigBool(SwitchboardConstants.DECORATION_AUDIO, false)) Audio.Soundclip.dhtin.play(-10.0f);
if (doublecheck > 0) { if (doublecheck > 0) {
Network.log.warn("Received " + doublecheck + "/" + urlc + " double URLs from peer " + otherPeerName); // double should not happen because we demanded only documents which we do not have yet Network.log.warn("Received " + doublecheck + "/" + urlc + " double URLs from peer " + otherPeerName); // double should not happen because we demanded only documents which we do not have yet
doublevalues = Integer.toString(doublecheck); doublevalues = Integer.toString(doublecheck);

@ -24,6 +24,8 @@
// along with this program; if not, write to the Free Software // along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package net.yacy.htroot.yacy;
import java.io.IOException; import java.io.IOException;
import net.yacy.cora.date.GenericFormatter; import net.yacy.cora.date.GenericFormatter;
@ -77,7 +79,7 @@ public class urls {
// find referrer, if there is one // find referrer, if there is one
try { try {
referrerURL = sb.getURL(entry.referrerhash()); referrerURL = sb.getURL(entry.referrerhash());
} catch (IOException e) { } catch (final IOException e) {
referrerURL = null; referrerURL = null;
ConcurrentLog.logException(e); ConcurrentLog.logException(e);
} }
@ -123,7 +125,7 @@ public class urls {
prop.put("item_" + c + "_pubDate", GenericFormatter.SHORT_SECOND_FORMATTER.format(entry.moddate())); prop.put("item_" + c + "_pubDate", GenericFormatter.SHORT_SECOND_FORMATTER.format(entry.moddate()));
prop.put("item_" + c + "_guid", ASCII.String(entry.hash())); prop.put("item_" + c + "_guid", ASCII.String(entry.hash()));
c++; c++;
} catch (IOException e) { } catch (final IOException e) {
ConcurrentLog.logException(e); ConcurrentLog.logException(e);
} }
} }

@ -778,9 +778,9 @@ public class YaCyDefaultServlet extends HttpServlet {
if (p < 0) { if (p < 0) {
return null; return null;
} }
target = target.substring(1, p); target = "net.yacy.htroot" + target.substring(0, p).replace('/', '.');
final Class<?> servletClass = Class.forName("net.yacy.htroot." + target); final Class<?> servletClass = Class.forName(target);
return servletClass; return servletClass;
} catch (final ClassNotFoundException e) { } catch (final ClassNotFoundException e) {

Loading…
Cancel
Save