boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

refactoring - moved htroot/yacy classes

Browse Source
pull/533/head
Michael Peter Christen 2 years ago
parent 48fcf3b3b5
commit 6d388bb7bf
15 changed files with 91 additions and 67 deletions
Show Stats Download Patch File Download Diff File

3
source/net/yacy/htroot/Wiki.java
Unescape View File

@ -1,4 +1,3 @@
package net.yacy.htroot;
// Wiki.java
// -----------------------
// part of the AnomicHTTPD caching proxy
@ -31,6 +30,8 @@ package net.yacy.htroot;
// javac -classpath .:../classes Wiki.java
// if the shell's current path is HTROOT
package net.yacy.htroot;
import java.io.IOException;
import java.util.Arrays;
import java.util.Date;

4
htroot/api/trail_p.java → source/net/yacy/htroot/api/trail_p.java
Unescape View File

@ -22,6 +22,8 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package net.yacy.htroot.api;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.search.Switchboard;
import net.yacy.server.serverObjects;
@ -34,7 +36,7 @@ public class trail_p {
final serverObjects prop = new serverObjects();
int c = 0;
for (String t: sb.trail) {
for (final String t: sb.trail) {
prop.put("trails_" + c++ + "_trail", t); // don't put in putHTML or putXML in, this is wrong!
}
prop.put("trails", c);

2
htroot/yacy/crawlReceipt.java → source/net/yacy/htroot/yacy/crawlReceipt.java
Unescape View File

@ -26,8 +26,10 @@
// You must compile this file with
// javac -classpath .:../classes crawlOrder.java
package net.yacy.htroot.yacy;
import java.io.IOException;
import net.yacy.cora.document.encoding.ASCII;
import net.yacy.cora.federate.solr.FailCategory;
import net.yacy.cora.protocol.RequestHeader;

8
htroot/yacy/hello.java → source/net/yacy/htroot/yacy/hello.java
Unescape View File

@ -27,6 +27,8 @@
// javac -classpath .:../../classes hello.java
// if the shell's current path is HTROOT
package net.yacy.htroot.yacy;
import java.io.IOException;
import java.net.InetAddress;
import java.net.MalformedURLException;
@ -38,8 +40,8 @@ import net.yacy.cora.document.id.MultiProtocolURL;
import net.yacy.cora.protocol.Domains;
import net.yacy.cora.protocol.HeaderFramework;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.peers.Network;
import net.yacy.peers.DHTSelection;
import net.yacy.peers.Network;
import net.yacy.peers.Protocol;
import net.yacy.peers.Seed;
import net.yacy.peers.graphics.ProfilingGraph;
@ -162,8 +164,8 @@ public final class hello {
int callbackRemain = Math.min(5, reportedips.size());
final long callbackStart = System.currentTimeMillis();
if (callbackRemain > 0 && reportedips.size() > 0) {
for (String reportedip: reportedips) {
int partialtimeout = ((int) (callbackStart + totalTimeout - System.currentTimeMillis())) / callbackRemain; // bad hack until a concurrent version is implemented
for (final String reportedip: reportedips) {
final int partialtimeout = ((int) (callbackStart + totalTimeout - System.currentTimeMillis())) / callbackRemain; // bad hack until a concurrent version is implemented
if (partialtimeout <= 0) break;
//ConcurrentLog.info("**hello-DEBUG**", "reportedip = " + reportedip + " is handled");
if (Seed.isProperIP(reportedip)) {

2
htroot/yacy/idx.java → source/net/yacy/htroot/yacy/idx.java
Unescape View File

@ -22,6 +22,8 @@
* If not, see <http://www.gnu.org/licenses/>.
*/
package net.yacy.htroot.yacy;
import java.util.Iterator;
import net.yacy.cora.document.encoding.ASCII;

4
htroot/yacy/list.java → source/net/yacy/htroot/yacy/list.java
Unescape View File

@ -25,21 +25,21 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package net.yacy.htroot.yacy;
import java.io.File;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.cora.util.CommonPattern;
import net.yacy.kelondro.util.FileUtils;
import net.yacy.peers.Seed;
import net.yacy.peers.Protocol;
import net.yacy.peers.Seed;
import net.yacy.search.Switchboard;
import net.yacy.search.SwitchboardConstants;
import net.yacy.server.serverCore;
import net.yacy.server.serverObjects;
import net.yacy.server.serverSwitch;
public final class list {
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {

6
htroot/yacy/message.java → source/net/yacy/htroot/yacy/message.java
Unescape View File

@ -27,12 +27,16 @@
// javac -classpath .:../../classes message.java
// if the shell's current path is HTROOT/yacy
package net.yacy.htroot.yacy;
import java.io.File;
import java.io.IOException;
import java.io.PrintWriter;
import java.net.InetAddress;
import java.util.Date;
import com.google.common.io.Files;
import net.yacy.cora.date.GenericFormatter;
import net.yacy.cora.document.encoding.UTF8;
import net.yacy.cora.protocol.Domains;
@ -49,8 +53,6 @@ import net.yacy.server.serverObjects;
import net.yacy.server.serverSwitch;
import net.yacy.utils.crypt;
import com.google.common.io.Files;
public final class message {

2
htroot/yacy/profile.java → source/net/yacy/htroot/yacy/profile.java
Unescape View File

@ -29,6 +29,8 @@
// javac -classpath .:../../Classes hello.java
// if the shell's current path is HTROOT
package net.yacy.htroot.yacy;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;

2
htroot/yacy/query.java → source/net/yacy/htroot/yacy/query.java
Unescape View File

@ -27,6 +27,8 @@
// javac -classpath .:../../classes query.java
// if the shell's current path is HTROOT
package net.yacy.htroot.yacy;
import java.io.IOException;
import net.yacy.cora.date.GenericFormatter;

12
htroot/yacy/search.java → source/net/yacy/htroot/yacy/search.java
Unescape View File

@ -28,6 +28,8 @@
// javac -classpath .:../../Classes search.java
// if the shell's current path is htroot/yacy
package net.yacy.htroot.yacy;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
@ -119,7 +121,7 @@ public final class search {
final boolean strictContentDom = post.getBoolean("strictContentDom");
final String filter = post.get("filter", ".*"); // a filter on the url
final int timezoneOffset = post.getInt("timezoneOffset", 0);
QueryModifier modifier = new QueryModifier(timezoneOffset);
final QueryModifier modifier = new QueryModifier(timezoneOffset);
modifier.sitehost = post.get("sitehost", ""); if (modifier.sitehost.isEmpty()) modifier.sitehost = null;
modifier.sitehash = post.get("sitehash", ""); if (modifier.sitehash.isEmpty()) modifier.sitehash = null;
modifier.author = post.get("author", ""); if (modifier.author.isEmpty()) modifier.author = null;
@ -225,7 +227,7 @@ public final class search {
if (query.isEmpty() && abstractSet != null) {
// this is _not_ a normal search, only a request for index abstracts
final Segment indexSegment = sb.index;
QueryGoal qg = new QueryGoal(abstractSet, new RowHandleSet(WordReferenceRow.urlEntryRow.primaryKeyLength, WordReferenceRow.urlEntryRow.objectOrder, 0));
final QueryGoal qg = new QueryGoal(abstractSet, new RowHandleSet(WordReferenceRow.urlEntryRow.primaryKeyLength, WordReferenceRow.urlEntryRow.objectOrder, 0));
theQuery = new QueryParams(
qg,
modifier,
@ -287,10 +289,10 @@ public final class search {
} else {
// retrieve index containers from search request
RowHandleSet allHashes = new RowHandleSet(WordReferenceRow.urlEntryRow.primaryKeyLength, WordReferenceRow.urlEntryRow.objectOrder, 0);
final RowHandleSet allHashes = new RowHandleSet(WordReferenceRow.urlEntryRow.primaryKeyLength, WordReferenceRow.urlEntryRow.objectOrder, 0);
try {allHashes.putAll(queryhashes);} catch (final SpaceExceededException e) {}
try {allHashes.putAll(excludehashes);} catch (final SpaceExceededException e) {}
QueryGoal qg = new QueryGoal(queryhashes, excludehashes);
final QueryGoal qg = new QueryGoal(queryhashes, excludehashes);
theQuery = new QueryParams(
qg,
modifier,
@ -400,7 +402,7 @@ public final class search {
prop.put("indexabstract", indexabstract.toString());
// prepare result
int resultCount = theSearch == null ? 0 : theSearch.getResultCount(); // theSearch may be null if we searched only for abstracts
final int resultCount = theSearch == null ? 0 : theSearch.getResultCount(); // theSearch may be null if we searched only for abstracts
if (resultCount == 0 || accu == null || accu.isEmpty()) {
// no results

48
htroot/yacy/seedlist.java → source/net/yacy/htroot/yacy/seedlist.java
Unescape View File

@ -19,6 +19,8 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package net.yacy.htroot.yacy;
import java.util.ArrayList;
import java.util.Map;
import java.util.Set;
@ -49,23 +51,23 @@ public final class seedlist {
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
// return variable that accumulates replacements
final Switchboard sb = (Switchboard) env;
int maxcount = Math.min(LISTMAX, post == null ? Integer.MAX_VALUE : post.getInt("maxcount", Integer.MAX_VALUE));
float minversion = Math.min(LISTMAX, post == null ? 0.0f : post.getFloat("minversion", 0.0f));
boolean nodeonly = post == null || !post.containsKey("node") ? false : post.getBoolean("node");
boolean includeme = post == null || !post.containsKey("me") ? true : post.getBoolean("me");
boolean addressonly = post == null || !post.containsKey("address") ? false : post.getBoolean("address");
String peername = post == null ? null : post.containsKey("my") ? sb.peers.myName() : post.get("peername");
final int maxcount = Math.min(LISTMAX, post == null ? Integer.MAX_VALUE : post.getInt("maxcount", Integer.MAX_VALUE));
final float minversion = Math.min(LISTMAX, post == null ? 0.0f : post.getFloat("minversion", 0.0f));
final boolean nodeonly = post == null || !post.containsKey("node") ? false : post.getBoolean("node");
final boolean includeme = post == null || !post.containsKey("me") ? true : post.getBoolean("me");
final boolean addressonly = post == null || !post.containsKey("address") ? false : post.getBoolean("address");
final String peername = post == null ? null : post.containsKey("my") ? sb.peers.myName() : post.get("peername");
final ArrayList<Seed> v;
if (post != null && post.containsKey("my")) {
v = new ArrayList<Seed>(1);
v.add(sb.peers.mySeed());
} else if (post != null && post.containsKey("id")) {
v = new ArrayList<Seed>(1);
Seed s = sb.peers.get(post.get("id"));
final Seed s = sb.peers.get(post.get("id"));
if (s != null) v.add(s);
} else if (post != null && post.containsKey("name")) {
v = new ArrayList<Seed>(1);
Seed s = sb.peers.lookupByName(post.get("name"));
final Seed s = sb.peers.lookupByName(post.get("name"));
if (s != null) v.add(s);
} else {
v= sb.peers.getSeedlist(maxcount, includeme, nodeonly, minversion);
@ -73,9 +75,9 @@ public final class seedlist {
final serverObjects prop = new serverObjects();
// write simple-encoded seed lines or json
String EXT = header.get(HeaderFramework.CONNECTION_PROP_EXT);
boolean json = EXT != null && EXT.equals("json");
boolean xml = EXT != null && EXT.equals("xml");
final String EXT = header.get(HeaderFramework.CONNECTION_PROP_EXT);
final boolean json = EXT != null && EXT.equals("json");
final boolean xml = EXT != null && EXT.equals("xml");
if (json) {
// check for JSONP
@ -89,17 +91,17 @@ public final class seedlist {
// construct json property lists
int count = 0;
for (int i = 0; i < v.size(); i++) {
Seed seed = v.get(i);
final Seed seed = v.get(i);
if (peername != null && !peername.equals(seed.getName())) continue;
Set<String> ips = seed.getIPs();
final Set<String> ips = seed.getIPs();
if (ips == null || ips.size() == 0) continue;
prop.putJSON("peers_" + count + "_map_0_k", Seed.HASH);
prop.put("peers_" + count + "_map_0_v", JSONObject.quote(seed.hash));
prop.put("peers_" + count + "_map_0_c", 1);
Map<String, String> map = seed.getMap();
final Map<String, String> map = seed.getMap();
int c = 1;
if (!addressonly) {
for (Map.Entry<String, String> m: map.entrySet()) {
for (final Map.Entry<String, String> m: map.entrySet()) {
prop.putJSON("peers_" + count + "_map_" + c + "_k", m.getKey());
prop.put("peers_" + count + "_map_" + c + "_v", JSONObject.quote(m.getValue()));
prop.put("peers_" + count + "_map_" + c + "_c", 1);
@ -107,9 +109,9 @@ public final class seedlist {
}
}
// construct a list of ips
StringBuilder a = new StringBuilder();
final StringBuilder a = new StringBuilder();
a.append('[');
for (String ip: ips) a.append(JSONObject.quote(seed.getPublicAddress(ip))).append(',');
for (final String ip: ips) a.append(JSONObject.quote(seed.getPublicAddress(ip))).append(',');
a.setCharAt(a.length()-1, ']');
prop.putJSON("peers_" + count + "_map_" + c + "_k", "Address");
prop.put("peers_" + count + "_map_" + c + "_v", a.toString());
@ -124,22 +126,22 @@ public final class seedlist {
} else if (xml) {
int count = 0;
for (int i = 0; i < v.size(); i++) {
Seed seed = v.get(i);
final Seed seed = v.get(i);
if (peername != null && !peername.equals(seed.getName())) continue;
Set<String> ips = seed.getIPs();
final Set<String> ips = seed.getIPs();
if (ips == null || ips.size() == 0) continue;
prop.putXML("peers_" + count + "_map_0_k", Seed.HASH);
prop.putXML("peers_" + count + "_map_0_v", seed.hash);
Map<String, String> map = seed.getMap();
final Map<String, String> map = seed.getMap();
int c = 1;
if (!addressonly) {
for (Map.Entry<String, String> m: map.entrySet()) {
for (final Map.Entry<String, String> m: map.entrySet()) {
prop.putXML("peers_" + count + "_map_" + c + "_k", m.getKey());
prop.putXML("peers_" + count + "_map_" + c + "_v", m.getValue());
c++;
}
}
for (String ip: ips) {
for (final String ip: ips) {
prop.putXML("peers_" + count + "_map_" + c + "_k", "Address");
prop.putXML("peers_" + count + "_map_" + c + "_v", seed.getPublicAddress(ip));
c++;
@ -152,7 +154,7 @@ public final class seedlist {
prop.put("peers", count);
} else {
final StringBuilder encoded = new StringBuilder(1024);
for (Seed seed: v) {
for (final Seed seed: v) {
encoded.append(seed.genSeedStr(null)).append(serverCore.CRLF_STRING);
}
prop.put("encoded", encoded.toString());

9
htroot/yacy/transferRWI.java → source/net/yacy/htroot/yacy/transferRWI.java
Unescape View File

@ -26,6 +26,7 @@
// You must compile this file with
// javac -classpath .:../classes transferRWI.java
package net.yacy.htroot.yacy;
import java.util.ArrayList;
import java.util.HashSet;
@ -126,8 +127,8 @@ public final class transferRWI {
result = "ok";
final StringBuilder unknownURLs = new StringBuilder(6000);
double load = Memory.getSystemLoadAverage();
float maxload = sb.getConfigFloat(SwitchboardConstants.INDEX_DIST_LOADPREREQ, 2.0f);
final double load = Memory.getSystemLoadAverage();
final float maxload = sb.getConfigFloat(SwitchboardConstants.INDEX_DIST_LOADPREREQ, 2.0f);
if (load > maxload) {
// too high local load. this is bad but we must reject this to protect ourself!
sb.getLog().info("Rejecting RWIs from peer " + otherPeerName + ", system has too high load = " + load + ", maxload = " + maxload);
@ -187,7 +188,7 @@ public final class transferRWI {
int received = 0;
int blocked = 0;
int count = 0;
Set<String> testids = new HashSet<String>();
final Set<String> testids = new HashSet<String>();
while (it.hasNext()) {
estring = it.next();
count++;
@ -231,7 +232,7 @@ public final class transferRWI {
testids.add(ASCII.String(urlHash));
received++;
}
for (String id: testids) {
for (final String id: testids) {
try {
if (!sb.index.fulltext().exists(id)) {
unknownURL.put(ASCII.getBytes(id));

6
htroot/yacy/transferURL.java → source/net/yacy/htroot/yacy/transferURL.java
Unescape View File

@ -26,6 +26,8 @@
// You must compile this file with
// javac -classpath .:../classes transferRWI.java
package net.yacy.htroot.yacy;
import java.io.IOException;
import java.text.ParseException;
import java.util.HashMap;
@ -91,7 +93,7 @@ public final class transferURL {
// read the urls from the other properties and store
String urls;
URIMetadataNode lEntry;
Map<String, URIMetadataNode> lEm = new HashMap<String, URIMetadataNode>();
final Map<String, URIMetadataNode> lEm = new HashMap<String, URIMetadataNode>();
for (int i = 0; i < urlc; i++) {
// read new lurl-entry
@ -145,7 +147,7 @@ public final class transferURL {
}
doublecheck = 0;
for (String id : lEm.keySet()) {
for (final String id : lEm.keySet()) {
if (sb.index.exists(id)) {
doublecheck++;
} else {

6
htroot/yacy/urls.java → source/net/yacy/htroot/yacy/urls.java
Unescape View File

@ -24,6 +24,8 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package net.yacy.htroot.yacy;
import java.io.IOException;
import net.yacy.cora.date.GenericFormatter;
@ -77,7 +79,7 @@ public class urls {
// find referrer, if there is one
try {
referrerURL = sb.getURL(entry.referrerhash());
} catch (IOException e) {
} catch (final IOException e) {
referrerURL = null;
ConcurrentLog.logException(e);
}
@ -123,7 +125,7 @@ public class urls {
prop.put("item_" + c + "_pubDate", GenericFormatter.SHORT_SECOND_FORMATTER.format(entry.moddate()));
prop.put("item_" + c + "_guid", ASCII.String(entry.hash()));
c++;
} catch (IOException e) {
} catch (final IOException e) {
ConcurrentLog.logException(e);
}
}

4
source/net/yacy/http/servlets/YaCyDefaultServlet.java
Unescape View File

@ -778,9 +778,9 @@ public class YaCyDefaultServlet extends HttpServlet {
if (p < 0) {
return null;
}
target = target.substring(1, p);
target = "net.yacy.htroot" + target.substring(0, p).replace('/', '.');
final Class<?> servletClass = Class.forName("net.yacy.htroot." + target);
final Class<?> servletClass = Class.forName(target);
return servletClass;
} catch (final ClassNotFoundException e) {

Write Preview
Loading…
Cancel
Save