boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

get the peer location more quickly

Browse Source
pull/1/head
Michael Peter Christen 13 years ago
parent 156d457aec
commit 597bb76e4f
10 changed files with 44 additions and 65 deletions
Show Stats Download Patch File Download Diff File

16
htroot/Network.java
Unescape View File

@ -30,6 +30,7 @@
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Locale;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
@ -38,6 +39,7 @@ import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;
import net.yacy.cora.protocol.ClientIdentification;
import net.yacy.cora.protocol.Domains;
import net.yacy.cora.protocol.HeaderFramework;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.kelondro.util.MapTools;
@ -288,7 +290,7 @@ public class Network {
String startURL;
Map<String, String> wikiMap;
Map<String, String> blogMap;
String userAgent, location = "";
String userAgent, location;
int PPM;
double QPM;
Pattern peerSearchPattern = null;
@ -367,11 +369,13 @@ public class Network {
userAgent = ClientIdentification.getUserAgent();
location = ClientIdentification.generateLocation();
} else {
userAgent = sb.peers.peerActions.getUserAgent(seed.getIP());
String loc = ClientIdentification.parseLocationInUserAgent(userAgent);
if (loc.length() >= 10) {
location = ClientIdentification.parseLocationInUserAgent(userAgent).substring(0, 10);
}
userAgent = sb.peers.peerActions.getUserAgent(seed.getIP());
location = ClientIdentification.parseLocationInUserAgent(userAgent);
}
if (location.length() > 10) location = location.substring(0, 10);
if (location.length() == 0) {
Locale l = Domains.getLocale(seed.getIP());
if (l != null) location = l.toString();
}
prop.putHTML(STR_TABLE_LIST + conCount + "_location", location);
if (complete) {

6
htroot/gsa/searchresult.java
Unescape View File

@ -24,6 +24,7 @@ import java.io.OutputStreamWriter;
import java.io.Writer;
import net.yacy.cora.document.UTF8;
import net.yacy.cora.protocol.HeaderFramework;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.cora.services.federated.solr.GSAResponseWriter;
import net.yacy.kelondro.logging.Log;
@ -74,6 +75,11 @@ public class searchresult {
// this uses the methods in the jetty servlet environment and can be removed if jetty in implemented
Switchboard sb = (Switchboard) env;
// remember the peer contact for peer statistics
final String clientip = header.get(HeaderFramework.CONNECTION_PROP_CLIENTIP, "<unknown>"); // read an artificial header addendum
final String userAgent = header.get(HeaderFramework.USER_AGENT, "<unknown>");
sb.peers.peerActions.setUserAgent(clientip, userAgent);
// check if user is allowed to search (can be switched in /ConfigPortal.html)
final boolean searchAllowed = sb.getConfigBool("publicSearchpage", true) || sb.verifyAuthentication(header);
if (!searchAllowed) return null;

6
htroot/solr/select.java
Unescape View File

@ -28,6 +28,7 @@ import java.util.Map;
import javax.servlet.ServletException;
import net.yacy.cora.document.UTF8;
import net.yacy.cora.protocol.HeaderFramework;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.cora.services.federated.solr.EnhancedXMLResponseWriter;
import net.yacy.cora.services.federated.solr.OpensearchResponseWriter;
@ -114,6 +115,11 @@ public class select {
// this uses the methods in the jetty servlet environment and can be removed if jetty in implemented
Switchboard sb = (Switchboard) env;
// remember the peer contact for peer statistics
final String clientip = header.get(HeaderFramework.CONNECTION_PROP_CLIENTIP, "<unknown>"); // read an artificial header addendum
final String userAgent = header.get(HeaderFramework.USER_AGENT, "<unknown>");
sb.peers.peerActions.setUserAgent(clientip, userAgent);
// check if user is allowed to search (can be switched in /ConfigPortal.html)
final boolean searchAllowed = sb.getConfigBool("publicSearchpage", true) || sb.verifyAuthentication(header);
if (!searchAllowed) return null;

6
htroot/yacy/hello.java
Unescape View File

@ -110,6 +110,7 @@ public final class hello {
// we easily know the caller's IP:
final String userAgent = header.get(HeaderFramework.USER_AGENT, "<unknown>");
sb.peers.peerActions.setUserAgent(clientip, userAgent);
final String reportedip = remoteSeed.getIP();
final String reportedPeerType = remoteSeed.get(Seed.PEERTYPE, Seed.PEERTYPE_JUNIOR);
final double clientversion = remoteSeed.getVersion();
@ -126,7 +127,6 @@ public final class hello {
// if the remote client has reported its own IP address and the client supports
// the port forwarding feature (if client version >= 0.383) then we try to
// connect to the reported IP address first
time = System.currentTimeMillis();
long time_backping = 0;
String backping_method = "none";
if (reportedip.length() > 0 &&
@ -138,6 +138,7 @@ public final class hello {
// try first the reportedip, since this may be a connect from a port-forwarding host
prop.put("yourip", reportedip);
remoteSeed.setIP(reportedip);
time = System.currentTimeMillis();
callback = Protocol.queryUrlCount(remoteSeed);
time_backping = System.currentTimeMillis() - time;
backping_method = "reportedip=" + reportedip;
@ -161,6 +162,7 @@ public final class hello {
prop.put("yourip", clientip);
remoteSeed.setIP(clientip);
time = System.currentTimeMillis();
callback = Protocol.queryUrlCount(remoteSeed);
time_backping = System.currentTimeMillis() - time;
backping_method = "clientip=" + clientip;
@ -199,8 +201,6 @@ public final class hello {
// update event tracker
EventTracker.update(EventTracker.EClass.PEERPING, new ProfilingGraph.EventPing(remoteSeed.getName(), sb.peers.myName(), false, connectedAfter - connectedBefore), false);
sb.peers.peerActions.setUserAgent(clientip, userAgent);
if (!(prop.get(Seed.YOURTYPE)).equals(reportedPeerType)) {
Network.log.logInfo("hello/server: changing remote peer '" + remoteSeed.getName() +
"' [" + reportedip +

7
htroot/yacy/query.java
Unescape View File

@ -49,8 +49,13 @@ public final class query {
// return variable that accumulates replacements
final Switchboard sb = (Switchboard) ss;
final serverObjects prop = new serverObjects();
// remember the peer contact for peer statistics
final String clientip = header.get(HeaderFramework.CONNECTION_PROP_CLIENTIP, "<unknown>"); // read an artificial header addendum
final String userAgent = header.get(HeaderFramework.USER_AGENT, "<unknown>");
sb.peers.peerActions.setUserAgent(clientip, userAgent);
final serverObjects prop = new serverObjects();
prop.put("magic", Network.magic);
if ((post == null) || (ss == null) || !Protocol.authentifyRequest(post, ss)) {

7
htroot/yacy/transferRWI.java
Unescape View File

@ -33,6 +33,7 @@ import java.util.Iterator;
import net.yacy.cora.document.ASCII;
import net.yacy.cora.document.RSSMessage;
import net.yacy.cora.document.UTF8;
import net.yacy.cora.protocol.HeaderFramework;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.cora.storage.HandleSet;
import net.yacy.kelondro.data.meta.URIMetadataRow;
@ -60,6 +61,12 @@ public final class transferRWI {
// return variable that accumulates replacements
final Switchboard sb = (Switchboard) env;
// remember the peer contact for peer statistics
final String clientip = header.get(HeaderFramework.CONNECTION_PROP_CLIENTIP, "<unknown>"); // read an artificial header addendum
final String userAgent = header.get(HeaderFramework.USER_AGENT, "<unknown>");
sb.peers.peerActions.setUserAgent(clientip, userAgent);
final serverObjects prop = new serverObjects();
final String contentType = header.getContentType();
if ((post == null) || (env == null)) {

4
source/net/yacy/cora/protocol/Domains.java
Unescape View File

@ -986,13 +986,15 @@ public class Domains {
if (host == null) return null;
final Locale locale = getLocale(dnsResolve(host));
if (locale != null && locale.getCountry() != null && locale.getCountry().length() > 0) return locale;
return null;
/*
final int p = host.lastIndexOf('.');
if (p < 0) return null;
String tld = host.substring(p + 1).toUpperCase();
if (tld.length() < 2) return null;
if (tld.length() > 2) tld = "US";
return new Locale("en", tld);
*/
}
/**

3
source/net/yacy/peers/Network.java
Unescape View File

@ -391,8 +391,7 @@ public class Network
attempts = PING_INITIAL;
}
final Map<byte[], String> ch = Switchboard.getSwitchboard().clusterhashes;
seeds =
PeerSelection.seedsByAge(this.sb.peers, true, attempts - ((ch == null) ? 0 : ch.size())); // best for fast connection
seeds = PeerSelection.seedsByAge(this.sb.peers, true, attempts - ((ch == null) ? 0 : ch.size())); // best for fast connection
// add also all peers from cluster if this is a public robinson cluster
if ( ch != null ) {
final Iterator<Map.Entry<byte[], String>> i = ch.entrySet().iterator();

4
source/net/yacy/peers/PeerActions.java
Unescape View File

@ -25,10 +25,10 @@
package net.yacy.peers;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import net.yacy.cora.document.ASCII;
import net.yacy.cora.document.RSSMessage;
import net.yacy.cora.storage.ConcurrentARC;
import net.yacy.kelondro.util.MapTools;
import net.yacy.peers.operation.yacyVersion;
@ -43,7 +43,7 @@ public class PeerActions {
public PeerActions(final SeedDB seedDB, final NewsPool newsPool) {
this.seedDB = seedDB;
this.newsPool = newsPool;
this.userAgents = new ConcurrentHashMap<String, String>();
this.userAgents = new ConcurrentARC<String, String>(10000, Runtime.getRuntime().availableProcessors() + 1);
this.disconnects = 0;
}

50
source/net/yacy/peers/dht/PeerSelection.java
Unescape View File

@ -33,10 +33,7 @@ import java.util.Map;
import java.util.Set;
import java.util.SortedMap;
import net.yacy.cora.date.AbstractFormatter;
import net.yacy.cora.document.ASCII;
import net.yacy.cora.sorting.ConcurrentScoreMap;
import net.yacy.cora.sorting.ScoreMap;
import net.yacy.cora.storage.HandleSet;
import net.yacy.cora.util.SpaceExceededException;
import net.yacy.kelondro.data.word.Word;
@ -433,53 +430,6 @@ public class PeerSelection {
}
/**
* get either the youngest or oldest peers from the seed db. Count as many as requested
* @param seedDB
* @param up if up = true then get the most recent peers, if up = false then get oldest
* @param count number of wanted peers
* @return a hash map of peer hashes to seed object
*/
public static Map<String, Seed> seedsByAgeX(final SeedDB seedDB, final boolean up, int count) {
if (count > seedDB.sizeConnected()) count = seedDB.sizeConnected();
// fill a score object
final ScoreMap<String> seedScore = new ConcurrentScoreMap<String>();
Seed ys;
long absage;
final Iterator<Seed> s = seedDB.seedsSortedConnected(!up, Seed.LASTSEEN);
int searchcount = 1000;
if (searchcount > seedDB.sizeConnected()) searchcount = seedDB.sizeConnected();
try {
while ((s.hasNext()) && (searchcount-- > 0)) {
ys = s.next();
if ((ys != null) && (ys.get(Seed.LASTSEEN, "").length() > 10)) try {
absage = Math.abs(System.currentTimeMillis() + AbstractFormatter.dayMillis - ys.getLastSeenUTC()) / 1000 / 60;
if (absage > Integer.MAX_VALUE) absage = Integer.MAX_VALUE;
seedScore.inc(ys.hash, (int) absage); // the higher absage, the older is the peer
} catch (final Exception e) {}
}
// result is now in the score object; create a result vector
final Map<String, Seed> result = new HashMap<String, Seed>();
final Iterator<String> it = seedScore.keys(up);
int c = 0;
while ((c < count) && (it.hasNext())) {
c++;
ys = seedDB.getConnected(it.next());
if ((ys != null) && (ys.hash != null)) result.put(ys.hash, ys);
}
return result;
} catch (final kelondroException e) {
Network.log.logSevere("Internal Error at yacySeedDB.seedsByAge: " + e.getMessage(), e);
return null;
}
}
/**
* get either the youngest or oldest peers from the seed db. Count as many as requested
* @param seedDB

Write Preview
Loading…
Cancel
Save