fix for remote search process

pull/1/head
Michael Peter Christen 11 years ago
parent 0843b12ef3
commit 97995a1dd9

@ -123,7 +123,7 @@ public class ConcurrentUpdateSolrConnector implements SolrConnector {
@Override @Override
public int bufferSize() { public int bufferSize() {
return this.updateCapacity; return this.docBuffer.size();
} }
@Override @Override

@ -503,31 +503,34 @@ public final class Protocol {
final long timestamp = System.currentTimeMillis(); final long timestamp = System.currentTimeMillis();
event.addExpectedRemoteReferences(count); event.addExpectedRemoteReferences(count);
SearchResult result; SearchResult result = null;
String clusteraddress = target.getPublicAddress(target.getIP()); for (String ip: target.getIPs()) {
if (target.clash(event.peers.mySeed().getIPs())) clusteraddress = "localhost:" + event.peers.mySeed().getPort(); if (ip.indexOf(':') >= 0) System.out.println("Search target: IPv6: " + ip);
try { String clusteraddress = target.getPublicAddress(ip);
result = if (target.clash(event.peers.mySeed().getIPs())) clusteraddress = "localhost:" + event.peers.mySeed().getPort();
new SearchResult( try {
event, result =
basicRequestParts(Switchboard.getSwitchboard(), target.hash, crypt.randomSalt()), new SearchResult(
wordhashes, event,
excludehashes, basicRequestParts(Switchboard.getSwitchboard(), target.hash, crypt.randomSalt()),
"", wordhashes,
language, excludehashes,
contentdom, "",
count, language,
time, contentdom,
maxDistance, count,
partitions, time,
target.getHexHash() + ".yacyh", maxDistance,
clusteraddress, partitions,
secondarySearchSuperviser target.getHexHash() + ".yacyh",
); clusteraddress,
} catch (final IOException e ) { secondarySearchSuperviser
Network.log.info("SEARCH failed, Peer: " + target.hash + ":" + target.getName() + " (" + e.getMessage() + ")"); );
event.peers.peerActions.peerDeparture(target, "search request to peer created io exception: " + e.getMessage()); } catch (final IOException e ) {
return -1; Network.log.info("SEARCH failed, Peer: " + target.hash + ":" + target.getName() + " (" + e.getMessage() + ")");
event.peers.peerActions.interfaceDeparture(target, ip);
return -1;
}
} }
// computation time // computation time
final long totalrequesttime = System.currentTimeMillis() - timestamp; final long totalrequesttime = System.currentTimeMillis() - timestamp;

@ -174,7 +174,6 @@ public class RemoteSearch extends Thread {
dhtPeers = DHTSelection.selectClusterPeers(event.peers, clusterselection); dhtPeers = DHTSelection.selectClusterPeers(event.peers, clusterselection);
} else { } else {
if (event.query.getQueryGoal().isCatchall() || event.query.getQueryGoal().getIncludeHashes().has(Segment.catchallHash)) { if (event.query.getQueryGoal().isCatchall() || event.query.getQueryGoal().getIncludeHashes().has(Segment.catchallHash)) {
if (event.query.modifier.sitehost != null && event.query.modifier.sitehost.length() > 0) { if (event.query.modifier.sitehost != null && event.query.modifier.sitehost.length() > 0) {
// select peers according to host name, not the query goal // select peers according to host name, not the query goal
String newGoal = Domains.getSmartSLD(event.query.modifier.sitehost); String newGoal = Domains.getSmartSLD(event.query.modifier.sitehost);
@ -186,29 +185,29 @@ public class RemoteSearch extends Thread {
random); random);
} else { } else {
// select just random peers // select just random peers
dhtPeers = DHTSelection.seedsByAge(event.peers, false, count).values(); dhtPeers = DHTSelection.seedsByAge(event.peers, false, event.peers.redundancy()).values();
}
} else {
dhtPeers = DHTSelection.selectDHTSearchTargets(
event.peers,
event.query.getQueryGoal().getIncludeHashes(),
minage,
redundancy, event.peers.redundancy(),
random);
// this set of peers may be too large and consume too many threads if more than one word is searched.
// to prevent overloading, we do a subset collection based on random to prevent the death of the own peer
// and to do a distributed load-balancing on the target peers
long targetSize = 1 + redundancy * event.peers.scheme.verticalPartitions(); // this is the maximum for one word plus one
if (dhtPeers.size() > targetSize) {
ArrayList<Seed> pa = new ArrayList<Seed>(dhtPeers.size());
pa.addAll(dhtPeers);
dhtPeers.clear();
for (int i = 0; i < targetSize; i++) dhtPeers.add(pa.remove(random.nextInt(pa.size())));
} }
}
dhtPeers = DHTSelection.selectDHTSearchTargets(
event.peers,
event.query.getQueryGoal().getIncludeHashes(),
minage,
redundancy, event.peers.redundancy(),
random);
// this set of peers may be too large and consume too many threads if more than one word is searched.
// to prevent overloading, we do a subset collection based on random to prevent the death of the own peer
// and to do a distributed load-balancing on the target peers
long targetSize = 1 + redundancy * event.peers.scheme.verticalPartitions(); // this is the maximum for one word plus one
if (dhtPeers.size() > targetSize) {
ArrayList<Seed> pa = new ArrayList<Seed>(dhtPeers.size());
pa.addAll(dhtPeers);
dhtPeers.clear();
for (int i = 0; i < targetSize; i++) dhtPeers.add(pa.remove(random.nextInt(pa.size())));
} }
} }
if (dhtPeers == null) dhtPeers = new HashSet<Seed>(); if (dhtPeers == null) dhtPeers = new HashSet<Seed>();
// select node targets // select node targets
final Collection<Seed> robinsonPeers = DHTSelection.selectExtraTargets(event.peers, event.query.getQueryGoal().getIncludeHashes(), minage, dhtPeers, robinsoncount, random); final Collection<Seed> robinsonPeers = DHTSelection.selectExtraTargets(event.peers, event.query.getQueryGoal().getIncludeHashes(), minage, dhtPeers, robinsoncount, random);

@ -140,9 +140,7 @@ public class serverAccessTracker {
track.add(new Track(System.currentTimeMillis(), accessPath)); track.add(new Track(System.currentTimeMillis(), accessPath));
clearTooOldAccess(track); clearTooOldAccess(track);
} }
if (Domains.isLocalhost(host)) lastLocalhostAccess = System.currentTimeMillis(); else { if (Domains.isLocalhost(host)) lastLocalhostAccess = System.currentTimeMillis();
System.out.println("******** Access not from localhost: " + host);
}
} }
public static Collection<Track> accessTrack(final String host) { public static Collection<Track> accessTrack(final String host) {

Loading…
Cancel
Save