fix for remote search process

pull/1/head
Michael Peter Christen 11 years ago
parent 0843b12ef3
commit 97995a1dd9

@ -123,7 +123,7 @@ public class ConcurrentUpdateSolrConnector implements SolrConnector {
@Override
public int bufferSize() {
return this.updateCapacity;
return this.docBuffer.size();
}
@Override

@ -503,31 +503,34 @@ public final class Protocol {
final long timestamp = System.currentTimeMillis();
event.addExpectedRemoteReferences(count);
SearchResult result;
String clusteraddress = target.getPublicAddress(target.getIP());
if (target.clash(event.peers.mySeed().getIPs())) clusteraddress = "localhost:" + event.peers.mySeed().getPort();
try {
result =
new SearchResult(
event,
basicRequestParts(Switchboard.getSwitchboard(), target.hash, crypt.randomSalt()),
wordhashes,
excludehashes,
"",
language,
contentdom,
count,
time,
maxDistance,
partitions,
target.getHexHash() + ".yacyh",
clusteraddress,
secondarySearchSuperviser
);
} catch (final IOException e ) {
Network.log.info("SEARCH failed, Peer: " + target.hash + ":" + target.getName() + " (" + e.getMessage() + ")");
event.peers.peerActions.peerDeparture(target, "search request to peer created io exception: " + e.getMessage());
return -1;
SearchResult result = null;
for (String ip: target.getIPs()) {
if (ip.indexOf(':') >= 0) System.out.println("Search target: IPv6: " + ip);
String clusteraddress = target.getPublicAddress(ip);
if (target.clash(event.peers.mySeed().getIPs())) clusteraddress = "localhost:" + event.peers.mySeed().getPort();
try {
result =
new SearchResult(
event,
basicRequestParts(Switchboard.getSwitchboard(), target.hash, crypt.randomSalt()),
wordhashes,
excludehashes,
"",
language,
contentdom,
count,
time,
maxDistance,
partitions,
target.getHexHash() + ".yacyh",
clusteraddress,
secondarySearchSuperviser
);
} catch (final IOException e ) {
Network.log.info("SEARCH failed, Peer: " + target.hash + ":" + target.getName() + " (" + e.getMessage() + ")");
event.peers.peerActions.interfaceDeparture(target, ip);
return -1;
}
}
// computation time
final long totalrequesttime = System.currentTimeMillis() - timestamp;

@ -174,7 +174,6 @@ public class RemoteSearch extends Thread {
dhtPeers = DHTSelection.selectClusterPeers(event.peers, clusterselection);
} else {
if (event.query.getQueryGoal().isCatchall() || event.query.getQueryGoal().getIncludeHashes().has(Segment.catchallHash)) {
if (event.query.modifier.sitehost != null && event.query.modifier.sitehost.length() > 0) {
// select peers according to host name, not the query goal
String newGoal = Domains.getSmartSLD(event.query.modifier.sitehost);
@ -186,29 +185,29 @@ public class RemoteSearch extends Thread {
random);
} else {
// select just random peers
dhtPeers = DHTSelection.seedsByAge(event.peers, false, count).values();
dhtPeers = DHTSelection.seedsByAge(event.peers, false, event.peers.redundancy()).values();
}
} else {
dhtPeers = DHTSelection.selectDHTSearchTargets(
event.peers,
event.query.getQueryGoal().getIncludeHashes(),
minage,
redundancy, event.peers.redundancy(),
random);
// this set of peers may be too large and consume too many threads if more than one word is searched.
// to prevent overloading, we do a subset collection based on random to prevent the death of the own peer
// and to do a distributed load-balancing on the target peers
long targetSize = 1 + redundancy * event.peers.scheme.verticalPartitions(); // this is the maximum for one word plus one
if (dhtPeers.size() > targetSize) {
ArrayList<Seed> pa = new ArrayList<Seed>(dhtPeers.size());
pa.addAll(dhtPeers);
dhtPeers.clear();
for (int i = 0; i < targetSize; i++) dhtPeers.add(pa.remove(random.nextInt(pa.size())));
}
}
dhtPeers = DHTSelection.selectDHTSearchTargets(
event.peers,
event.query.getQueryGoal().getIncludeHashes(),
minage,
redundancy, event.peers.redundancy(),
random);
// this set of peers may be too large and consume too many threads if more than one word is searched.
// to prevent overloading, we do a subset collection based on random to prevent the death of the own peer
// and to do a distributed load-balancing on the target peers
long targetSize = 1 + redundancy * event.peers.scheme.verticalPartitions(); // this is the maximum for one word plus one
if (dhtPeers.size() > targetSize) {
ArrayList<Seed> pa = new ArrayList<Seed>(dhtPeers.size());
pa.addAll(dhtPeers);
dhtPeers.clear();
for (int i = 0; i < targetSize; i++) dhtPeers.add(pa.remove(random.nextInt(pa.size())));
}
}
if (dhtPeers == null) dhtPeers = new HashSet<Seed>();
// select node targets
final Collection<Seed> robinsonPeers = DHTSelection.selectExtraTargets(event.peers, event.query.getQueryGoal().getIncludeHashes(), minage, dhtPeers, robinsoncount, random);

@ -140,9 +140,7 @@ public class serverAccessTracker {
track.add(new Track(System.currentTimeMillis(), accessPath));
clearTooOldAccess(track);
}
if (Domains.isLocalhost(host)) lastLocalhostAccess = System.currentTimeMillis(); else {
System.out.println("******** Access not from localhost: " + host);
}
if (Domains.isLocalhost(host)) lastLocalhostAccess = System.currentTimeMillis();
}
public static Collection<Track> accessTrack(final String host) {

Loading…
Cancel
Save