boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

refactoring in remote search and stub for remote node peer selection

Browse Source
pull/1/head
orbiter 13 years ago
parent d7ea45f698
commit 404b0aab09
5 changed files with 68 additions and 12 deletions
Show Stats Download Patch File Download Diff File

19
source/net/yacy/peers/Protocol.java
Unescape View File

@ -77,6 +77,8 @@ import net.yacy.cora.protocol.ClientIdentification;
import net.yacy.cora.protocol.Domains;
import net.yacy.cora.protocol.http.HTTPClient;
import net.yacy.cora.services.federated.opensearch.SRURSSConnector;
import net.yacy.cora.services.federated.solr.RemoteSolrConnector;
import net.yacy.cora.services.federated.solr.SolrConnector;
import net.yacy.cora.services.federated.yacy.CacheStrategy;
import net.yacy.cora.util.SpaceExceededException;
import net.yacy.kelondro.data.meta.URIMetadata;
@ -110,6 +112,7 @@ import net.yacy.search.ranking.RankingProfile;
import net.yacy.search.snippet.TextSnippet;
import org.apache.http.entity.mime.content.ContentBody;
import org.apache.solr.common.SolrDocumentList;
import de.anomic.crawler.ResultURLs;
import de.anomic.crawler.ResultURLs.EventOrigin;
@ -1029,6 +1032,22 @@ public final class Protocol
}
}
public static void solrQuery(
final RWIProcess containerCache,
final String solrURL, final String querystring, final int offset, final int count) {
containerCache.oneFeederStarted();
SolrConnector solrConnector;
try {
solrConnector = new RemoteSolrConnector(solrURL);
SolrDocumentList docList = solrConnector.query(querystring, offset, count);
} catch (IOException e) {
Log.logException(e);
} finally {
containerCache.oneFeederTerminated();
}
}
public static Map<String, String> permissionMessage(final SeedDB seedDB, final String targetHash) {
// ask for allowed message size and attachement size
// if this replies null, the peer does not answer

13
source/net/yacy/peers/RemoteSearch.java
Unescape View File

@ -25,7 +25,6 @@
package net.yacy.peers;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import java.util.SortedMap;
import java.util.regex.Pattern;
@ -47,6 +46,7 @@ public class RemoteSearch extends Thread {
private static final ThreadGroup ysThreadGroup = new ThreadGroup("yacySearchThreadGroup");
final private SearchEvent event;
final private String wordhashes, excludehashes, urlhashes, sitehash, authorhash, contentdom;
final private boolean global;
final private int partitions;
@ -66,6 +66,7 @@ public class RemoteSearch extends Thread {
final private SeedDB peers;
public RemoteSearch(
final SearchEvent event,
final String wordhashes, final String excludehashes,
final String urlhashes, // this is the field that is filled during a secondary search to restrict to specific urls that are to be retrieved
final Pattern prefer,
@ -86,6 +87,7 @@ public class RemoteSearch extends Thread {
super(ysThreadGroup, "yacySearch_" + targetPeer.getName());
//System.out.println("DEBUG - yacySearch thread " + this.getName() + " initialized " + ((urlhashes.isEmpty()) ? "(primary)" : "(secondary)"));
assert wordhashes.length() >= 12;
this.event = event;
this.wordhashes = wordhashes;
this.excludehashes = excludehashes;
this.urlhashes = urlhashes;
@ -159,7 +161,7 @@ public class RemoteSearch extends Thread {
}
public static void primaryRemoteSearches(
final List<RemoteSearch> searchThreads,
final SearchEvent event,
final String wordhashes, final String excludehashes,
final Pattern prefer, final Pattern filter,
final QueryParams.Modifier modifier,
@ -200,20 +202,22 @@ public class RemoteSearch extends Thread {
if (targetPeers[i] == null || targetPeers[i].hash == null) continue;
try {
RemoteSearch rs = new RemoteSearch(
event,
wordhashes, excludehashes, "", prefer, filter, modifier,
language, sitehash, authorhash, contentdom,
count, time, maxDist, true, targets, targetPeers[i],
indexSegment, peers, containerCache, secondarySearchSuperviser, blacklist, rankingProfile, constraint);
rs.start();
searchThreads.add(rs);
event.primarySearchThreadsL.add(rs);
} catch (final OutOfMemoryError e) {
Log.logException(e);
break;
}
}
}
public static RemoteSearch secondaryRemoteSearch(
final SearchEvent event,
final Set<String> wordhashes, final String urlhashes,
final long time,
final Segment indexSegment,
@ -235,6 +239,7 @@ public class RemoteSearch extends Thread {
StringBuilder whs = new StringBuilder(24);
for (String s: wordhashes) whs.append(s);
final RemoteSearch searchThread = new RemoteSearch(
event,
whs.toString(), "", urlhashes, QueryParams.matchnothing_pattern, QueryParams.catchall_pattern, new QueryParams.Modifier(""), "", "", "", "all", 20, time, 9999, true, 0, targetPeer,
indexSegment, peers, containerCache, null, blacklist, rankingProfile, constraint);
searchThread.start();

5
source/net/yacy/peers/Seed.java
Unescape View File

@ -1143,6 +1143,11 @@ public class Seed implements Cloneable, Comparable<Seed>, Comparator<Seed>
return o1.compareTo(o2);
}
@Override
public boolean equals(Object other) {
return this.hash.equals(((Seed) other).hash);
}
public static void main(final String[] args) {
final ScoreMap<Integer> s = new ClusteredScoreMap<Integer>();
for ( int i = 0; i < 10000; i++ ) {

31
source/net/yacy/peers/dht/PeerSelection.java
Unescape View File

@ -30,6 +30,7 @@ import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.Set;
import java.util.SortedMap;
@ -71,6 +72,34 @@ public class PeerSelection {
}
return l.toArray(new Seed[l.size()]);
}
public static Seed[] selectNodeSearchTargets(final SeedDB seedDB, int maxCount, Set<Seed> omit) {
if (seedDB == null) { return null; }
final List<Seed> goodSeeds = new ArrayList<Seed>();
final List<Seed> optionalSeeds = new ArrayList<Seed>();
Seed seed;
Iterator<Seed> seedenum = seedDB.seedsConnected(true, true, Seed.randomHash(), 1.041f);
int c = seedDB.sizeConnected();
while (seedenum.hasNext() && c-- > 0 && goodSeeds.size() < maxCount) {
seed = seedenum.next();
if (seed == null || omit.contains(seed)) continue;
if (seed.getFlagRootNode()) {
goodSeeds.add(seed);
} else {
optionalSeeds.add(seed);
}
}
Random r = new Random(System.currentTimeMillis());
while (goodSeeds.size() < maxCount && optionalSeeds.size() > 0) {
goodSeeds.add(optionalSeeds.remove(r.nextInt(optionalSeeds.size())));
}
// produce return set
Seed[] result = new Seed[goodSeeds.size()];
result = goodSeeds.toArray(result);
return result;
}
public static Seed[] selectSearchTargets(
final SeedDB seedDB,
@ -163,7 +192,7 @@ public class PeerSelection {
result = regularSeeds.values().toArray(result);
return result;
}
private static void selectDHTPositions(
final SeedDB seedDB,
byte[] wordhash,

12
source/net/yacy/search/query/SearchEvent.java
Unescape View File

@ -63,8 +63,7 @@ import net.yacy.search.ranking.ReferenceOrder;
import net.yacy.search.snippet.ResultEntry;
import de.anomic.data.WorkTables;
public final class SearchEvent
{
public final class SearchEvent {
public enum Type {
INITIALIZATION,
@ -95,7 +94,7 @@ public final class SearchEvent
private final SecondarySearchSuperviser secondarySearchSuperviser;
// class variables for remote searches
private final List<RemoteSearch> primarySearchThreadsL;
public final List<RemoteSearch> primarySearchThreadsL;
private RemoteSearch[] secondarySearchThreads;
private final SortedMap<byte[], String> preselectedPeerHashes;
private final Thread localSearchThread;
@ -167,7 +166,7 @@ public final class SearchEvent
public void run() {
Thread.currentThread().setName("SearchEvent.primaryRemoteSearches");
RemoteSearch.primaryRemoteSearches(
SearchEvent.this.primarySearchThreadsL,
SearchEvent.this,
QueryParams.hashSet2hashString(SearchEvent.this.query.query_include_hashes),
QueryParams.hashSet2hashString(SearchEvent.this.query.query_exclude_hashes),
SearchEvent.this.query.prefer,
@ -304,7 +303,7 @@ public final class SearchEvent
}
SearchEventCache.put(this.query.id(false), this);
}
public ReferenceOrder getOrder() {
return this.order;
}
@ -691,6 +690,7 @@ public final class SearchEvent
this.checkedPeers.add(peer);
SearchEvent.this.secondarySearchThreads[c++] =
RemoteSearch.secondaryRemoteSearch(
SearchEvent.this,
words,
urls.toString(),
6000,
@ -703,9 +703,7 @@ public final class SearchEvent
SearchEvent.this.query.constraint,
SearchEvent.this.preselectedPeerHashes);
}
}
}
public SnippetProcess result() {

Write Preview
Loading…
Cancel
Save