boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

code simplifications

Browse Source
pull/1/head
Michael Peter Christen 13 years ago
parent 755f5e76cf
commit f00733186b
7 changed files with 133 additions and 314 deletions
Show Stats Download Patch File Download Diff File

2
htroot/Connections_p.java
Unescape View File

@ -64,7 +64,7 @@ public final class Connections_p {
doNameLookup = post.getBoolean("nameLookup");
if (post.containsKey("closeServerSession")) {
final String sessionName = post.get("closeServerSession", null);
sb.closeSessions("10_httpd", sessionName);
sb.closeSessions(sessionName);
prop.put("LOCATION","");
return prop;
}

20
source/de/anomic/server/serverSwitch.java
Unescape View File

@ -31,10 +31,8 @@ import java.io.IOException;
import java.io.InputStreamReader;
import java.io.Reader;
import java.net.InetAddress;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.NavigableMap;
import java.util.Random;
@ -462,26 +460,10 @@ public class serverSwitch
}
}
public String[] sessionsOlderThan(String threadName, long timeout) {
final List<String> list = new ArrayList<String>();
final WorkflowThread st = getThread(threadName);
for ( final Session s : serverCore.getJobList() ) {
if ( !s.isAlive() ) {
continue;
}
if ( s.getTime() > timeout ) {
list.add(s.getName());
}
}
return (String[]) list.toArray();
}
public void closeSessions(String threadName, String sessionName) {
public void closeSessions(String sessionName) {
if ( sessionName == null ) {
return;
}
final WorkflowThread st = getThread(threadName);
for ( final Session s : serverCore.getJobList() ) {
if ( (s.isAlive()) && (s.getName().equals(sessionName)) ) {

53
source/net/yacy/interaction/AugmentHtmlStream.java
Unescape View File

@ -3,20 +3,15 @@ package net.yacy.interaction;
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.StringReader;
import java.net.URLEncoder;
import net.yacy.yacy;
import net.yacy.cora.document.ASCII;
import net.yacy.cora.protocol.Domains;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.cora.protocol.http.HTTPClient;
import net.yacy.document.Document;
import net.yacy.document.TextParser;
import net.yacy.kelondro.data.meta.DigestURI;
import net.yacy.kelondro.logging.Log;
import net.yacy.search.Switchboard;
@ -89,28 +84,6 @@ public class AugmentHtmlStream {
}
/**
* load snippet from resource text file
*
* @return text from resource text file
*/
private static String loadPart(String part) {
String result = "";
try {
BufferedReader in = new BufferedReader(new FileReader(yacy.homedir + File.separatorChar + "htroot"
+ File.separatorChar + "interaction" + File.separatorChar
+ "parts" + File.separatorChar + part));
String str;
while ((str = in.readLine()) != null) {
result += str;
}
in.close();
} catch (IOException e) {
}
return result;
}
public static StringBuffer process(StringBuffer data, DigestURI url, RequestHeader requestHeader) {
String action = requestHeader.get("YACYACTION");
@ -150,28 +123,28 @@ public class AugmentHtmlStream {
Doc = processAddDoctype(Doc);
augmented = true;
}
if (sb.getConfigBool("augmentation.reparse", false) == true) {
org.jsoup.nodes.Document d = Jsoup.parse(Doc);
d.title ("yacy - "+d.title());
if (sb.getConfigBool("interaction.overlayinteraction.enabled", false) == true) {
d.head().append (loadInternal("env/templates/jqueryheader.template", requestHeader));
d.head().append ("<script type='text/javascript'>"+loadInternal("interaction_elements/interaction.js", requestHeader)+"</script>");
d.head().append ("<script type='text/javascript'>"+loadInternal("interaction_elements/interaction_metadata.js", requestHeader)+"</script>");
d.head().append ("<script type='text/javascript'>"+loadInternal("interaction_elements/interaction_metadata.js", requestHeader)+"</script>");
d.body().append (loadInternal("interaction_elements/OverlayInteraction.html?action="+action+"&urlhash="+ ASCII.String(url.hash()) +"&url="+url.toNormalform(false, true), requestHeader));
d.body().append (loadInternal("interaction_elements/Footer.html?action="+action+"&urlhash="+ ASCII.String(url.hash()) +"&url="+url.toNormalform(false, true), requestHeader));
d.body().append (loadInternal("interaction_elements/Footer.html?action="+action+"&urlhash="+ ASCII.String(url.hash()) +"&url="+url.toNormalform(false, true), requestHeader));
}
Doc = d.html();
augmented = true;
}

170
source/net/yacy/peers/Protocol.java
Unescape View File

@ -45,11 +45,9 @@ package net.yacy.peers;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.InetAddress;
import java.net.MalformedURLException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
@ -58,12 +56,10 @@ import java.util.List;
import java.util.Map;
import java.util.SortedMap;
import java.util.TreeMap;
import java.util.regex.Pattern;
import net.yacy.migration;
import net.yacy.cora.date.GenericFormatter;
import net.yacy.cora.document.ASCII;
import net.yacy.cora.document.Classification;
import net.yacy.cora.document.JSONArray;
import net.yacy.cora.document.JSONException;
import net.yacy.cora.document.JSONObject;
@ -88,7 +84,6 @@ import net.yacy.kelondro.data.word.WordReference;
import net.yacy.kelondro.data.word.WordReferenceFactory;
import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.order.Base64Order;
import net.yacy.kelondro.order.Bitfield;
import net.yacy.kelondro.order.Digest;
import net.yacy.kelondro.rwi.Reference;
import net.yacy.kelondro.rwi.ReferenceContainer;
@ -105,10 +100,8 @@ import net.yacy.search.EventTracker;
import net.yacy.search.Switchboard;
import net.yacy.search.SwitchboardConstants;
import net.yacy.search.index.Segment;
import net.yacy.search.query.QueryParams;
import net.yacy.search.query.RWIProcess;
import net.yacy.search.query.SearchEvent;
import net.yacy.search.ranking.RankingProfile;
import net.yacy.search.snippet.TextSnippet;
import org.apache.http.entity.mime.content.ContentBody;
@ -593,12 +586,10 @@ public final class Protocol
}
public static int search(
final Seed mySeed,
final SearchEvent event,
final String wordhashes,
final String excludehashes,
final String urlhashes,
final Pattern prefer,
final Pattern filter,
final String modifier,
final String language,
final String sitehash,
@ -610,12 +601,8 @@ public final class Protocol
final boolean global,
final int partitions,
final Seed target,
final Segment indexSegment,
final RWIProcess containerCache,
final SearchEvent.SecondarySearchSuperviser secondarySearchSuperviser,
final Blacklist blacklist,
final RankingProfile rankingProfile,
final Bitfield constraint) {
final Blacklist blacklist) {
// send a search request to peer with remote Hash
// INPUT:
@ -632,18 +619,16 @@ public final class Protocol
// duetime : maximum time that a peer should spent to create a result
final long timestamp = System.currentTimeMillis();
containerCache.addExpectedRemoteReferences(count);
event.rankingProcess.addExpectedRemoteReferences(count);
SearchResult result;
try {
result =
new SearchResult(
event,
basicRequestParts(Switchboard.getSwitchboard(), target.hash, crypt.randomSalt()),
mySeed,
wordhashes,
excludehashes,
urlhashes,
prefer,
filter,
modifier,
language,
sitehash,
@ -656,9 +641,8 @@ public final class Protocol
partitions,
target.getHexHash() + ".yacyh",
target.getClusterAddress(),
secondarySearchSuperviser,
rankingProfile,
constraint);
secondarySearchSuperviser
);
} catch ( final IOException e ) {
Network.log.logInfo("SEARCH failed, Peer: "
+ target.hash
@ -757,10 +741,10 @@ public final class Protocol
// passed all checks, store url
try {
indexSegment.fulltext().putMetadata(urlEntry);
event.getQuery().getSegment().fulltext().putMetadata(urlEntry);
ResultURLs.stack(
urlEntry,
mySeed.hash.getBytes(),
event.peers.mySeed().hash.getBytes(),
UTF8.getBytes(target.hash),
EventOrigin.QUERIES);
} catch ( final IOException e ) {
@ -792,14 +776,14 @@ public final class Protocol
// store remote result to local result container
// insert one container into the search result buffer
// one is enough, only the references are used, not the word
containerCache.add(container.get(0), false, target.getName() + "/" + target.hash, result.joincount, time);
containerCache.addFinalize();
containerCache.addExpectedRemoteReferences(-count);
event.rankingProcess.add(container.get(0), false, target.getName() + "/" + target.hash, result.joincount, time);
event.rankingProcess.addFinalize();
event.rankingProcess.addExpectedRemoteReferences(-count);
// insert the containers to the index
for ( final ReferenceContainer<WordReference> c : container ) {
try {
indexSegment.termIndex().add(c);
event.getQuery().getSegment().termIndex().add(c);
} catch ( final Exception e ) {
Log.logException(e);
}
@ -822,9 +806,9 @@ public final class Protocol
+ result.references.length
+ " topics");
// add references twice, so they can be counted (must have at least 2 entries)
synchronized ( containerCache ) {
containerCache.addTopic(result.references);
containerCache.addTopic(result.references);
synchronized ( event.rankingProcess ) {
event.rankingProcess.addTopic(result.references);
event.rankingProcess.addTopic(result.references);
}
}
@ -895,13 +879,11 @@ public final class Protocol
public Map<byte[], String> indexabstract; // index abstracts, a collection of url-hashes per word
public SearchResult(
final SearchEvent event,
final Map<String, ContentBody> parts,
final Seed mySeed,
final String wordhashes,
final String excludehashes,
final String urlhashes,
final Pattern prefer,
final Pattern filter,
final String modifier,
final String language,
final String sitehash,
@ -914,9 +896,8 @@ public final class Protocol
final int partitions,
final String hostname,
final String hostaddress,
final SearchEvent.SecondarySearchSuperviser secondarySearchSuperviser,
final RankingProfile rankingProfile,
final Bitfield constraint) throws IOException {
final SearchEvent.SecondarySearchSuperviser secondarySearchSuperviser
) throws IOException {
// send a search request to peer with remote Hash
//if (hostaddress.equals(mySeed.getClusterAddress())) hostaddress = "127.0.0.1:" + mySeed.getPort(); // for debugging
@ -943,7 +924,7 @@ public final class Protocol
keyBody.writeTo(baos);
key = baos.toString();
}
parts.put("myseed", UTF8.StringBody((mySeed == null) ? "" : mySeed.genSeedStr(key)));
parts.put("myseed", UTF8.StringBody((event.peers.mySeed() == null) ? "" : event.peers.mySeed().genSeedStr(key)));
parts.put("count", UTF8.StringBody(Integer.toString(Math.max(10, count))));
parts.put("time", UTF8.StringBody(Long.toString(Math.max(3000, time))));
parts.put("resource", UTF8.StringBody(((global) ? "global" : "local")));
@ -952,8 +933,8 @@ public final class Protocol
parts.put("exclude", UTF8.StringBody(excludehashes));
parts.put("duetime", UTF8.StringBody("1000"));
parts.put("urls", UTF8.StringBody(urlhashes));
parts.put("prefer", UTF8.StringBody(prefer.pattern()));
parts.put("filter", UTF8.StringBody(filter.pattern()));
parts.put("prefer", UTF8.StringBody(event.getQuery().prefer.pattern()));
parts.put("filter", UTF8.StringBody(event.getQuery().urlMask.pattern()));
parts.put("modifier", UTF8.StringBody(modifier));
parts.put("language", UTF8.StringBody(language));
parts.put("sitehash", UTF8.StringBody(sitehash));
@ -961,8 +942,8 @@ public final class Protocol
parts.put("contentdom", UTF8.StringBody(contentdom));
parts.put("ttl", UTF8.StringBody("0"));
parts.put("maxdist", UTF8.StringBody(Integer.toString(maxDistance)));
parts.put("profile", UTF8.StringBody(crypt.simpleEncode(rankingProfile.toExternalString())));
parts.put("constraint", UTF8.StringBody((constraint == null) ? "" : constraint.exportB64()));
parts.put("profile", UTF8.StringBody(crypt.simpleEncode(event.getQuery().ranking.toExternalString())));
parts.put("constraint", UTF8.StringBody((event.getQuery().constraint == null) ? "" : event.getQuery().constraint.exportB64()));
if ( secondarySearchSuperviser != null ) {
parts.put("abstracts", UTF8.StringBody("auto"));
// resultMap = FileUtils.table(HTTPConnector.getConnector(MultiProtocolURI.yacybotUserAgent).post(new MultiProtocolURI("http://" + hostaddress + "/yacy/search.html"), 60000, hostname, parts));
@ -1040,14 +1021,14 @@ public final class Protocol
try {
solrConnector = new RemoteSolrConnector(solrURL);
SolrDocumentList docList = solrConnector.query(querystring, offset, count);
} catch (IOException e) {
Log.logException(e);
} finally {
containerCache.oneFeederTerminated();
}
}
public static Map<String, String> permissionMessage(final SeedDB seedDB, final String targetHash) {
// ask for allowed message size and attachement size
// if this replies null, the peer does not answer
@ -1503,107 +1484,6 @@ public final class Protocol
}
}
public static void main(final String[] args) {
if ( args.length > 2 ) {
// search a remote peer. arguments:
// first arg: path to application home
// second arg: address of target peer
// third arg: search word or file name with list of search words
// i.e. /Data/workspace1/yacy/ localhost:8090 /Data/workspace1/yacy/test/words/searchtest.words
System.out.println("yacyClient Test");
final File searchwordfile = new File(args[2]);
final List<String> searchlines = new ArrayList<String>();
if ( searchwordfile.exists() ) {
Iterator<String> i;
try {
i = FileUtils.strings(FileUtils.read(searchwordfile));
while ( i.hasNext() ) {
searchlines.add(i.next());
}
} catch ( final IOException e ) {
e.printStackTrace();
System.exit(-1);
}
} else {
searchlines.add(args[2]);
}
for ( final String line : searchlines ) {
final byte[] wordhashe =
ASCII.getBytes(QueryParams.hashSet2hashString(Word.words2hashesHandles(QueryParams
.cleanQuery(line)[0])));
final long time = System.currentTimeMillis();
SearchResult result;
try {
result =
new SearchResult(basicRequestParts((String) null, (String) null, "freeworld"), null, // sb.peers.mySeed(),
ASCII.String(wordhashe),
"", // excludehashes,
"", // urlhashes,
QueryParams.matchnothing_pattern, // prefer,
QueryParams.catchall_pattern, // filter,
"", // modifier
"", // language,
"", // sitehash,
"", // authorhash,
"all", // contentdom,
10, // count,
3000, // time,
1000, // maxDistance,
true, //global,
16, // partitions,
"",
args[1],
null, //secondarySearchSuperviser,
new RankingProfile(Classification.ContentDomain.TEXT), // rankingProfile,
null // constraint);
);
for ( final URIMetadata link : result.links ) {
System.out.println(link.url().toNormalform(true, false));
System.out.println(link.snippet());
}
} catch ( final IOException e ) {
// TODO Auto-generated catch block
e.printStackTrace();
}
System.out.println("Search Time: " + (System.currentTimeMillis() - time));
}
System.exit(0);
} else if ( args.length == 1 ) {
System.out.println("wput Test");
// connection params
MultiProtocolURI url = null;
try {
url = new MultiProtocolURI(args[0]);
} catch ( final MalformedURLException e ) {
Log.logException(e);
}
if ( url == null ) {
System.exit(1);
return;
}
final String vhost = url.getHost();
final int timeout = 10000;
// new data
final Map<String, ContentBody> newpost = new LinkedHashMap<String, ContentBody>();
newpost.put("process", UTF8.StringBody("permission"));
newpost.put("purpose", UTF8.StringBody("crcon"));
byte[] res;
try {
// res = HTTPConnector.getConnector(MultiProtocolURI.yacybotUserAgent).post(url, timeout, vhost, newpost, true);
final HTTPClient httpClient = new HTTPClient(ClientIdentification.getUserAgent(), timeout);
res = httpClient.POSTbytes(url, vhost, newpost, true);
System.out.println(UTF8.String(res));
} catch ( final IOException e1 ) {
Log.logException(e1);
}
}
try {
net.yacy.cora.protocol.http.HTTPClient.closeConnectionManager();
} catch ( final InterruptedException e ) {
Log.logException(e);
}
}
public static final boolean authentifyRequest(final serverObjects post, final serverSwitch env) {
if ( post == null || env == null ) {
return false;

157
source/net/yacy/peers/RemoteSearch.java
Unescape View File

@ -27,19 +27,14 @@ package net.yacy.peers;
import java.util.Iterator;
import java.util.Set;
import java.util.SortedMap;
import java.util.regex.Pattern;
import net.yacy.cora.document.ASCII;
import net.yacy.cora.storage.HandleSet;
import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.order.Bitfield;
import net.yacy.peers.dht.PeerSelection;
import net.yacy.repository.Blacklist;
import net.yacy.search.index.Segment;
import net.yacy.search.query.QueryParams;
import net.yacy.search.query.RWIProcess;
import net.yacy.search.query.SearchEvent;
import net.yacy.search.ranking.RankingProfile;
public class RemoteSearch extends Thread {
@ -50,49 +45,37 @@ public class RemoteSearch extends Thread {
final private String wordhashes, excludehashes, urlhashes, sitehash, authorhash, contentdom;
final private boolean global;
final private int partitions;
final private Segment indexSegment;
final private RWIProcess containerCache;
final private SearchEvent.SecondarySearchSuperviser secondarySearchSuperviser;
final private Blacklist blacklist;
final private Seed targetPeer;
private int urls;
private final int count, maxDistance;
private final long time;
final private RankingProfile rankingProfile;
final private Pattern prefer, filter;
final private QueryParams.Modifier modifier;
final private String language;
final private Bitfield constraint;
final private SeedDB peers;
public RemoteSearch(
final SearchEvent event,
final String wordhashes, final String excludehashes,
final String urlhashes, // this is the field that is filled during a secondary search to restrict to specific urls that are to be retrieved
final Pattern prefer,
final Pattern filter,
final QueryParams.Modifier modifier,
final String language,
final String sitehash, final String authorhash, final String contentdom,
final int count, final long time, final int maxDistance,
final boolean global, final int partitions,
final String sitehash,
final String authorhash,
final String contentdom,
final int count,
final long time,
final int maxDistance,
final boolean global,
final int partitions,
final Seed targetPeer,
final Segment indexSegment,
final SeedDB peers,
final RWIProcess containerCache,
final SearchEvent.SecondarySearchSuperviser secondarySearchSuperviser,
final Blacklist blacklist,
final RankingProfile rankingProfile,
final Bitfield constraint) {
final Blacklist blacklist) {
super(ysThreadGroup, "yacySearch_" + targetPeer.getName());
//System.out.println("DEBUG - yacySearch thread " + this.getName() + " initialized " + ((urlhashes.isEmpty()) ? "(primary)" : "(secondary)"));
assert wordhashes.length() >= 12;
this.event = event;
this.wordhashes = wordhashes;
this.excludehashes = excludehashes;
this.urlhashes = urlhashes;
this.prefer = prefer;
this.filter = filter;
this.modifier = modifier;
this.language = language;
this.sitehash = sitehash;
@ -100,9 +83,6 @@ public class RemoteSearch extends Thread {
this.contentdom = contentdom;
this.global = global;
this.partitions = partitions;
this.indexSegment = indexSegment;
this.peers = peers;
this.containerCache = containerCache;
this.secondarySearchSuperviser = secondarySearchSuperviser;
this.blacklist = blacklist;
this.targetPeer = targetPeer;
@ -110,34 +90,42 @@ public class RemoteSearch extends Thread {
this.count = count;
this.time = time;
this.maxDistance = maxDistance;
this.rankingProfile = rankingProfile;
this.constraint = constraint;
}
@Override
public void run() {
this.containerCache.oneFeederStarted();
this.event.rankingProcess.oneFeederStarted();
try {
this.urls = Protocol.search(
this.peers.mySeed(),
this.wordhashes, this.excludehashes, this.urlhashes,
this.prefer, this.filter, this.modifier.getModifier(),
this.language, this.sitehash, this.authorhash, this.contentdom,
this.count, this.time, this.maxDistance, this.global, this.partitions,
this.targetPeer, this.indexSegment, this.containerCache, this.secondarySearchSuperviser,
this.blacklist, this.rankingProfile, this.constraint);
this.event,
this.wordhashes,
this.excludehashes,
this.urlhashes,
this.modifier.getModifier(),
this.language,
this.sitehash,
this.authorhash,
this.contentdom,
this.count,
this.time,
this.maxDistance,
this.global,
this.partitions,
this.targetPeer,
this.secondarySearchSuperviser,
this.blacklist);
if (this.urls >= 0) {
// urls is an array of url hashes. this is only used for log output
if (this.urlhashes != null && this.urlhashes.length() > 0) Network.log.logInfo("SECONDARY REMOTE SEARCH - remote peer " + this.targetPeer.hash + ":" + this.targetPeer.getName() + " contributed " + this.urls + " links for word hash " + this.wordhashes);
this.peers.mySeed().incRI(this.urls);
this.peers.mySeed().incRU(this.urls);
this.event.peers.mySeed().incRI(this.urls);
this.event.peers.mySeed().incRU(this.urls);
} else {
Network.log.logInfo("REMOTE SEARCH - no answer from remote peer " + this.targetPeer.hash + ":" + this.targetPeer.getName());
}
} catch (final Exception e) {
Log.logException(e);
} finally {
this.containerCache.oneFeederTerminated();
this.event.rankingProcess.oneFeederTerminated();
}
}
@ -162,21 +150,8 @@ public class RemoteSearch extends Thread {
public static void primaryRemoteSearches(
final SearchEvent event,
final String wordhashes, final String excludehashes,
final Pattern prefer, final Pattern filter,
final QueryParams.Modifier modifier,
final String language,
final String sitehash,
final String authorhash,
final String contentdom,
final int count, final long time, final int maxDist,
final Segment indexSegment,
final SeedDB peers,
final RWIProcess containerCache,
final SearchEvent.SecondarySearchSuperviser secondarySearchSuperviser,
final int count, final long time,
final Blacklist blacklist,
final RankingProfile rankingProfile,
final Bitfield constraint,
final SortedMap<byte[], String> clusterselection,
final int burstRobinsonPercent,
final int burstMultiwordPercent) {
@ -184,17 +159,15 @@ public class RemoteSearch extends Thread {
//if (wordIndex.seedDB.mySeed() == null || wordIndex.seedDB.mySeed().getPublicAddress() == null) { return null; }
// prepare seed targets and threads
assert language != null;
assert wordhashes.length() >= 12 : "wordhashes = " + wordhashes;
final Seed[] targetPeers =
(clusterselection == null) ?
PeerSelection.selectSearchTargets(
peers,
QueryParams.hashes2Set(wordhashes),
peers.redundancy(),
event.peers,
event.getQuery().query_include_hashes,
event.peers.redundancy(),
burstRobinsonPercent,
burstMultiwordPercent)
: PeerSelection.selectClusterPeers(peers, clusterselection);
: PeerSelection.selectClusterPeers(event.peers, clusterselection);
if (targetPeers == null) return;
final int targets = targetPeers.length;
if (targets == 0) return;
@ -203,10 +176,22 @@ public class RemoteSearch extends Thread {
try {
RemoteSearch rs = new RemoteSearch(
event,
wordhashes, excludehashes, "", prefer, filter, modifier,
language, sitehash, authorhash, contentdom,
count, time, maxDist, true, targets, targetPeers[i],
indexSegment, peers, containerCache, secondarySearchSuperviser, blacklist, rankingProfile, constraint);
QueryParams.hashSet2hashString(event.getQuery().query_include_hashes),
QueryParams.hashSet2hashString(event.getQuery().query_exclude_hashes),
"",
event.getQuery().modifier,
event.getQuery().targetlang == null ? "" : event.getQuery().targetlang,
event.getQuery().sitehash == null ? "" : event.getQuery().sitehash,
event.getQuery().authorhash == null ? "" : event.getQuery().authorhash,
event.getQuery().contentdom == null ? "all" : event.getQuery().contentdom.toString(),
count,
time,
event.getQuery().maxDistance,
true,
targets,
targetPeers[i],
event.secondarySearchSuperviser,
blacklist);
rs.start();
event.primarySearchThreadsL.add(rs);
} catch (final OutOfMemoryError e) {
@ -215,33 +200,43 @@ public class RemoteSearch extends Thread {
}
}
}
public static RemoteSearch secondaryRemoteSearch(
final SearchEvent event,
final Set<String> wordhashes, final String urlhashes,
final Set<String> wordhashes,
final String urlhashes,
final long time,
final Segment indexSegment,
final SeedDB peers,
final RWIProcess containerCache,
final String targethash, final Blacklist blacklist,
final RankingProfile rankingProfile,
final Bitfield constraint, final SortedMap<byte[], String> clusterselection) {
final String targethash,
final Blacklist blacklist) {
// check own peer status
if (peers.mySeed() == null || peers.mySeed().getPublicAddress() == null) { return null; }
if (event.peers.mySeed() == null || event.peers.mySeed().getPublicAddress() == null) { return null; }
assert urlhashes != null;
assert urlhashes.length() > 0;
// prepare seed targets and threads
final Seed targetPeer = peers.getConnected(targethash);
final Seed targetPeer = event.peers.getConnected(targethash);
if (targetPeer == null || targetPeer.hash == null) return null;
if (clusterselection != null) targetPeer.setAlternativeAddress(clusterselection.get(ASCII.getBytes(targetPeer.hash)));
StringBuilder whs = new StringBuilder(24);
for (String s: wordhashes) whs.append(s);
if (event.preselectedPeerHashes != null) targetPeer.setAlternativeAddress(event.preselectedPeerHashes.get(ASCII.getBytes(targetPeer.hash)));
final RemoteSearch searchThread = new RemoteSearch(
event,
whs.toString(), "", urlhashes, QueryParams.matchnothing_pattern, QueryParams.catchall_pattern, new QueryParams.Modifier(""), "", "", "", "all", 20, time, 9999, true, 0, targetPeer,
indexSegment, peers, containerCache, null, blacklist, rankingProfile, constraint);
QueryParams.hashSet2hashString(wordhashes),
"",
urlhashes,
new QueryParams.Modifier(""),
"",
"",
"",
"all",
20,
time,
9999,
true,
0,
targetPeer,
null,
blacklist);
searchThread.start();
return searchThread;
}

11
source/net/yacy/search/query/QueryParams.java
Unescape View File

@ -367,6 +367,17 @@ public final class QueryParams {
return ASCII.String(bb);
}
public static String hashSet2hashString(final Set<String> hashes) {
final byte[] bb = new byte[hashes.size() * Word.commonHashLength];
int p = 0;
for (final String s : hashes) {
assert s.length() == Word.commonHashLength : "hash = " + s;
System.arraycopy(ASCII.getBytes(s), 0, bb, p, Word.commonHashLength);
p += Word.commonHashLength;
}
return ASCII.String(bb);
}
public static String anonymizedQueryHashes(final HandleSet hashes) {
// create a more anonymized representation of a query hashes for logging
final Iterator<byte[]> i = hashes.iterator();

34
source/net/yacy/search/query/SearchEvent.java
Unescape View File

@ -86,17 +86,17 @@ public final class SearchEvent {
// class variables that may be implemented with an abstract class
private long eventTime;
private QueryParams query;
private final SeedDB peers;
public final SeedDB peers;
private final WorkTables workTables;
private final RWIProcess rankingProcess; // ordered search results, grows dynamically as all the query threads enrich this container
public final RWIProcess rankingProcess; // ordered search results, grows dynamically as all the query threads enrich this container
private final SnippetProcess resultFetcher;
private final SecondarySearchSuperviser secondarySearchSuperviser;
public final SecondarySearchSuperviser secondarySearchSuperviser;
// class variables for remote searches
public final List<RemoteSearch> primarySearchThreadsL;
private RemoteSearch[] secondarySearchThreads;
private final SortedMap<byte[], String> preselectedPeerHashes;
public final SortedMap<byte[], String> preselectedPeerHashes;
private final Thread localSearchThread;
private final SortedMap<byte[], Integer> IACount;
private final SortedMap<byte[], String> IAResults;
@ -167,25 +167,9 @@ public final class SearchEvent {
Thread.currentThread().setName("SearchEvent.primaryRemoteSearches");
RemoteSearch.primaryRemoteSearches(
SearchEvent.this,
QueryParams.hashSet2hashString(SearchEvent.this.query.query_include_hashes),
QueryParams.hashSet2hashString(SearchEvent.this.query.query_exclude_hashes),
SearchEvent.this.query.prefer,
SearchEvent.this.query.urlMask,
SearchEvent.this.query.modifier,
SearchEvent.this.query.targetlang == null ? "" : SearchEvent.this.query.targetlang,
SearchEvent.this.query.sitehash == null ? "" : SearchEvent.this.query.sitehash,
SearchEvent.this.query.authorhash == null ? "" : SearchEvent.this.query.authorhash,
SearchEvent.this.query.contentdom == null ? "all" : SearchEvent.this.query.contentdom.toString(),
remote_maxcount,
remote_maxtime,
SearchEvent.this.query.maxDistance,
SearchEvent.this.query.getSegment(),
peers,
SearchEvent.this.rankingProcess,
SearchEvent.this.secondarySearchSuperviser,
Switchboard.urlBlacklist,
SearchEvent.this.query.ranking,
SearchEvent.this.query.constraint,
(SearchEvent.this.query.domType == QueryParams.Searchdom.GLOBAL) ? null : preselectedPeerHashes,
burstRobinsonPercent,
burstMultiwordPercent);
@ -303,7 +287,7 @@ public final class SearchEvent {
}
SearchEventCache.put(this.query.id(false), this);
}
public ReferenceOrder getOrder() {
return this.order;
}
@ -694,14 +678,8 @@ public final class SearchEvent {
words,
urls.toString(),
6000,
SearchEvent.this.query.getSegment(),
SearchEvent.this.peers,
SearchEvent.this.rankingProcess,
peer,
Switchboard.urlBlacklist,
SearchEvent.this.query.ranking,
SearchEvent.this.query.constraint,
SearchEvent.this.preselectedPeerHashes);
Switchboard.urlBlacklist);
}
}
}

Write Preview
Loading…
Cancel
Save