boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

- redesign of a part of the remote search client to make it possible to have a test environment for remote search performance tests

Browse Source 
- added a remote search test main methods in yacyClient

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7146 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 15 years ago
parent 2e75879504
commit 7ebef56add
4 changed files with 324 additions and 257 deletions
Show Stats Download Patch File Download Diff File

1
source/de/anomic/search/Switchboard.java
Unescape View File

@ -672,6 +672,7 @@ public final class Switchboard extends serverSwitch {
this.trail = new ArrayList<String>(); this.trail = new ArrayList<String>();
log.logConfig("Finished Switchboard Initialization"); log.logConfig("Finished Switchboard Initialization");
sb = this;
} }
private void setDefaultSegments() { private void setDefaultSegments() {

497
source/de/anomic/yacy/yacyClient.java
Unescape View File

@ -47,11 +47,11 @@ import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.io.UnsupportedEncodingException; import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException; import java.net.MalformedURLException;
//import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.Iterator; import java.util.Iterator;
import java.util.LinkedHashMap; import java.util.LinkedHashMap;
//import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.TreeMap; import java.util.TreeMap;
import java.util.regex.Pattern; import java.util.regex.Pattern;
@ -84,12 +84,13 @@ import org.apache.http.entity.mime.content.StringBody;
import de.anomic.crawler.ResultURLs; import de.anomic.crawler.ResultURLs;
import de.anomic.crawler.retrieval.EventOrigin; import de.anomic.crawler.retrieval.EventOrigin;
import de.anomic.crawler.retrieval.HTTPLoader; import de.anomic.crawler.retrieval.HTTPLoader;
import de.anomic.search.ContentDomain;
import de.anomic.search.QueryParams;
import de.anomic.search.RankingProfile; import de.anomic.search.RankingProfile;
import de.anomic.search.RankingProcess; import de.anomic.search.RankingProcess;
import de.anomic.search.SearchEvent; import de.anomic.search.SearchEvent;
import de.anomic.search.Segment; import de.anomic.search.Segment;
import de.anomic.search.Switchboard; import de.anomic.search.Switchboard;
import de.anomic.search.SwitchboardConstants;
import de.anomic.search.TextSnippet; import de.anomic.search.TextSnippet;
import de.anomic.server.serverCore; import de.anomic.server.serverCore;
import de.anomic.tools.crypt; import de.anomic.tools.crypt;
@ -397,161 +398,100 @@ public final class yacyClient {
// partitions : number of remote peers that are asked (for evaluation of QPM) // partitions : number of remote peers that are asked (for evaluation of QPM)
// duetime : maximum time that a peer should spent to create a result // duetime : maximum time that a peer should spent to create a result
// prepare request
final String salt = crypt.randomSalt();
final long timestamp = System.currentTimeMillis(); final long timestamp = System.currentTimeMillis();
boolean thisIsASecondarySearch = urlhashes.length() > 0; SearchResult result;
assert !thisIsASecondarySearch || secondarySearchSuperviser == null;
// send request
Map<String, String> result = null;
try { try {
// result = FileUtils.table(HttpConnector.wput("http://" + target.getClusterAddress() + "/yacy/search.html", target.getHexHash() + ".yacyh", post, 60000)); result = searchClient(
final LinkedHashMap<String,ContentBody> parts = yacyNetwork.basicRequestParts(Switchboard.getSwitchboard(), target.hash, salt); yacyNetwork.basicRequestParts(Switchboard.getSwitchboard(), target.hash, crypt.randomSalt()),
parts.put("myseed", new StringBody(mySeed.genSeedStr(salt))); mySeed, wordhashes, excludehashes, urlhashes, prefer, filter, language,
parts.put("count", new StringBody(Integer.toString(Math.max(10, count)))); sitehash, authorhash, count, maxDistance, global, partitions, target.getHexHash() + ".yacyh", target.getClusterAddress(),
parts.put("resource", new StringBody(((global) ? "global" : "local"))); secondarySearchSuperviser, rankingProfile, constraint);
parts.put("partitions", new StringBody(Integer.toString(partitions)));
parts.put("query", new StringBody(wordhashes));
parts.put("exclude", new StringBody(excludehashes));
parts.put("duetime", new StringBody("1000"));
parts.put("urls", new StringBody(urlhashes));
parts.put("prefer", new StringBody(prefer.toString()));
parts.put("filter", new StringBody(filter.toString()));
parts.put("language", new StringBody(language));
parts.put("sitehash", new StringBody(sitehash));
parts.put("authorhash", new StringBody(authorhash));
parts.put("ttl", new StringBody("0"));
parts.put("maxdist", new StringBody(Integer.toString(maxDistance)));
parts.put("profile", new StringBody(crypt.simpleEncode(rankingProfile.toExternalString())));
parts.put("constraint", new StringBody((constraint == null) ? "" : constraint.exportB64()));
if (secondarySearchSuperviser != null)
parts.put("abstracts", new StringBody("auto"));
result = FileUtils.table(HTTPConnector.getConnector(HTTPLoader.crawlerUserAgent).post(new MultiProtocolURI("http://" + target.getClusterAddress() + "/yacy/search.html"), 60000, target.getHexHash() + ".yacyh", parts));
} catch (final IOException e) { } catch (final IOException e) {
yacyCore.log.logInfo("SEARCH failed, Peer: " + target.hash + ":" + target.getName() + " (" + e.getMessage() + "), score=" + target.selectscore); yacyCore.log.logInfo("SEARCH failed, Peer: " + target.hash + ":" + target.getName() + " (" + e.getMessage() + "), score=" + target.selectscore);
//yacyCore.peerActions.peerDeparture(target, "search request to peer created io exception: " + e.getMessage()); //yacyCore.peerActions.peerDeparture(target, "search request to peer created io exception: " + e.getMessage());
return -1; return -1;
} }
if (result == null) return -1;
if (result == null || result.isEmpty()) { // computation time
if (yacyCore.log.isFine()) yacyCore.log.logFine("SEARCH failed FROM " final long totalrequesttime = System.currentTimeMillis() - timestamp;
+ target.hash
+ ":" // create containers
+ target.getName() final int words = wordhashes.length() / Word.commonHashLength;
+ " (zero response), score=" assert words > 0 : "wordhashes = " + wordhashes;
+ target.selectscore); final ReferenceContainer<WordReference>[] container = new ReferenceContainer[words];
return -1; for (int i = 0; i < words; i++) {
} try {
// compute all computation times
final long totalrequesttime = System.currentTimeMillis() - timestamp;
// OUTPUT:
// version : application version of responder
// uptime : uptime in seconds of responder
// total : number of total available LURL's for this search
// count : number of returned LURL's for this search
// resource<n> : LURL of search
// fwhop : hops (depth) of forwards that had been performed to construct this result
// fwsrc : peers that helped to construct this result
// fwrec : peers that would have helped to construct this result (recommendations)
// searchtime : time that the peer actually spent to create the result
// references : references (search hints) that was calculated during search
// now create a plasmaIndex out of this result
// System.out.println("yacyClient: " + ((urlhashes.length() == 0) ? "primary" : "secondary")+ " search result = " + result.toString()); // debug
int urlcount = 0, joincount = 0;
try {
joincount = Integer.parseInt(result.get("joincount")); // the complete number of hits at remote site
urlcount = Integer.parseInt(result.get("count")); // the number of hits that are returned in the result list
} catch (final NumberFormatException e) {
yacyCore.log.logInfo("SEARCH failed FROM " + target.hash + ":" + target.getName() + ", wrong output format: " + e.getMessage());
//yacyCore.peerActions.peerDeparture(target, "search request to peer created number format exception");
return -1;
}
// System.out.println("***result count " + results);
// create containers
final int words = wordhashes.length() / Word.commonHashLength;
assert words > 0 : "wordhashes = " + wordhashes;
final ReferenceContainer<WordReference>[] container = new ReferenceContainer[words];
for (int i = 0; i < words; i++) {
try {
container[i] = ReferenceContainer.emptyContainer(Segment.wordReferenceFactory, wordhashes.substring(i * Word.commonHashLength, (i + 1) * Word.commonHashLength).getBytes(), count); container[i] = ReferenceContainer.emptyContainer(Segment.wordReferenceFactory, wordhashes.substring(i * Word.commonHashLength, (i + 1) * Word.commonHashLength).getBytes(), count);
} catch (RowSpaceExceededException e) { } catch (RowSpaceExceededException e) {
Log.logException(e); Log.logException(e);
return -1; return -1;
} }
} }
// insert results to containers // insert results to containers
URIMetadataRow urlEntry; for (URIMetadataRow urlEntry: result.links) {
for (int n = 0; n < urlcount; n++) { // get one single search result
// get one single search result if (urlEntry == null) continue;
urlEntry = URIMetadataRow.importEntry(result.get("resource" + n)); assert (urlEntry.hash().length == 12) : "urlEntry.hash() = " + new String(urlEntry.hash());
if (urlEntry == null) continue; if (urlEntry.hash().length != 12) continue; // bad url hash
assert (urlEntry.hash().length == 12) : "urlEntry.hash() = " + new String(urlEntry.hash()); final URIMetadataRow.Components metadata = urlEntry.metadata();
if (urlEntry.hash().length != 12) continue; // bad url hash if (metadata == null) continue;
final URIMetadataRow.Components metadata = urlEntry.metadata(); if (blacklist.isListed(Blacklist.BLACKLIST_SEARCH, metadata.url())) {
if (metadata == null) continue; if (yacyCore.log.isInfo()) yacyCore.log.logInfo("remote search: filtered blacklisted url " + metadata.url() + " from peer " + target.getName());
if (blacklist.isListed(Blacklist.BLACKLIST_SEARCH, metadata.url())) { continue; // block with backlist
if (yacyCore.log.isInfo()) yacyCore.log.logInfo("remote search: filtered blacklisted url " + metadata.url() + " from peer " + target.getName()); }
continue; // block with backlist
}
final String urlRejectReason = Switchboard.getSwitchboard().crawlStacker.urlInAcceptedDomain(metadata.url()); final String urlRejectReason = Switchboard.getSwitchboard().crawlStacker.urlInAcceptedDomain(metadata.url());
if (urlRejectReason != null) { if (urlRejectReason != null) {
if (yacyCore.log.isInfo()) yacyCore.log.logInfo("remote search: rejected url '" + metadata.url() + "' (" + urlRejectReason + ") from peer " + target.getName()); if (yacyCore.log.isInfo()) yacyCore.log.logInfo("remote search: rejected url '" + metadata.url() + "' (" + urlRejectReason + ") from peer " + target.getName());
continue; // reject url outside of our domain continue; // reject url outside of our domain
} }
// save the url entry // save the url entry
Reference entry = urlEntry.word(); Reference entry = urlEntry.word();
if (entry == null) { if (entry == null) {
if (yacyCore.log.isWarning()) yacyCore.log.logWarning("remote search: no word attached from peer " + target.getName() + ", version " + target.getVersion()); if (yacyCore.log.isWarning()) yacyCore.log.logWarning("remote search: no word attached from peer " + target.getName() + ", version " + target.getVersion());
continue; // no word attached continue; // no word attached
} }
// the search-result-url transports all the attributes of word indexes // the search-result-url transports all the attributes of word indexes
if (!Base64Order.enhancedCoder.equal(entry.metadataHash(), urlEntry.hash())) { if (!Base64Order.enhancedCoder.equal(entry.metadataHash(), urlEntry.hash())) {
yacyCore.log.logInfo("remote search: url-hash " + new String(urlEntry.hash()) + " does not belong to word-attached-hash " + new String(entry.metadataHash()) + "; url = " + metadata.url() + " from peer " + target.getName()); yacyCore.log.logInfo("remote search: url-hash " + new String(urlEntry.hash()) + " does not belong to word-attached-hash " + new String(entry.metadataHash()) + "; url = " + metadata.url() + " from peer " + target.getName());
continue; // spammed continue; // spammed
} }
// passed all checks, store url // passed all checks, store url
try { try {
indexSegment.urlMetadata().store(urlEntry); indexSegment.urlMetadata().store(urlEntry);
crawlResults.stack(urlEntry, mySeed.hash.getBytes(), target.hash.getBytes(), EventOrigin.QUERIES); crawlResults.stack(urlEntry, mySeed.hash.getBytes(), target.hash.getBytes(), EventOrigin.QUERIES);
} catch (final IOException e) { } catch (final IOException e) {
yacyCore.log.logSevere("could not store search result", e); yacyCore.log.logSevere("could not store search result", e);
continue; // db-error continue; // db-error
} }
if (urlEntry.snippet() != null) { if (urlEntry.snippet() != null) {
// we don't store the snippets along the url entry, // we don't store the snippets along the url entry,
// because they are search-specific. // because they are search-specific.
// instead, they are placed in a snipped-search cache. // instead, they are placed in a snipped-search cache.
// System.out.println("--- RECEIVED SNIPPET '" + link.snippet() + "'"); // System.out.println("--- RECEIVED SNIPPET '" + link.snippet() + "'");
TextSnippet.storeToCache(wordhashes, new String(urlEntry.hash()), urlEntry.snippet()); TextSnippet.storeToCache(wordhashes, new String(urlEntry.hash()), urlEntry.snippet());
} }
// add the url entry to the word indexes // add the url entry to the word indexes
for (int m = 0; m < words; m++) { for (int m = 0; m < words; m++) {
try { try {
container[m].add(entry); container[m].add(entry);
} catch (RowSpaceExceededException e) { } catch (RowSpaceExceededException e) {
Log.logException(e); Log.logException(e);
break; break;
} }
} }
} }
// store remote result to local result container // store remote result to local result container
// insert one container into the search result buffer // insert one container into the search result buffer
// one is enough, only the references are used, not the word // one is enough, only the references are used, not the word
containerCache.add(container[0], false, target.getName() + "/" + target.hash, joincount); containerCache.add(container[0], false, target.getName() + "/" + target.hash, result.joincount);
// insert the containers to the index // insert the containers to the index
for (ReferenceContainer<WordReference> c: container) try { for (ReferenceContainer<WordReference> c: container) try {
@ -559,71 +499,180 @@ public final class yacyClient {
} catch (Exception e) { } catch (Exception e) {
Log.logException(e); Log.logException(e);
} }
yacyCore.log.logInfo("remote search: peer " + target.getName() + " sent " + container[0].size() + "/" + joincount + " references for " + (thisIsASecondarySearch ? "a secondary search" : "joined word queries")); boolean thisIsASecondarySearch = urlhashes.length() > 0;
assert !thisIsASecondarySearch || secondarySearchSuperviser == null;
yacyCore.log.logInfo("remote search: peer " + target.getName() + " sent " + container[0].size() + "/" + result.joincount + " references for " + (thisIsASecondarySearch ? "a secondary search" : "joined word queries"));
// integrate remote top-words/topics // integrate remote top-words/topics
final String references = result.get("references"); if (result.references != null && result.references.length > 0) {
if (references != null && references.length() > 0) { yacyCore.log.logInfo("remote search: peer " + target.getName() + " sent " + result.references.length + " topics");
yacyCore.log.logInfo("remote search: peer " + target.getName() + " sent topics: " + references);
// add references twice, so they can be counted (must have at least 2 entries) // add references twice, so they can be counted (must have at least 2 entries)
String[] rs = references.split(",");
synchronized (containerCache) { synchronized (containerCache) {
containerCache.addTopic(rs); containerCache.addTopic(result.references);
containerCache.addTopic(rs); containerCache.addTopic(result.references);
} }
} }
// read index abstract // read index abstract
if (secondarySearchSuperviser != null) { if (secondarySearchSuperviser != null) {
final Iterator<Map.Entry<String, String>> i = result.entrySet().iterator(); String wordhash;
Map.Entry<String, String> entry; String whacc = "";
String wordhash; ByteBuffer ci;
String whacc = ""; int ac = 0;
ByteBuffer ci; for (Map.Entry<byte[], String> abstractEntry: result.indexabstract.entrySet()) {
int ac = 0; wordhash = new String(abstractEntry.getKey());
abstractparser: while (i.hasNext()) { whacc += wordhash;
entry = i.next(); try {
if (entry.getKey().startsWith("indexabstract.")) { ci = new ByteBuffer(abstractEntry.getValue().getBytes("UTF-8"));
wordhash = entry.getKey().substring(14); } catch (UnsupportedEncodingException e) {
if (wordhash.charAt(0) == '[') break abstractparser; Log.logException(e);
whacc += wordhash; return -1;
try { }
ci = new ByteBuffer(entry.getValue().getBytes("UTF-8")); secondarySearchSuperviser.addAbstract(wordhash, ReferenceContainer.decompressIndex(ci, target.hash));
} catch (UnsupportedEncodingException e) { ac++;
Log.logException(e); }
return -1; if (ac > 0) {
} secondarySearchSuperviser.commitAbstract();
//System.out.println("DEBUG-ABSTRACTFETCH: for word hash " + wordhash + " received " + ci.toString()); yacyCore.log.logInfo("remote search: peer " + target.getName() + " sent " + ac + " index abstracts for words "+ whacc);
secondarySearchSuperviser.addAbstract(wordhash, ReferenceContainer.decompressIndex(ci, target.hash)); }
ac++; }
}
}
if (ac > 0) {
secondarySearchSuperviser.commitAbstract();
yacyCore.log.logInfo("remote search: peer " + target.getName() + " sent " + ac + " index abstracts for words "+ whacc);
}
}
// generate statistics // generate statistics
long searchtime; if (yacyCore.log.isFine()) yacyCore.log.logFine("SEARCH "
try { + result.urlcount
searchtime = Integer.parseInt(result.get("searchtime")); + " URLS FROM "
} catch (final NumberFormatException e) { + target.hash
searchtime = totalrequesttime; + ":"
} + target.getName()
if (yacyCore.log.isFine()) yacyCore.log.logFine("SEARCH " + ", score="
+ urlcount + target.selectscore
+ " URLS FROM " + ", searchtime=" + result.searchtime + ", netdelay="
+ target.hash + (totalrequesttime - result.searchtime) + ", references="
+ ":" + result.references);
+ target.getName() return result.urlcount;
+ ", score=" }
+ target.selectscore
+ ", searchtime=" + searchtime + ", netdelay=" public static SearchResult searchClient(
+ (totalrequesttime - searchtime) + ", references=" LinkedHashMap<String,ContentBody> parts,
+ result.get("references")); final yacySeed mySeed,
return urlcount; final String wordhashes,
} final String excludehashes,
final String urlhashes,
final Pattern prefer,
final Pattern filter,
final String language,
final String sitehash,
final String authorhash,
final int count,
final int maxDistance,
final boolean global,
final int partitions,
String hostname,
String hostaddress,
final SearchEvent.SecondarySearchSuperviser secondarySearchSuperviser,
final RankingProfile rankingProfile,
final Bitfield constraint
) throws IOException {
// send a search request to peer with remote Hash
// INPUT:
// iam : complete seed of the requesting peer
// youare : seed hash of the target peer, used for testing network stability
// key : transmission key for response
// search : a list of search words
// hsearch : a string of word hashes
// fwdep : forward depth. if "0" then peer may NOT ask another peer for more results
// fwden : forward deny, a list of seed hashes. They may NOT be target of forward hopping
// count : maximum number of wanted results
// global : if "true", then result may consist of answers from other peers
// partitions : number of remote peers that are asked (for evaluation of QPM)
// duetime : maximum time that a peer should spent to create a result
// send request
Map<String, String> resultMap = null;
parts.put("myseed", new StringBody((mySeed == null) ? "" : mySeed.genSeedStr(parts.get("key").toString())));
parts.put("count", new StringBody(Integer.toString(Math.max(10, count))));
parts.put("resource", new StringBody(((global) ? "global" : "local")));
parts.put("partitions", new StringBody(Integer.toString(partitions)));
parts.put("query", new StringBody(wordhashes));
parts.put("exclude", new StringBody(excludehashes));
parts.put("duetime", new StringBody("1000"));
parts.put("urls", new StringBody(urlhashes));
parts.put("prefer", new StringBody(prefer.toString()));
parts.put("filter", new StringBody(filter.toString()));
parts.put("language", new StringBody(language));
parts.put("sitehash", new StringBody(sitehash));
parts.put("authorhash", new StringBody(authorhash));
parts.put("ttl", new StringBody("0"));
parts.put("maxdist", new StringBody(Integer.toString(maxDistance)));
parts.put("profile", new StringBody(crypt.simpleEncode(rankingProfile.toExternalString())));
parts.put("constraint", new StringBody((constraint == null) ? "" : constraint.exportB64()));
if (secondarySearchSuperviser != null) parts.put("abstracts", new StringBody("auto"));
resultMap = FileUtils.table(HTTPConnector.getConnector(HTTPLoader.crawlerUserAgent).post(new MultiProtocolURI("http://" + hostaddress + "/yacy/search.html"), 60000, hostname, parts));
//resultMap = FileUtils.table(HTTPConnector.getConnector(HTTPLoader.crawlerUserAgent).post(new MultiProtocolURI("http://" + target.getClusterAddress() + "/yacy/search.html"), 60000, target.getHexHash() + ".yacyh", parts));
if (resultMap == null || resultMap.isEmpty()) throw new IOException("resultMap is NULL");
return new SearchResult(resultMap);
}
public static class SearchResult {
public String version; // version : application version of responder
public String uptime; // uptime : uptime in seconds of responder
public String fwhop; // hops (depth) of forwards that had been performed to construct this result
public String fwsrc; // peers that helped to construct this result
public String fwrec; // peers that would have helped to construct this result (recommendations)
public int urlcount; // number of returned LURL's for this search
public int joincount; //
public Map<byte[], Integer> indexcount; //
public long searchtime; // time that the peer actually spent to create the result
public String[] references; // search hints, the top-words
public List<URIMetadataRow> links; // LURLs of search
public Map<byte[], String> indexabstract; // index abstracts, a collection of url-hashes per word
public SearchResult(Map<String, String> resultMap) throws IOException {
try {
this.searchtime = Integer.parseInt(resultMap.get("searchtime"));
} catch (final NumberFormatException e) {
throw new IOException("wrong output format for searchtime: " + e.getMessage());
}
try {
this.joincount = Integer.parseInt(resultMap.get("joincount")); // the complete number of hits at remote site
} catch (final NumberFormatException e) {
throw new IOException("wrong output format for joincount: " + e.getMessage());
}
try {
this.urlcount = Integer.parseInt(resultMap.get("count")); // the number of hits that are returned in the result list
} catch (final NumberFormatException e) {
throw new IOException("wrong output format for count: " + e.getMessage());
}
this.fwhop = resultMap.get("fwhop");
this.fwsrc = resultMap.get("fwsrc");
this.fwrec = resultMap.get("fwrec");
// scan the result map for entries with special prefix
indexcount = new TreeMap<byte[], Integer>(Base64Order.enhancedCoder);
indexabstract = new TreeMap<byte[], String>(Base64Order.enhancedCoder);
for (Map.Entry<String, String> entry: resultMap.entrySet()) {
if (entry.getKey().startsWith("indexcount.")) {
indexcount.put(entry.getKey().substring(11).getBytes(), Integer.parseInt(entry.getValue()));
}
if (entry.getKey().startsWith("indexabstract.")) {
indexabstract.put(entry.getKey().substring(14).getBytes(), entry.getValue());
}
}
references = resultMap.get("references").split(",");
this.links = new ArrayList<URIMetadataRow>(this.urlcount);
for (int n = 0; n < this.urlcount; n++) {
// get one single search result
String resultLine = resultMap.get("resource" + n);
if (resultLine == null) continue;
URIMetadataRow urlEntry = URIMetadataRow.importEntry(resultLine);
if (urlEntry == null) continue;
this.links.add(urlEntry);
}
}
}
public static Map<String, String> permissionMessage(final yacySeedDB seedDB, final String targetHash) { public static Map<String, String> permissionMessage(final yacySeedDB seedDB, final String targetHash) {
// ask for allowed message size and attachement size // ask for allowed message size and attachement size
@ -1029,32 +1078,66 @@ public final class yacyClient {
} }
} }
public static void main(final String[] args) { public static void main(final String[] args) {
if(args.length > 1) { if(args.length > 2) {
System.out.println("yacyClient Test"); // search a remote peer. arguments:
try { // first arg: path to application home
final Switchboard sb = new Switchboard(new File(args[0]), new File(args[0]), "httpProxy.init", "DATA/SETTINGS/yacy.conf"); // second arg: address of target peer
/*final yacyCore core =*/ new yacyCore(sb); // third arg: search word or file name with list of search words
sb.loadSeedLists(); System.out.println("yacyClient Test");
final yacySeed target = sb.peers.getConnected(args[1]); File searchwordfile = new File(args[2]);
final byte[] wordhashe = Word.word2hash("test"); List<String> searchlines = new ArrayList<String>();
//System.out.println("permission=" + permissionMessage(args[1])); if (searchwordfile.exists()) {
Iterator<String> i;
final byte[] content = new MultiProtocolURI("http://" + target.getPublicAddress() + "/yacy/search.html" + try {
"?myseed=" + sb.peers.mySeed().genSeedStr(null) + i = FileUtils.strings(FileUtils.read(searchwordfile));
"&youare=" + target.hash + "&key=" + while (i.hasNext()) searchlines.add(i.next());
"&myseed=" + sb.peers.mySeed() .genSeedStr(null) + } catch (IOException e) {
"&count=10" + e.printStackTrace();
"&resource=global" + System.exit(-1);
"&query=" + new String(wordhashe) + }
"&network.unit.name=" + Switchboard.getSwitchboard().getConfig(SwitchboardConstants.NETWORK_NAME, yacySeed.DFLT_NETWORK_UNIT)).get(HTTPLoader.crawlerUserAgent, 10000); } else {
final Map<String, String> result = FileUtils.table(content); searchlines.add(args[2]);
System.out.println("Result=" + result.toString()); }
} catch (final Exception e) { for (String line: searchlines) {
Log.logException(e); final byte[] wordhashe = QueryParams.hashSet2hashString(Word.words2hashesHandles(QueryParams.cleanQuery(line)[0])).getBytes();
} long time = System.currentTimeMillis();
System.exit(0); SearchResult result;
try {
result = searchClient(
yacyNetwork.basicRequestParts((String) null, (String) null, "freeworld"),
null, // sb.peers.mySeed(),
new String(wordhashe),
"", // excludehashes,
"", // urlhashes,
Pattern.compile(""), // prefer,
Pattern.compile(".*"), // filter,
"", // language,
"", // sitehash,
"", // authorhash,
10, // count,
1000, // maxDistance,
true, //global,
16, // partitions,
"", args[1],
null, //secondarySearchSuperviser,
new RankingProfile(ContentDomain.TEXT), // rankingProfile,
null // constraint);
);
if (result == null) {
System.out.println("no response");
} else {
for (URIMetadataRow link: result.links) {
System.out.println(link.metadata().url().toNormalform(true, false));
System.out.println(link.snippet());
}
}
} catch (IOException e) {
e.printStackTrace();
}
System.out.println("Search Time: " + (System.currentTimeMillis() - time));
}
System.exit(0);
} else if(args.length == 1) { } else if(args.length == 1) {
System.out.println("wput Test"); System.out.println("wput Test");
// connection params // connection params
@ -1064,7 +1147,7 @@ public final class yacyClient {
} catch (final MalformedURLException e) { } catch (final MalformedURLException e) {
Log.logException(e); Log.logException(e);
} }
if(url == null) { if (url == null) {
System.exit(1); System.exit(1);
return; return;
} }

77
source/de/anomic/yacy/yacyNetwork.java
Unescape View File

@ -77,54 +77,13 @@ public class yacyNetwork {
return false; return false;
} }
// public static final List<Part> basicRequestPost(final Switchboard sb, final String targetHash, final String salt) { public static final LinkedHashMap<String,ContentBody> basicRequestParts(final Switchboard sb, final String targetHash, final String salt) {
// // put in all the essentials for routing and network authentication
// // generate a session key
// final ArrayList<Part> post = new ArrayList<Part>();
// post.add(new DefaultCharsetStringPart("key", salt));
//
// // just standard identification essentials
// post.add(new DefaultCharsetStringPart("iam", sb.peers.mySeed().hash));
// if (targetHash != null) post.add(new DefaultCharsetStringPart("youare", targetHash));
//
// // time information for synchronization
// post.add(new DefaultCharsetStringPart("mytime", DateFormatter.formatShortSecond(new Date())));
// post.add(new DefaultCharsetStringPart("myUTC", Long.toString(System.currentTimeMillis())));
//
// // network identification
// post.add(new DefaultCharsetStringPart(SwitchboardConstants.NETWORK_NAME, Switchboard.getSwitchboard().getConfig(SwitchboardConstants.NETWORK_NAME, yacySeed.DFLT_NETWORK_UNIT)));
//
// // authentication essentials
// final String authenticationControl = sb.getConfig("network.unit.protocol.control", "uncontrolled");
// final String authenticationMethod = sb.getConfig("network.unit.protocol.request.authentication.method", "");
// if ((authenticationControl.equals("controlled")) && (authenticationMethod.length() > 0)) {
// if (authenticationMethod.equals("salted-magic-sim")) {
// // generate an authentication essential using the salt, the iam-hash and the network magic
// final String magic = sb.getConfig("network.unit.protocol.request.authentication.essentials", "");
// final String md5 = Digest.encodeMD5Hex(salt + sb.peers.mySeed().hash + magic);
// post.add(new DefaultCharsetStringPart("magicmd5", md5));
// }
// }
//
// return post;
// }
public static final LinkedHashMap<String,ContentBody> basicRequestParts(final Switchboard sb, final String targetHash, final String salt) throws UnsupportedEncodingException {
// put in all the essentials for routing and network authentication // put in all the essentials for routing and network authentication
// generate a session key // generate a session key
final LinkedHashMap<String,ContentBody> parts = new LinkedHashMap<String,ContentBody>(); final LinkedHashMap<String,ContentBody> parts = basicRequestParts(sb.peers.mySeed().hash, targetHash, Switchboard.getSwitchboard().getConfig(SwitchboardConstants.NETWORK_NAME, yacySeed.DFLT_NETWORK_UNIT));
parts.put("key", new StringBody(salt)); try {
parts.put("key", new StringBody(salt));
// just standard identification essentials } catch (UnsupportedEncodingException e) {}
parts.put("iam", new StringBody(sb.peers.mySeed().hash));
if (targetHash != null) parts.put("youare", new StringBody(targetHash));
// time information for synchronization
parts.put("mytime", new StringBody(DateFormatter.formatShortSecond(new Date())));
parts.put("myUTC", new StringBody(Long.toString(System.currentTimeMillis())));
// network identification
parts.put(SwitchboardConstants.NETWORK_NAME, new StringBody(Switchboard.getSwitchboard().getConfig(SwitchboardConstants.NETWORK_NAME, yacySeed.DFLT_NETWORK_UNIT)));
// authentication essentials // authentication essentials
final String authenticationControl = sb.getConfig("network.unit.protocol.control", "uncontrolled"); final String authenticationControl = sb.getConfig("network.unit.protocol.control", "uncontrolled");
@ -134,11 +93,35 @@ public class yacyNetwork {
// generate an authentication essential using the salt, the iam-hash and the network magic // generate an authentication essential using the salt, the iam-hash and the network magic
final String magic = sb.getConfig("network.unit.protocol.request.authentication.essentials", ""); final String magic = sb.getConfig("network.unit.protocol.request.authentication.essentials", "");
final String md5 = Digest.encodeMD5Hex(salt + sb.peers.mySeed().hash + magic); final String md5 = Digest.encodeMD5Hex(salt + sb.peers.mySeed().hash + magic);
parts.put("magicmd5", new StringBody(md5)); try {
parts.put("magicmd5", new StringBody(md5));
} catch (UnsupportedEncodingException e) {}
} }
} }
return parts; return parts;
} }
public static final LinkedHashMap<String,ContentBody> basicRequestParts(String myHash, String targetHash, String networkName) {
// put in all the essentials for routing and network authentication
// generate a session key
final LinkedHashMap<String,ContentBody> parts = new LinkedHashMap<String,ContentBody>();
// just standard identification essentials
if (myHash != null)
try {
parts.put("iam", new StringBody(myHash));
if (targetHash != null) parts.put("youare", new StringBody(targetHash));
// time information for synchronization
parts.put("mytime", new StringBody(DateFormatter.formatShortSecond(new Date())));
parts.put("myUTC", new StringBody(Long.toString(System.currentTimeMillis())));
// network identification
parts.put(SwitchboardConstants.NETWORK_NAME, new StringBody(networkName));
} catch (UnsupportedEncodingException e) {}
return parts;
}
} }

4
source/de/anomic/yacy/yacySeed.java
Unescape View File

@ -752,9 +752,9 @@ public class yacySeed implements Cloneable {
// yacyCore.log.logFinest("genRemoteSeed: seedStr=" + seedStr + " key=" + key); // yacyCore.log.logFinest("genRemoteSeed: seedStr=" + seedStr + " key=" + key);
// check protocol and syntax of seed // check protocol and syntax of seed
if (seedStr == null) { return null; } if (seedStr == null || seedStr.length() == 0) return null;
final String seed = crypt.simpleDecode(seedStr, key); final String seed = crypt.simpleDecode(seedStr, key);
if (seed == null) { return null; } if (seed == null || seed.length() == 0) return null;
// extract hash // extract hash
final ConcurrentHashMap<String, String> dna = MapTools.string2map(seed, ","); final ConcurrentHashMap<String, String> dna = MapTools.string2map(seed, ",");

Write Preview
Loading…
Cancel
Save