boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

not using HTTPConnector anymore

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7674 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
sixcooler 14 years ago
parent 9f1f47ec67
commit a3e707283d
4 changed files with 62 additions and 23 deletions
Show Stats Download Patch File Download Diff File

43
source/de/anomic/yacy/yacyClient.java
Unescape View File

@ -62,7 +62,7 @@ import net.yacy.cora.document.RSSFeed;
import net.yacy.cora.document.RSSMessage; import net.yacy.cora.document.RSSMessage;
import net.yacy.cora.document.RSSReader; import net.yacy.cora.document.RSSReader;
import net.yacy.cora.document.UTF8; import net.yacy.cora.document.UTF8;
import net.yacy.cora.protocol.http.HTTPConnector; import net.yacy.cora.protocol.http.HTTPClient;
import net.yacy.cora.services.federated.opensearch.SRURSSConnector; import net.yacy.cora.services.federated.opensearch.SRURSSConnector;
import net.yacy.kelondro.data.meta.URIMetadataRow; import net.yacy.kelondro.data.meta.URIMetadataRow;
import net.yacy.kelondro.data.word.Word; import net.yacy.kelondro.data.word.Word;
@ -98,10 +98,14 @@ public final class yacyClient {
private static byte[] postToFile(final yacySeed target, final String filename, final Map<String,ContentBody> parts, final int timeout) throws IOException { private static byte[] postToFile(final yacySeed target, final String filename, final Map<String,ContentBody> parts, final int timeout) throws IOException {
return HTTPConnector.getConnector(MultiProtocolURI.yacybotUserAgent).post(new MultiProtocolURI("http://" + target.getClusterAddress() + "/yacy/" + filename), timeout, target.getHexHash() + ".yacyh", parts); // return HTTPConnector.getConnector(MultiProtocolURI.yacybotUserAgent).post(new MultiProtocolURI("http://" + target.getClusterAddress() + "/yacy/" + filename), timeout, target.getHexHash() + ".yacyh", parts);
final HTTPClient httpClient = new HTTPClient(MultiProtocolURI.yacybotUserAgent, timeout);
return httpClient.POSTbytes(new MultiProtocolURI("http://" + target.getClusterAddress() + "/yacy/" + filename), target.getHexHash() + ".yacyh", parts, false);
} }
private static byte[] postToFile(final yacySeedDB seedDB, final String targetHash, final String filename, final Map<String,ContentBody> parts, final int timeout) throws IOException { private static byte[] postToFile(final yacySeedDB seedDB, final String targetHash, final String filename, final Map<String,ContentBody> parts, final int timeout) throws IOException {
return HTTPConnector.getConnector(MultiProtocolURI.yacybotUserAgent).post(new MultiProtocolURI("http://" + targetAddress(seedDB, targetHash) + "/yacy/" + filename), timeout, yacySeed.b64Hash2hexHash(targetHash)+ ".yacyh", parts); // return HTTPConnector.getConnector(MultiProtocolURI.yacybotUserAgent).post(new MultiProtocolURI("http://" + targetAddress(seedDB, targetHash) + "/yacy/" + filename), timeout, yacySeed.b64Hash2hexHash(targetHash)+ ".yacyh", parts);
final HTTPClient httpClient = new HTTPClient(MultiProtocolURI.yacybotUserAgent, timeout);
return httpClient.POSTbytes(new MultiProtocolURI("http://" + targetAddress(seedDB, targetHash) + "/yacy/" + filename), yacySeed.b64Hash2hexHash(targetHash)+ ".yacyh", parts, false);
} }
/** /**
@ -137,7 +141,9 @@ public final class yacyClient {
parts.put("seed", UTF8.StringBody(mySeed.genSeedStr(salt))); parts.put("seed", UTF8.StringBody(mySeed.genSeedStr(salt)));
// send request // send request
final long start = System.currentTimeMillis(); final long start = System.currentTimeMillis();
final byte[] content = HTTPConnector.getConnector(MultiProtocolURI.yacybotUserAgent).post(new MultiProtocolURI("http://" + address + "/yacy/hello.html"), 30000, yacySeed.b64Hash2hexHash(otherHash) + ".yacyh", parts); // final byte[] content = HTTPConnector.getConnector(MultiProtocolURI.yacybotUserAgent).post(new MultiProtocolURI("http://" + address + "/yacy/hello.html"), 30000, yacySeed.b64Hash2hexHash(otherHash) + ".yacyh", parts);
final HTTPClient httpClient = new HTTPClient(MultiProtocolURI.yacybotUserAgent, 30000);
final byte[] content = httpClient.POSTbytes(new MultiProtocolURI("http://" + address + "/yacy/hello.html"), yacySeed.b64Hash2hexHash(otherHash) + ".yacyh", parts, false);
yacyCore.log.logInfo("yacyClient.hello thread '" + Thread.currentThread().getName() + "' contacted peer at " + address + ", received " + ((content == null) ? "null" : content.length) + " bytes, time = " + (System.currentTimeMillis() - start) + " milliseconds"); yacyCore.log.logInfo("yacyClient.hello thread '" + Thread.currentThread().getName() + "' contacted peer at " + address + ", received " + ((content == null) ? "null" : content.length) + " bytes, time = " + (System.currentTimeMillis() - start) + " milliseconds");
result = FileUtils.table(content); result = FileUtils.table(content);
} catch (final Exception e) { } catch (final Exception e) {
@ -341,7 +347,9 @@ public final class yacyClient {
parts.put("call", UTF8.StringBody("remotecrawl")); parts.put("call", UTF8.StringBody("remotecrawl"));
parts.put("count", UTF8.StringBody(Integer.toString(maxCount))); parts.put("count", UTF8.StringBody(Integer.toString(maxCount)));
parts.put("time", UTF8.StringBody(Long.toString(maxTime))); parts.put("time", UTF8.StringBody(Long.toString(maxTime)));
final byte[] result = HTTPConnector.getConnector(MultiProtocolURI.yacybotUserAgent).post(new MultiProtocolURI("http://" + target.getClusterAddress() + "/yacy/urls.xml"), (int) maxTime, target.getHexHash() + ".yacyh", parts); // final byte[] result = HTTPConnector.getConnector(MultiProtocolURI.yacybotUserAgent).post(new MultiProtocolURI("http://" + target.getClusterAddress() + "/yacy/urls.xml"), (int) maxTime, target.getHexHash() + ".yacyh", parts);
final HTTPClient httpClient = new HTTPClient(MultiProtocolURI.yacybotUserAgent, (int) maxTime);
final byte[] result = httpClient.POSTbytes(new MultiProtocolURI("http://" + target.getClusterAddress() + "/yacy/urls.xml"), target.getHexHash() + ".yacyh", parts, false);
final RSSReader reader = RSSReader.parse(RSSFeed.DEFAULT_MAXSIZE, result); final RSSReader reader = RSSReader.parse(RSSFeed.DEFAULT_MAXSIZE, result);
if (reader == null) { if (reader == null) {
yacyCore.log.logWarning("yacyClient.queryRemoteCrawlURLs failed asking peer '" + target.getName() + "': probably bad response from remote peer (1), reader == null"); yacyCore.log.logWarning("yacyClient.queryRemoteCrawlURLs failed asking peer '" + target.getName() + "': probably bad response from remote peer (1), reader == null");
@ -644,9 +652,12 @@ public final class yacyClient {
parts.put("profile", UTF8.StringBody(crypt.simpleEncode(rankingProfile.toExternalString()))); parts.put("profile", UTF8.StringBody(crypt.simpleEncode(rankingProfile.toExternalString())));
parts.put("constraint", UTF8.StringBody((constraint == null) ? "" : constraint.exportB64())); parts.put("constraint", UTF8.StringBody((constraint == null) ? "" : constraint.exportB64()));
if (secondarySearchSuperviser != null) parts.put("abstracts", UTF8.StringBody("auto")); if (secondarySearchSuperviser != null) parts.put("abstracts", UTF8.StringBody("auto"));
resultMap = FileUtils.table(HTTPConnector.getConnector(MultiProtocolURI.yacybotUserAgent).post(new MultiProtocolURI("http://" + hostaddress + "/yacy/search.html"), 60000, hostname, parts)); // resultMap = FileUtils.table(HTTPConnector.getConnector(MultiProtocolURI.yacybotUserAgent).post(new MultiProtocolURI("http://" + hostaddress + "/yacy/search.html"), 60000, hostname, parts));
//resultMap = FileUtils.table(HTTPConnector.getConnector(MultiProtocolURI.crawlerUserAgent).post(new MultiProtocolURI("http://" + target.getClusterAddress() + "/yacy/search.html"), 60000, target.getHexHash() + ".yacyh", parts)); //resultMap = FileUtils.table(HTTPConnector.getConnector(MultiProtocolURI.crawlerUserAgent).post(new MultiProtocolURI("http://" + target.getClusterAddress() + "/yacy/search.html"), 60000, target.getHexHash() + ".yacyh", parts));
final HTTPClient httpClient = new HTTPClient(MultiProtocolURI.yacybotUserAgent, 60000);
resultMap = FileUtils.table(httpClient.POSTbytes(new MultiProtocolURI("http://" + hostaddress + "/yacy/search.html"), hostname, parts, false));
// evaluate request result // evaluate request result
if (resultMap == null || resultMap.isEmpty()) throw new IOException("resultMap is NULL"); if (resultMap == null || resultMap.isEmpty()) throw new IOException("resultMap is NULL");
try { try {
@ -791,7 +802,9 @@ public final class yacyClient {
parts.put("wordh", UTF8.StringBody(wordhashes)); parts.put("wordh", UTF8.StringBody(wordhashes));
parts.put("lurlEntry", UTF8.StringBody(((entry == null) ? "" : crypt.simpleEncode(entry.toString(), salt)))); parts.put("lurlEntry", UTF8.StringBody(((entry == null) ? "" : crypt.simpleEncode(entry.toString(), salt))));
// send request // send request
final byte[] content = HTTPConnector.getConnector(MultiProtocolURI.yacybotUserAgent).post(new MultiProtocolURI("http://" + address + "/yacy/crawlReceipt.html"), 10000, target.getHexHash() + ".yacyh", parts); // final byte[] content = HTTPConnector.getConnector(MultiProtocolURI.yacybotUserAgent).post(new MultiProtocolURI("http://" + address + "/yacy/crawlReceipt.html"), 10000, target.getHexHash() + ".yacyh", parts);
final HTTPClient httpClient = new HTTPClient(MultiProtocolURI.yacybotUserAgent, 10000);
final byte[] content = httpClient.POSTbytes(new MultiProtocolURI("http://" + address + "/yacy/crawlReceipt.html"), target.getHexHash() + ".yacyh", parts, false);
return FileUtils.table(content); return FileUtils.table(content);
} catch (final Exception e) { } catch (final Exception e) {
// most probably a network time-out exception // most probably a network time-out exception
@ -930,7 +943,9 @@ public final class yacyClient {
parts.put("wordc", UTF8.StringBody(Integer.toString(indexes.size()))); parts.put("wordc", UTF8.StringBody(Integer.toString(indexes.size())));
parts.put("entryc", UTF8.StringBody(Integer.toString(indexcount))); parts.put("entryc", UTF8.StringBody(Integer.toString(indexcount)));
parts.put("indexes", UTF8.StringBody(entrypost.toString())); parts.put("indexes", UTF8.StringBody(entrypost.toString()));
final byte[] content = HTTPConnector.getConnector(MultiProtocolURI.yacybotUserAgent).post(new MultiProtocolURI("http://" + address + "/yacy/transferRWI.html"), timeout, targetSeed.getHexHash() + ".yacyh", parts, gzipBody); // final byte[] content = HTTPConnector.getConnector(MultiProtocolURI.yacybotUserAgent).post(new MultiProtocolURI("http://" + address + "/yacy/transferRWI.html"), timeout, targetSeed.getHexHash() + ".yacyh", parts, gzipBody);
final HTTPClient httpClient = new HTTPClient(MultiProtocolURI.yacybotUserAgent, timeout);
final byte[] content = httpClient.POSTbytes(new MultiProtocolURI("http://" + address + "/yacy/transferRWI.html"), targetSeed.getHexHash() + ".yacyh", parts, gzipBody);
final Iterator<String> v = FileUtils.strings(content); final Iterator<String> v = FileUtils.strings(content);
// this should return a list of urlhashes that are unknown // this should return a list of urlhashes that are unknown
@ -974,7 +989,9 @@ public final class yacyClient {
} }
try { try {
parts.put("urlc", UTF8.StringBody(Integer.toString(urlc))); parts.put("urlc", UTF8.StringBody(Integer.toString(urlc)));
final byte[] content = HTTPConnector.getConnector(MultiProtocolURI.yacybotUserAgent).post(new MultiProtocolURI("http://" + address + "/yacy/transferURL.html"), timeout, targetSeed.getHexHash() + ".yacyh", parts, gzipBody); // final byte[] content = HTTPConnector.getConnector(MultiProtocolURI.yacybotUserAgent).post(new MultiProtocolURI("http://" + address + "/yacy/transferURL.html"), timeout, targetSeed.getHexHash() + ".yacyh", parts, gzipBody);
final HTTPClient httpClient = new HTTPClient(MultiProtocolURI.yacybotUserAgent, timeout);
final byte[] content = httpClient.POSTbytes(new MultiProtocolURI("http://" + address + "/yacy/transferURL.html"), targetSeed.getHexHash() + ".yacyh", parts, gzipBody);
final Iterator<String> v = FileUtils.strings(content); final Iterator<String> v = FileUtils.strings(content);
final Map<String, String> result = FileUtils.table(v); final Map<String, String> result = FileUtils.table(v);
@ -996,7 +1013,9 @@ public final class yacyClient {
if (address == null) { address = "localhost:8090"; } if (address == null) { address = "localhost:8090"; }
try { try {
final Map<String,ContentBody> parts = yacyNetwork.basicRequestParts(Switchboard.getSwitchboard(), targetSeed.hash, salt); final Map<String,ContentBody> parts = yacyNetwork.basicRequestParts(Switchboard.getSwitchboard(), targetSeed.hash, salt);
final byte[] content = HTTPConnector.getConnector(MultiProtocolURI.yacybotUserAgent).post(new MultiProtocolURI("http://" + address + "/yacy/profile.html"), 5000, targetSeed.getHexHash() + ".yacyh", parts); // final byte[] content = HTTPConnector.getConnector(MultiProtocolURI.yacybotUserAgent).post(new MultiProtocolURI("http://" + address + "/yacy/profile.html"), 5000, targetSeed.getHexHash() + ".yacyh", parts);
final HTTPClient httpclient = new HTTPClient(MultiProtocolURI.yacybotUserAgent, 5000);
final byte[] content = httpclient.POSTbytes(new MultiProtocolURI("http://" + address + "/yacy/profile.html"), targetSeed.getHexHash() + ".yacyh", parts, false);
return FileUtils.table(content); return FileUtils.table(content);
} catch (final Exception e) { } catch (final Exception e) {
yacyCore.log.logWarning("yacyClient.getProfile error:" + e.getMessage()); yacyCore.log.logWarning("yacyClient.getProfile error:" + e.getMessage());
@ -1084,7 +1103,9 @@ public final class yacyClient {
newpost.put("purpose", UTF8.StringBody("crcon")); newpost.put("purpose", UTF8.StringBody("crcon"));
byte[] res; byte[] res;
try { try {
res = HTTPConnector.getConnector(MultiProtocolURI.yacybotUserAgent).post(url, timeout, vhost, newpost, true); // res = HTTPConnector.getConnector(MultiProtocolURI.yacybotUserAgent).post(url, timeout, vhost, newpost, true);
final HTTPClient httpClient = new HTTPClient(MultiProtocolURI.yacybotUserAgent, timeout);
res = httpClient.POSTbytes(url, vhost, newpost, true);
System.out.println(UTF8.String(res)); System.out.println(UTF8.String(res));
} catch (IOException e1) { } catch (IOException e1) {
Log.logException(e1); Log.logException(e1);

23
source/net/yacy/cora/protocol/http/HTTPClient.java
Unescape View File

@ -113,6 +113,12 @@ public class HTTPClient {
HttpProtocolParams.setUserAgent(httpClient.getParams(), defaultAgent); HttpProtocolParams.setUserAgent(httpClient.getParams(), defaultAgent);
} }
public HTTPClient(final String defaultAgent, final int timeout) {
super();
HttpProtocolParams.setUserAgent(httpClient.getParams(), defaultAgent);
this.timeout = timeout;
}
public static void setDefaultUserAgent(final String defaultAgent) { public static void setDefaultUserAgent(final String defaultAgent) {
HttpProtocolParams.setUserAgent(httpClient.getParams(), defaultAgent); HttpProtocolParams.setUserAgent(httpClient.getParams(), defaultAgent);
} }
@ -269,6 +275,14 @@ public class HTTPClient {
public byte[] GETbytes(final String uri) throws IOException { public byte[] GETbytes(final String uri) throws IOException {
return GETbytes(uri, Long.MAX_VALUE); return GETbytes(uri, Long.MAX_VALUE);
} }
/**
* This method GETs a page from the server.
*
* @param uri the url to get
* @return content bytes
* @throws IOException
*/
public byte[] GETbytes(final MultiProtocolURI url) throws IOException { public byte[] GETbytes(final MultiProtocolURI url) throws IOException {
return GETbytes(url, Long.MAX_VALUE); return GETbytes(url, Long.MAX_VALUE);
} }
@ -284,7 +298,16 @@ public class HTTPClient {
public byte[] GETbytes(final String uri, long maxBytes) throws IOException { public byte[] GETbytes(final String uri, long maxBytes) throws IOException {
return GETbytes(new MultiProtocolURI(uri), maxBytes); return GETbytes(new MultiProtocolURI(uri), maxBytes);
} }
/**
* This method GETs a page from the server.
*
* @param uri the url to get
* @param maxBytes to get
* @return content bytes
* @throws IOException
*/
public byte[] GETbytes(final MultiProtocolURI url, long maxBytes) throws IOException { public byte[] GETbytes(final MultiProtocolURI url, long maxBytes) throws IOException {
boolean localhost = url.getHost().equals("localhost"); boolean localhost = url.getHost().equals("localhost");
String urix = url.toNormalform(true, false, !localhost, false); String urix = url.toNormalform(true, false, !localhost, false);

13
source/net/yacy/cora/protocol/http/HTTPConnector.java
Unescape View File

@ -36,6 +36,7 @@ import org.apache.http.entity.mime.content.ContentBody;
/** /**
* This Connector is a convenience class to access the protocol-specific http client class. * This Connector is a convenience class to access the protocol-specific http client class.
* @deprecated this class is not used anymore
*/ */
public class HTTPConnector { public class HTTPConnector {
@ -79,17 +80,9 @@ public class HTTPConnector {
* @throws IOException * @throws IOException
*/ */
public byte[] post(final MultiProtocolURI url, final int timeout, final String vhost, final Map<String, ContentBody> post, final boolean usegzip) throws IOException { public byte[] post(final MultiProtocolURI url, final int timeout, final String vhost, final Map<String, ContentBody> post, final boolean usegzip) throws IOException {
final HTTPClient client = new HTTPClient(); final HTTPClient client = new HTTPClient(this.userAgent);
client.setTimout(timeout); client.setTimout(timeout);
client.setUserAgent(this.userAgent); return client.POSTbytes(url, vhost, post, usegzip);
client.setHost(vhost);
byte[] b;
try {
b = client.POSTbytes(url, url.getHost(), post, usegzip);
} finally {
client.finish();
}
return b;
} }
} }

6
source/net/yacy/cora/services/federated/opensearch/SRURSSConnector.java
Unescape View File

@ -43,7 +43,7 @@ import net.yacy.cora.document.RSSFeed;
import net.yacy.cora.document.RSSMessage; import net.yacy.cora.document.RSSMessage;
import net.yacy.cora.document.RSSReader; import net.yacy.cora.document.RSSReader;
import net.yacy.cora.document.UTF8; import net.yacy.cora.document.UTF8;
import net.yacy.cora.protocol.http.HTTPConnector; import net.yacy.cora.protocol.http.HTTPClient;
import net.yacy.cora.services.federated.SearchAccumulator; import net.yacy.cora.services.federated.SearchAccumulator;
import net.yacy.cora.services.federated.SearchHub; import net.yacy.cora.services.federated.SearchHub;
@ -202,7 +202,9 @@ public class SRURSSConnector extends Thread implements SearchAccumulator {
parts.put("verify", cacheStrategy == null ? UTF8.StringBody("false") : UTF8.StringBody(cacheStrategy.toName())); parts.put("verify", cacheStrategy == null ? UTF8.StringBody("false") : UTF8.StringBody(cacheStrategy.toName()));
parts.put("resource", UTF8.StringBody(global ? "global" : "local")); parts.put("resource", UTF8.StringBody(global ? "global" : "local"));
parts.put("nav", UTF8.StringBody("none")); parts.put("nav", UTF8.StringBody("none"));
result = HTTPConnector.getConnector(userAgent == null ? MultiProtocolURI.yacybotUserAgent : userAgent).post(new MultiProtocolURI(rssSearchServiceURL), (int) timeout, uri.getHost(), parts); // result = HTTPConnector.getConnector(userAgent == null ? MultiProtocolURI.yacybotUserAgent : userAgent).post(new MultiProtocolURI(rssSearchServiceURL), (int) timeout, uri.getHost(), parts);
final HTTPClient httpClient = new HTTPClient(userAgent == null ? MultiProtocolURI.yacybotUserAgent : userAgent, (int) timeout);
result = httpClient.POSTbytes(new MultiProtocolURI(rssSearchServiceURL), uri.getHost(), parts, false);
final RSSReader reader = RSSReader.parse(RSSFeed.DEFAULT_MAXSIZE, result); final RSSReader reader = RSSReader.parse(RSSFeed.DEFAULT_MAXSIZE, result);
if (reader == null) { if (reader == null) {

Write Preview
Loading…
Cancel
Save