diff --git a/source/de/anomic/yacy/yacyClient.java b/source/de/anomic/yacy/yacyClient.java index 96a1e0ac8..4cd0a17c6 100644 --- a/source/de/anomic/yacy/yacyClient.java +++ b/source/de/anomic/yacy/yacyClient.java @@ -60,6 +60,7 @@ import java.util.regex.Pattern; import net.yacy.cora.document.RSSFeed; import net.yacy.cora.document.RSSMessage; import net.yacy.cora.document.RSSReader; +import net.yacy.cora.protocol.ByteArrayBody; import net.yacy.cora.protocol.HttpConnector; import net.yacy.cora.services.Search; import net.yacy.kelondro.data.meta.URIMetadataRow; @@ -77,7 +78,7 @@ import net.yacy.kelondro.util.ByteBuffer; import net.yacy.kelondro.util.FileUtils; import net.yacy.repository.Blacklist; -import org.apache.commons.httpclient.methods.multipart.ByteArrayPartSource; +//import org.apache.commons.httpclient.methods.multipart.ByteArrayPartSource; import org.apache.commons.httpclient.methods.multipart.Part; import org.apache.http.entity.mime.content.ContentBody; import org.apache.http.entity.mime.content.StringBody; @@ -85,7 +86,6 @@ import org.apache.http.entity.mime.content.StringBody; import de.anomic.crawler.ResultURLs; import de.anomic.crawler.retrieval.EventOrigin; import de.anomic.crawler.retrieval.HTTPLoader; -import de.anomic.http.client.DefaultCharsetFilePart; import de.anomic.http.client.DefaultCharsetStringPart; import de.anomic.http.client.Client; import de.anomic.http.server.HeaderFramework; @@ -147,10 +147,10 @@ public final class yacyClient { // final List post = yacyNetwork.basicRequestPost(Switchboard.getSwitchboard(), null, salt); try { // generate request - final LinkedHashMap parts = yacyNetwork.basicRequestParts(Switchboard.getSwitchboard(), null, salt); // post.add(new DefaultCharsetStringPart("count", "20")); - parts.put("count", new StringBody("20")); // post.add(new DefaultCharsetStringPart("seed", mySeed.genSeedStr(salt))); + final LinkedHashMap parts = yacyNetwork.basicRequestParts(Switchboard.getSwitchboard(), null, salt); + parts.put("count", new StringBody("20")); parts.put("seed", new StringBody(mySeed.genSeedStr(salt))); // send request final long start = System.currentTimeMillis(); @@ -336,15 +336,20 @@ public final class yacyClient { // prepare request final String salt = crypt.randomSalt(); - final List post = yacyNetwork.basicRequestPost(Switchboard.getSwitchboard(), target.hash, salt); - post.add(new DefaultCharsetStringPart("call", "remotecrawl")); - post.add(new DefaultCharsetStringPart("count", Integer.toString(maxCount))); - post.add(new DefaultCharsetStringPart("time", Long.toString(maxTime))); +// final List post = yacyNetwork.basicRequestPost(Switchboard.getSwitchboard(), target.hash, salt); +// post.add(new DefaultCharsetStringPart("call", "remotecrawl")); +// post.add(new DefaultCharsetStringPart("count", Integer.toString(maxCount))); +// post.add(new DefaultCharsetStringPart("time", Long.toString(maxTime))); // send request try { /* a long time-out is needed */ - final byte[] result = HttpConnector.wput("http://" + target.getClusterAddress() + "/yacy/urls.xml", target.getHexHash() + ".yacyh", post, (int) maxTime); +// final byte[] result = HttpConnector.wput("http://" + target.getClusterAddress() + "/yacy/urls.xml", target.getHexHash() + ".yacyh", post, (int) maxTime); + final LinkedHashMap parts = yacyNetwork.basicRequestParts(Switchboard.getSwitchboard(), target.hash, salt); + parts.put("call", new StringBody("remotecrawl")); + parts.put("count", new StringBody(Integer.toString(maxCount))); + parts.put("time", new StringBody(Long.toString(maxTime))); + final byte[] result = HttpConnector.wput("http://" + target.getClusterAddress() + "/yacy/urls.xml", target.getHexHash() + ".yacyh", parts, (int) maxTime); final RSSReader reader = RSSReader.parse(result); if (reader == null) { yacyCore.log.logWarning("yacyClient.queryRemoteCrawlURLs failed asking peer '" + target.getName() + "': probably bad response from remote peer (1), reader == null"); @@ -417,25 +422,25 @@ public final class yacyClient { // prepare request final String salt = crypt.randomSalt(); - final List post = yacyNetwork.basicRequestPost(Switchboard.getSwitchboard(), target.hash, salt); - post.add(new DefaultCharsetStringPart("myseed", mySeed.genSeedStr(salt))); - post.add(new DefaultCharsetStringPart("count", Integer.toString(Math.max(10, count)))); - post.add(new DefaultCharsetStringPart("resource", ((global) ? "global" : "local"))); - post.add(new DefaultCharsetStringPart("partitions", Integer.toString(partitions))); - post.add(new DefaultCharsetStringPart("query", wordhashes)); - post.add(new DefaultCharsetStringPart("exclude", excludehashes)); - post.add(new DefaultCharsetStringPart("duetime", "1000")); - post.add(new DefaultCharsetStringPart("urls", urlhashes)); - post.add(new DefaultCharsetStringPart("prefer", prefer.toString())); - post.add(new DefaultCharsetStringPart("filter", filter.toString())); - post.add(new DefaultCharsetStringPart("language", language)); - post.add(new DefaultCharsetStringPart("sitehash", sitehash)); - post.add(new DefaultCharsetStringPart("authorhash", authorhash)); - post.add(new DefaultCharsetStringPart("ttl", "0")); - post.add(new DefaultCharsetStringPart("maxdist", Integer.toString(maxDistance))); - post.add(new DefaultCharsetStringPart("profile", crypt.simpleEncode(rankingProfile.toExternalString()))); - post.add(new DefaultCharsetStringPart("constraint", (constraint == null) ? "" : constraint.exportB64())); - if (secondarySearchSuperviser != null) post.add(new DefaultCharsetStringPart("abstracts", "auto")); +// final List post = yacyNetwork.basicRequestPost(Switchboard.getSwitchboard(), target.hash, salt); +// post.add(new DefaultCharsetStringPart("myseed", mySeed.genSeedStr(salt))); +// post.add(new DefaultCharsetStringPart("count", Integer.toString(Math.max(10, count)))); +// post.add(new DefaultCharsetStringPart("resource", ((global) ? "global" : "local"))); +// post.add(new DefaultCharsetStringPart("partitions", Integer.toString(partitions))); +// post.add(new DefaultCharsetStringPart("query", wordhashes)); +// post.add(new DefaultCharsetStringPart("exclude", excludehashes)); +// post.add(new DefaultCharsetStringPart("duetime", "1000")); +// post.add(new DefaultCharsetStringPart("urls", urlhashes)); +// post.add(new DefaultCharsetStringPart("prefer", prefer.toString())); +// post.add(new DefaultCharsetStringPart("filter", filter.toString())); +// post.add(new DefaultCharsetStringPart("language", language)); +// post.add(new DefaultCharsetStringPart("sitehash", sitehash)); +// post.add(new DefaultCharsetStringPart("authorhash", authorhash)); +// post.add(new DefaultCharsetStringPart("ttl", "0")); +// post.add(new DefaultCharsetStringPart("maxdist", Integer.toString(maxDistance))); +// post.add(new DefaultCharsetStringPart("profile", crypt.simpleEncode(rankingProfile.toExternalString()))); +// post.add(new DefaultCharsetStringPart("constraint", (constraint == null) ? "" : constraint.exportB64())); +// if (secondarySearchSuperviser != null) post.add(new DefaultCharsetStringPart("abstracts", "auto")); final long timestamp = System.currentTimeMillis(); boolean thisIsASecondarySearch = urlhashes.length() > 0; assert !thisIsASecondarySearch || secondarySearchSuperviser == null; @@ -443,7 +448,28 @@ public final class yacyClient { // send request Map result = null; try { - result = FileUtils.table(HttpConnector.wput("http://" + target.getClusterAddress() + "/yacy/search.html", target.getHexHash() + ".yacyh", post, 60000)); +// result = FileUtils.table(HttpConnector.wput("http://" + target.getClusterAddress() + "/yacy/search.html", target.getHexHash() + ".yacyh", post, 60000)); + final LinkedHashMap parts = yacyNetwork.basicRequestParts(Switchboard.getSwitchboard(), target.hash, salt); + parts.put("myseed", new StringBody(mySeed.genSeedStr(salt))); + parts.put("count", new StringBody(Integer.toString(Math.max(10, count)))); + parts.put("resource", new StringBody(((global) ? "global" : "local"))); + parts.put("partitions", new StringBody(Integer.toString(partitions))); + parts.put("query", new StringBody(wordhashes)); + parts.put("exclude", new StringBody(excludehashes)); + parts.put("duetime", new StringBody("1000")); + parts.put("urls", new StringBody(urlhashes)); + parts.put("prefer", new StringBody(prefer.toString())); + parts.put("filter", new StringBody(filter.toString())); + parts.put("language", new StringBody(language)); + parts.put("sitehash", new StringBody(sitehash)); + parts.put("authorhash", new StringBody(authorhash)); + parts.put("ttl", new StringBody("0")); + parts.put("maxdist", new StringBody(Integer.toString(maxDistance))); + parts.put("profile", new StringBody(crypt.simpleEncode(rankingProfile.toExternalString()))); + parts.put("constraint", new StringBody((constraint == null) ? "" : constraint.exportB64())); + if (secondarySearchSuperviser != null) + parts.put("abstracts", new StringBody("auto")); + result = FileUtils.table(HttpConnector.wput("http://" + target.getClusterAddress() + "/yacy/search.html", target.getHexHash() + ".yacyh", parts, 60000)); } catch (final IOException e) { yacyCore.log.logInfo("SEARCH failed, Peer: " + target.hash + ":" + target.getName() + " (" + e.getMessage() + "), score=" + target.selectscore); //yacyCore.peerActions.peerDeparture(target, "search request to peer created io exception: " + e.getMessage()); @@ -706,16 +732,23 @@ public final class yacyClient { // prepare request final String salt = crypt.randomSalt(); - final List post = yacyNetwork.basicRequestPost(Switchboard.getSwitchboard(), null, salt); - post.add(new DefaultCharsetStringPart("process", "permission")); - post.add(new DefaultCharsetStringPart("purpose", "crcon")); - post.add(new DefaultCharsetStringPart("filename", filename)); - post.add(new DefaultCharsetStringPart("filesize", Long.toString(filesize))); - post.add(new DefaultCharsetStringPart("can-send-protocol", "http")); +// final List post = yacyNetwork.basicRequestPost(Switchboard.getSwitchboard(), null, salt); +// post.add(new DefaultCharsetStringPart("process", "permission")); +// post.add(new DefaultCharsetStringPart("purpose", "crcon")); +// post.add(new DefaultCharsetStringPart("filename", filename)); +// post.add(new DefaultCharsetStringPart("filesize", Long.toString(filesize))); +// post.add(new DefaultCharsetStringPart("can-send-protocol", "http")); // send request try { - final byte[] content = HttpConnector.wput("http://" + targetAddress + "/yacy/transfer.html", targetAddress, post, 10000); + // final byte[] content = HttpConnector.wput("http://" + targetAddress + "/yacy/transfer.html", targetAddress, post, 10000); + final LinkedHashMap parts = yacyNetwork.basicRequestParts(Switchboard.getSwitchboard(), null, salt); + parts.put("process", new StringBody("permission")); + parts.put("purpose", new StringBody("crcon")); + parts.put("filename", new StringBody(filename)); + parts.put("filesize", new StringBody(Long.toString(filesize))); + parts.put("can-send-protocol", new StringBody("http")); + final byte[] content = HttpConnector.wput("http://" + targetAddress + "/yacy/transfer.html", targetAddress, parts, 10000); final Map result = FileUtils.table(content); return result; } catch (final Exception e) { @@ -729,17 +762,25 @@ public final class yacyClient { // prepare request final String salt = crypt.randomSalt(); - final List post = yacyNetwork.basicRequestPost(Switchboard.getSwitchboard(), null, salt); - post.add(new DefaultCharsetStringPart("process", "store")); - post.add(new DefaultCharsetStringPart("purpose", "crcon")); - post.add(new DefaultCharsetStringPart("filesize", Long.toString(file.length))); - post.add(new DefaultCharsetStringPart("md5", Digest.encodeMD5Hex(file))); - post.add(new DefaultCharsetStringPart("access", access)); - post.add(new DefaultCharsetFilePart("filename", new ByteArrayPartSource(filename, file))); +// final List post = yacyNetwork.basicRequestPost(Switchboard.getSwitchboard(), null, salt); +// post.add(new DefaultCharsetStringPart("process", "store")); +// post.add(new DefaultCharsetStringPart("purpose", "crcon")); +// post.add(new DefaultCharsetStringPart("filesize", Long.toString(file.length))); +// post.add(new DefaultCharsetStringPart("md5", Digest.encodeMD5Hex(file))); +// post.add(new DefaultCharsetStringPart("access", access)); +// post.add(new DefaultCharsetFilePart("filename", new ByteArrayPartSource(filename, file))); // send request try { - final byte[] content = HttpConnector.wput("http://" + targetAddress + "/yacy/transfer.html", targetAddress, post, 20000); +// final byte[] content = HttpConnector.wput("http://" + targetAddress + "/yacy/transfer.html", targetAddress, post, 20000); + final LinkedHashMap parts = yacyNetwork.basicRequestParts(Switchboard.getSwitchboard(), null, salt); + parts.put("process", new StringBody("store")); + parts.put("purpose", new StringBody("crcon")); + parts.put("filesize", new StringBody(Long.toString(file.length))); + parts.put("md5", new StringBody(Digest.encodeMD5Hex(file))); + parts.put("access", new StringBody(access)); + parts.put("filename", new ByteArrayBody(file, filename)); + final byte[] content = HttpConnector.wput("http://" + targetAddress + "/yacy/transfer.html", targetAddress, parts, 20000); final Map result = FileUtils.table(content); return result; } catch (final Exception e) { @@ -800,13 +841,13 @@ public final class yacyClient { // prepare request final String salt = crypt.randomSalt(); - final List post = yacyNetwork.basicRequestPost(Switchboard.getSwitchboard(), target.hash, salt); - post.add(new DefaultCharsetStringPart("process", process)); - post.add(new DefaultCharsetStringPart("urlhash", ((entry == null) ? "" : new String(entry.hash())))); - post.add(new DefaultCharsetStringPart("result", result)); - post.add(new DefaultCharsetStringPart("reason", reason)); - post.add(new DefaultCharsetStringPart("wordh", wordhashes)); - post.add(new DefaultCharsetStringPart("lurlEntry", ((entry == null) ? "" : crypt.simpleEncode(entry.toString(), salt)))); +// final List post = yacyNetwork.basicRequestPost(Switchboard.getSwitchboard(), target.hash, salt); +// post.add(new DefaultCharsetStringPart("process", process)); +// post.add(new DefaultCharsetStringPart("urlhash", ((entry == null) ? "" : new String(entry.hash())))); +// post.add(new DefaultCharsetStringPart("result", result)); +// post.add(new DefaultCharsetStringPart("reason", reason)); +// post.add(new DefaultCharsetStringPart("wordh", wordhashes)); +// post.add(new DefaultCharsetStringPart("lurlEntry", ((entry == null) ? "" : crypt.simpleEncode(entry.toString(), salt)))); // determining target address final String address = target.getClusterAddress(); @@ -814,7 +855,17 @@ public final class yacyClient { // send request try { - final byte[] content = HttpConnector.wput("http://" + address + "/yacy/crawlReceipt.html", target.getHexHash() + ".yacyh", post, 10000); +// final byte[] content = HttpConnector.wput("http://" + address + "/yacy/crawlReceipt.html", target.getHexHash() + ".yacyh", post, 10000); + // prepare request + final LinkedHashMap parts = yacyNetwork.basicRequestParts(Switchboard.getSwitchboard(), target.hash, salt); + parts.put("process", new StringBody(process)); + parts.put("urlhash", new StringBody(((entry == null) ? "" : new String(entry.hash())))); + parts.put("result", new StringBody(result)); + parts.put("reason", new StringBody(reason)); + parts.put("wordh", new StringBody(wordhashes)); + parts.put("lurlEntry", new StringBody(((entry == null) ? "" : crypt.simpleEncode(entry.toString(), salt)))); + // send request + final byte[] content = HttpConnector.wput("http://" + address + "/yacy/crawlReceipt.html", target.getHexHash() + ".yacyh", parts, 10000); return FileUtils.table(content); } catch (final Exception e) { // most probably a network time-out exception @@ -1028,12 +1079,14 @@ public final class yacyClient { // this post a message to the remote message board final String salt = crypt.randomSalt(); - final List post = yacyNetwork.basicRequestPost(Switchboard.getSwitchboard(), targetSeed.hash, salt); +// final List post = yacyNetwork.basicRequestPost(Switchboard.getSwitchboard(), targetSeed.hash, salt); String address = targetSeed.getClusterAddress(); if (address == null) { address = "localhost:8080"; } try { - final byte[] content = HttpConnector.wput("http://" + address + "/yacy/profile.html", targetSeed.getHexHash() + ".yacyh", post, 5000); +// final byte[] content = HttpConnector.wput("http://" + address + "/yacy/profile.html", targetSeed.getHexHash() + ".yacyh", post, 5000); + final LinkedHashMap parts = yacyNetwork.basicRequestParts(Switchboard.getSwitchboard(), targetSeed.hash, salt); + final byte[] content = HttpConnector.wput("http://" + address + "/yacy/profile.html", targetSeed.getHexHash() + ".yacyh", parts, 5000); return FileUtils.table(content); } catch (final Exception e) { yacyCore.log.logSevere("yacyClient.getProfile error:" + e.getMessage()); diff --git a/source/net/yacy/cora/protocol/ByteArrayBody.java b/source/net/yacy/cora/protocol/ByteArrayBody.java new file mode 100644 index 000000000..19086bb33 --- /dev/null +++ b/source/net/yacy/cora/protocol/ByteArrayBody.java @@ -0,0 +1,59 @@ +package net.yacy.cora.protocol; + +import java.io.IOException; +import java.io.OutputStream; + +import org.apache.http.entity.mime.MIME; +import org.apache.http.entity.mime.content.AbstractContentBody; + +public class ByteArrayBody extends AbstractContentBody { + + private final String filename; + private final byte[] bytes; + + /** + * + * @param bytes of 'file' + * @param filename + */ + public ByteArrayBody(final byte[] bytes, final String filename) { + super("application/octet-stream"); + this.bytes = bytes; + this.filename = filename; + } + + /** + * @deprecated use {@link #writeTo(OutputStream)} + */ + @Deprecated + public void writeTo(final OutputStream out, int mode) throws IOException { + writeTo(out); + } + + @Override + public void writeTo(OutputStream outputStream) throws IOException { + outputStream.write(bytes); + outputStream.flush(); + } + + @Override + public String getFilename() { + return this.filename; + } + + @Override + public String getCharset() { + return null; + } + + @Override + public long getContentLength() { + return bytes.length; + } + + @Override + public String getTransferEncoding() { + return MIME.ENC_BINARY; + } + +} diff --git a/source/net/yacy/cora/protocol/HttpConnector.java b/source/net/yacy/cora/protocol/HttpConnector.java index 1a32bf5f8..ec90b33f8 100644 --- a/source/net/yacy/cora/protocol/HttpConnector.java +++ b/source/net/yacy/cora/protocol/HttpConnector.java @@ -38,19 +38,19 @@ import de.anomic.http.server.ResponseContainer; public class HttpConnector { - /** - * send data to the server named by vhost - * - * @param address address of the server - * @param vhost name of the server at address which should respond - * @param post data to send (name-value-pairs) - * @param timeout in milliseconds - * @return response body - * @throws IOException - */ - public static byte[] wput(final String url, final String vhost, final List post, final int timeout) throws IOException { - return wput(url, vhost, post, timeout, false); - } +// /** +// * send data to the server named by vhost +// * +// * @param address address of the server +// * @param vhost name of the server at address which should respond +// * @param post data to send (name-value-pairs) +// * @param timeout in milliseconds +// * @return response body +// * @throws IOException +// */ +// public static byte[] wput(final String url, final String vhost, final List post, final int timeout) throws IOException { +// return wput(url, vhost, post, timeout, false); +// } /** * send data to the server named by vhost @@ -84,7 +84,17 @@ public class HttpConnector { return content; } - public static byte[] wput(final String url, final String vhost, LinkedHashMap post, final int timeout) throws IOException { + /** + * send data to the server named by vhost + * + * @param url address of the server + * @param vhost name of the server at address which should respond + * @param post data to send (name-value-pairs) + * @param timeout in milliseconds + * @return response body + * @throws IOException + */ + public static byte[] wput(final String url, final String vhost, LinkedHashMap post, final int timeout) throws IOException { final Client client = new Client(); client.setTimout(timeout); client.setUserAgent(HTTPLoader.yacyUserAgent); @@ -93,7 +103,16 @@ public class HttpConnector { return client.POSTbytes(url, post); } - public static byte[] wget(final String url, final String vhost, final int timeout) throws IOException { + /** + * get data from the server named by vhost + * + * @param url address of the server + * @param vhost name of the server at address which should respond + * @param timeout in milliseconds + * @return response body + * @throws IOException + */ + public static byte[] wget(final String url, final String vhost, final int timeout) throws IOException { final Client client = new Client(); client.setTimout(timeout); client.setUserAgent(HTTPLoader.yacyUserAgent); diff --git a/source/net/yacy/cora/services/Search.java b/source/net/yacy/cora/services/Search.java index 819167d7a..9d79ba57a 100644 --- a/source/net/yacy/cora/services/Search.java +++ b/source/net/yacy/cora/services/Search.java @@ -23,9 +23,10 @@ package net.yacy.cora.services; import java.io.IOException; import java.net.MalformedURLException; -import java.nio.charset.Charset; -import java.util.ArrayList; -import java.util.List; +//import java.nio.charset.Charset; +//import java.util.ArrayList; +import java.util.LinkedHashMap; +//import java.util.List; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; @@ -35,8 +36,10 @@ import net.yacy.cora.document.RSSMessage; import net.yacy.cora.document.RSSReader; import net.yacy.cora.protocol.HttpConnector; -import org.apache.commons.httpclient.methods.multipart.Part; -import org.apache.commons.httpclient.methods.multipart.StringPart; +//import org.apache.commons.httpclient.methods.multipart.Part; +//import org.apache.commons.httpclient.methods.multipart.StringPart; +import org.apache.http.entity.mime.content.ContentBody; +import org.apache.http.entity.mime.content.StringBody; public class Search { @@ -116,16 +119,23 @@ public class Search { } // prepare request - final List post = new ArrayList(); - post.add(new StringPart("query", query, Charset.defaultCharset().name())); - post.add(new StringPart("startRecord", Integer.toString(startRecord), Charset.defaultCharset().name())); - post.add(new StringPart("maximumRecords", Long.toString(maximumRecords), Charset.defaultCharset().name())); - post.add(new StringPart("verify", verify ? "true" : "false", Charset.defaultCharset().name())); - post.add(new StringPart("resource", global ? "global" : "local", Charset.defaultCharset().name())); +// final List post = new ArrayList(); +// post.add(new StringPart("query", query, Charset.defaultCharset().name())); +// post.add(new StringPart("startRecord", Integer.toString(startRecord), Charset.defaultCharset().name())); +// post.add(new StringPart("maximumRecords", Long.toString(maximumRecords), Charset.defaultCharset().name())); +// post.add(new StringPart("verify", verify ? "true" : "false", Charset.defaultCharset().name())); +// post.add(new StringPart("resource", global ? "global" : "local", Charset.defaultCharset().name())); // send request try { - final byte[] result = HttpConnector.wput(rssSearchServiceURL, uri.getHost(), post, (int) timeout); +// final byte[] result = HttpConnector.wput(rssSearchServiceURL, uri.getHost(), post, (int) timeout); + final LinkedHashMap parts = new LinkedHashMap(); + parts.put("query", new StringBody(query)); + parts.put("startRecord", new StringBody(Integer.toString(startRecord))); + parts.put("maximumRecords", new StringBody(Long.toString(maximumRecords))); + parts.put("verify", new StringBody(verify ? "true" : "false")); + parts.put("resource", new StringBody(global ? "global" : "local")); + final byte[] result = HttpConnector.wput(rssSearchServiceURL, uri.getHost(), parts, (int) timeout); //String debug = new String(result); System.out.println("*** DEBUG: " + debug); final RSSReader reader = RSSReader.parse(result); if (reader == null) {