diff --git a/htroot/Blog.java b/htroot/Blog.java index 7a5a59a08..091ae5b8a 100644 --- a/htroot/Blog.java +++ b/htroot/Blog.java @@ -78,7 +78,7 @@ public class Blog { final boolean xml = (header.get(HeaderFramework.CONNECTION_PROP_PATH)).endsWith(".xml"); - final String address = sb.peers.mySeed().getPublicAddress(); + final String address = sb.peers.mySeed().getPublicAddress(sb.peers.mySeed().getIP()); prop.put("mode_admin", hasRights ? "1" : "0"); @@ -192,7 +192,7 @@ public class Blog { prop.putHTML("mode_author", UTF8.String(author)); prop.putHTML("mode_subject", post.get("subject","")); prop.put("mode_date", dateString(new Date())); - prop.putWiki(sb.peers.mySeed().getPublicAddress(), "mode_page", post.get("content", "")); + prop.putWiki(sb.peers.mySeed().getPublicAddress(sb.peers.mySeed().getIP()), "mode_page", post.get("content", "")); prop.putHTML("mode_page-code", post.get("content", "")); } else { diff --git a/htroot/BlogComments.java b/htroot/BlogComments.java index 06a2d3508..7870f1bf2 100644 --- a/htroot/BlogComments.java +++ b/htroot/BlogComments.java @@ -175,7 +175,7 @@ public class BlogComments { prop.putHTML("mode_allow_author", UTF8.String(author)); prop.putHTML("mode_subject", post.get("subject","")); prop.put("mode_date", dateString(new Date())); - prop.putWiki(sb.peers.mySeed().getPublicAddress(), "mode_page", post.get("content", "")); + prop.putWiki(sb.peers.mySeed().getPublicAddress(sb.peers.mySeed().getIP()), "mode_page", post.get("content", "")); prop.put("mode_page-code", post.get("content", "")); } else { // show blog-entry/entries @@ -191,7 +191,7 @@ public class BlogComments { prop.putHTML("mode_allow_author", UTF8.String(author)); prop.put("mode_comments", page.getCommentsSize()); prop.put("mode_date", dateString(page.getDate())); - prop.putWiki(sb.peers.mySeed().getPublicAddress(), "mode_page", page.getPage()); + prop.putWiki(sb.peers.mySeed().getPublicAddress(sb.peers.mySeed().getIP()), "mode_page", page.getPage()); if (hasRights) { prop.put("mode_admin", "1"); prop.put("mode_admin_pageid", page.getKey()); @@ -234,7 +234,7 @@ public class BlogComments { if (!xml) { prop.putHTML("mode_entries_"+count+"_subject", UTF8.String(entry.getSubject())); prop.putHTML("mode_entries_"+count+"_author", UTF8.String(entry.getAuthor())); - prop.putWiki(sb.peers.mySeed().getPublicAddress(), "mode_entries_"+count+"_page", entry.getPage()); + prop.putWiki(sb.peers.mySeed().getPublicAddress(sb.peers.mySeed().getIP()), "mode_entries_"+count+"_page", entry.getPage()); } else { prop.putHTML("mode_entries_"+count+"_subject", UTF8.String(entry.getSubject())); prop.putHTML("mode_entries_"+count+"_author", UTF8.String(entry.getAuthor())); diff --git a/htroot/Bookmarks.java b/htroot/Bookmarks.java index 74733aa15..266d94cf4 100644 --- a/htroot/Bookmarks.java +++ b/htroot/Bookmarks.java @@ -102,7 +102,7 @@ public class Bookmarks { */ // set peer address - prop.put("address", sb.peers.mySeed().getPublicAddress()); + prop.put("address", sb.peers.mySeed().getPublicAddress(sb.peers.mySeed().getIP())); //defaultvalues if(isAdmin) { diff --git a/htroot/ConfigPortal.java b/htroot/ConfigPortal.java index 474d22644..cedcdbf9b 100644 --- a/htroot/ConfigPortal.java +++ b/htroot/ConfigPortal.java @@ -219,7 +219,7 @@ public class ConfigPortal { prop.put("target_selected_special_searchresult", "searchresult".equals(target_special) ? 1 : 0); prop.put("target_special_pattern", sb.getConfig(SwitchboardConstants.SEARCH_TARGET_SPECIAL_PATTERN, "")); - String myaddress = (sb.peers == null) ? null : sb.peers.mySeed() == null ? null : sb.peers.mySeed().getPublicAddress(); + String myaddress = (sb.peers == null) ? null : sb.peers.mySeed() == null ? null : sb.peers.mySeed().getPublicAddress(sb.peers.mySeed().getIP()); if (myaddress == null) { myaddress = "localhost:" + sb.getLocalPort("port", 8090); } diff --git a/htroot/ConfigRobotsTxt_p.java b/htroot/ConfigRobotsTxt_p.java index 362bcba99..0f7e254db 100644 --- a/htroot/ConfigRobotsTxt_p.java +++ b/htroot/ConfigRobotsTxt_p.java @@ -44,7 +44,7 @@ public class ConfigRobotsTxt_p { final servletProperties prop = new servletProperties(); final RobotsTxtConfig rbc = ((Switchboard)env).robotstxtConfig; - prop.put("clientname", sb.peers.mySeed().getPublicAddress()); + prop.put("clientname", sb.peers.mySeed().getPublicAddress(sb.peers.mySeed().getIP())); if (post != null) { if (post.containsKey("save")) { diff --git a/htroot/ConfigSearchBox.java b/htroot/ConfigSearchBox.java index 393777203..c64bf7293 100644 --- a/htroot/ConfigSearchBox.java +++ b/htroot/ConfigSearchBox.java @@ -34,7 +34,7 @@ public class ConfigSearchBox { final serverObjects prop = new serverObjects(); final Switchboard sb = (Switchboard) env; - String myaddress = sb.peers.mySeed().getPublicAddress(); + String myaddress = sb.peers.mySeed().getPublicAddress(sb.peers.mySeed().getIP()); if (myaddress == null) myaddress = "localhost:" + sb.getLocalPort("port", 8090); prop.put("myaddress", myaddress); return prop; diff --git a/htroot/Load_MediawikiWiki.java b/htroot/Load_MediawikiWiki.java index d40647951..bb18ba755 100644 --- a/htroot/Load_MediawikiWiki.java +++ b/htroot/Load_MediawikiWiki.java @@ -38,7 +38,7 @@ public class Load_MediawikiWiki { final serverObjects prop = new serverObjects(); // define visible variables - String a = sb.peers.mySeed().getPublicAddress(); + String a = sb.peers.mySeed().getPublicAddress(sb.peers.mySeed().getIP()); if (a == null) a = "localhost:" + sb.getLocalPort("port", 8090); final boolean intranet = sb.getConfig(SwitchboardConstants.NETWORK_NAME, "").equals("intranet"); final String repository = "http://" + a + "/repository/"; diff --git a/htroot/Load_PHPBB3.java b/htroot/Load_PHPBB3.java index 299a2bd97..bc9b3d9f0 100644 --- a/htroot/Load_PHPBB3.java +++ b/htroot/Load_PHPBB3.java @@ -38,7 +38,7 @@ public class Load_PHPBB3 { final serverObjects prop = new serverObjects(); // define visible variables - String a = sb.peers.mySeed().getPublicAddress(); + String a = sb.peers.mySeed().getPublicAddress(sb.peers.mySeed().getIP()); if (a == null) a = "localhost:" + sb.getLocalPort("port", 8090); final boolean intranet = sb.getConfig(SwitchboardConstants.NETWORK_NAME, "").equals("intranet"); final String repository = "http://" + a + "/repository/"; diff --git a/htroot/Messages_p.java b/htroot/Messages_p.java index ac05dc53f..6e66c40ce 100644 --- a/htroot/Messages_p.java +++ b/htroot/Messages_p.java @@ -59,7 +59,7 @@ public class Messages_p { final serverObjects prop = new serverObjects(); // set peer address / name - final String peerAddress = sb.peers.mySeed().getPublicAddress(); + final String peerAddress = sb.peers.mySeed().getPublicAddress(sb.peers.mySeed().getIP()); final String peerName = sb.peers.mySeed().getName(); prop.put("peerAddress", peerAddress); prop.putXML("peerName", peerName); @@ -162,7 +162,7 @@ public class Messages_p { prop.putXML("mode_subject", message.subject()); String theMessage = null; theMessage = UTF8.String(message.message()); - prop.putWiki(sb.peers.mySeed().getPublicAddress(), "mode_message", theMessage); + prop.putWiki(sb.peers.mySeed().getPublicAddress(sb.peers.mySeed().getIP()), "mode_message", theMessage); prop.put("mode_hash", message.authorHash()); prop.putXML("mode_key", key); } diff --git a/htroot/News.java b/htroot/News.java index 15c6dbfc8..63e8d6a99 100644 --- a/htroot/News.java +++ b/htroot/News.java @@ -157,11 +157,11 @@ public class News { title = record.attribute("title", ""); description = record.attribute("url", ""); } else if (category.equals(NewsPool.CATEGORY_WIKI_UPDATE)) { - link = (seed == null)? "" : "http://" + seed.getPublicAddress() + "/Wiki.html?page=" + record.attribute("page", ""); + link = (seed == null)? "" : "http://" + seed.getPublicAddress(seed.getIP()) + "/Wiki.html?page=" + record.attribute("page", ""); title = record.attribute("author", "Anonymous") + ": " + record.attribute("page", ""); description = "Wiki Update: " + record.attribute("description", ""); } else if (category.equals(NewsPool.CATEGORY_BLOG_ADD)) { - link = (seed == null)? "" : "http://" + seed.getPublicAddress() + "/Blog.html?page=" + record.attribute("page", ""); + link = (seed == null)? "" : "http://" + seed.getPublicAddress(seed.getIP()) + "/Blog.html?page=" + record.attribute("page", ""); title = record.attribute("author", "Anonymous") + ": " + record.attribute("page", ""); description = "Blog Entry: " + record.attribute("subject", ""); } else { @@ -180,7 +180,7 @@ public class News { } // adding the peer address - prop.put("address", sb.peers.mySeed().getPublicAddress()); + prop.put("address", sb.peers.mySeed().getPublicAddress(sb.peers.mySeed().getIP())); // return rewrite properties return prop; diff --git a/htroot/Status.java b/htroot/Status.java index 42a5d7a47..bdc486972 100644 --- a/htroot/Status.java +++ b/htroot/Status.java @@ -233,7 +233,7 @@ public class Status prop.put("warningGoOnline", "1"); } else { prop.put("peerAddress", "1"); // Address - prop.put("peerAddress_address", sb.peers.mySeed().getPublicAddress()); + prop.put("peerAddress_address", sb.peers.mySeed().getPublicAddress(sb.peers.mySeed().getIP())); prop.putXML("peerAddress_peername", sb.peers.mySeed().getName().toLowerCase()); } } diff --git a/htroot/Table_YMark_p.java b/htroot/Table_YMark_p.java index 13bdae6aa..786f7d5a2 100644 --- a/htroot/Table_YMark_p.java +++ b/htroot/Table_YMark_p.java @@ -274,7 +274,7 @@ public class Table_YMark_p { } // adding the peer address - prop.put("address", sb.peers.mySeed().getPublicAddress()); + prop.put("address", sb.peers.mySeed().getPublicAddress(sb.peers.mySeed().getIP())); // return rewrite properties return prop; diff --git a/htroot/Tables_p.java b/htroot/Tables_p.java index 784c4da8a..ad5bbaf5b 100644 --- a/htroot/Tables_p.java +++ b/htroot/Tables_p.java @@ -190,7 +190,7 @@ public class Tables_p { } // adding the peer address - prop.put("address", sb.peers.mySeed().getPublicAddress()); + prop.put("address", sb.peers.mySeed().getPublicAddress(sb.peers.mySeed().getIP())); // return rewrite properties return prop; diff --git a/htroot/ViewProfile.java b/htroot/ViewProfile.java index 7a56c5ca8..0a1879571 100644 --- a/htroot/ViewProfile.java +++ b/htroot/ViewProfile.java @@ -169,7 +169,7 @@ public class ViewProfile { prop.put("success_" + key, "1"); // only comments get "wikified" if(key.equals("comment")){ - prop.putWiki(sb.peers.mySeed().getPublicAddress(), + prop.putWiki(sb.peers.mySeed().getPublicAddress(sb.peers.mySeed().getIP()), "success_" + key + "_value", entry.getValue().replaceAll("\r", "").replaceAll("\\\\n", "\n")); prop.put("success_" + key + "_b64value", Base64Order.standardCoder.encodeString(entry.getValue())); diff --git a/htroot/Wiki.java b/htroot/Wiki.java index 7ca1185fe..cc2e57441 100644 --- a/htroot/Wiki.java +++ b/htroot/Wiki.java @@ -147,7 +147,7 @@ public class Wiki { prop.putHTML("mode_pagename", pagename); prop.putHTML("mode_author", author); prop.put("mode_date", dateString(new Date())); - prop.putWiki(sb.peers.mySeed().getPublicAddress(), "mode_page", post.get("content", "")); + prop.putWiki(sb.peers.mySeed().getPublicAddress(sb.peers.mySeed().getIP()), "mode_page", post.get("content", "")); prop.putHTML("mode_page-code", post.get("content", "")); } //end contrib of [MN] @@ -237,7 +237,7 @@ public class Wiki { prop.putHTML("mode_versioning_pagename", pagename); prop.putHTML("mode_versioning_author", oentry.author()); prop.put("mode_versioning_date", dateString(oentry.date())); - prop.putWiki(sb.peers.mySeed().getPublicAddress(), "mode_versioning_page", oentry.page()); + prop.putWiki(sb.peers.mySeed().getPublicAddress(sb.peers.mySeed().getIP()), "mode_versioning_page", oentry.page()); prop.putHTML("mode_versioning_page-code", UTF8.String(oentry.page())); } } catch (final IOException e) { @@ -252,7 +252,7 @@ public class Wiki { prop.putHTML("mode_pagename", pagename); prop.putHTML("mode_author", page.author()); prop.put("mode_date", dateString(page.date())); - prop.putWiki(sb.peers.mySeed().getPublicAddress(), "mode_page", page.page()); + prop.putWiki(sb.peers.mySeed().getPublicAddress(sb.peers.mySeed().getIP()), "mode_page", page.page()); prop.put("controls", "0"); prop.putHTML("controls_pagename", pagename); diff --git a/htroot/api/bookmarks/get_bookmarks.java b/htroot/api/bookmarks/get_bookmarks.java index 4b3c724a0..c53eda150 100644 --- a/htroot/api/bookmarks/get_bookmarks.java +++ b/htroot/api/bookmarks/get_bookmarks.java @@ -68,7 +68,7 @@ public class get_bookmarks { prop.putHTML("display_user", username); // set peer address - prop.put("display_address", sb.peers.mySeed().getPublicAddress()); + prop.put("display_address", sb.peers.mySeed().getPublicAddress(sb.peers.mySeed().getIP())); prop.put("display_peer", sb.peers.mySeed().getName()); final int itemsPerPage; // items per page diff --git a/htroot/api/bookmarks/get_folders.java b/htroot/api/bookmarks/get_folders.java index 90f44efe7..a5184d80f 100644 --- a/htroot/api/bookmarks/get_folders.java +++ b/htroot/api/bookmarks/get_folders.java @@ -33,7 +33,7 @@ public class get_folders { prop.putHTML("display_user", username); // set peer address - prop.put("display_address", sb.peers.mySeed().getPublicAddress()); + prop.put("display_address", sb.peers.mySeed().getPublicAddress(sb.peers.mySeed().getIP())); prop.put("display_peer", sb.peers.mySeed().getName()); String root = "/"; diff --git a/htroot/api/ynetSearch.java b/htroot/api/ynetSearch.java index cc6cd23f6..9081549c1 100644 --- a/htroot/api/ynetSearch.java +++ b/htroot/api/ynetSearch.java @@ -15,8 +15,8 @@ import net.yacy.server.serverSwitch; public class ynetSearch { public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { - final Switchboard switchboard = (Switchboard) env; - final boolean isAdmin=switchboard.verifyAuthentication(header); + final Switchboard sb = (Switchboard) env; + final boolean isAdmin=sb.verifyAuthentication(header); final serverObjects prop = new serverObjects(); if(post != null){ @@ -32,7 +32,7 @@ public class ynetSearch { String searchaddress = post.get("url"); if (!searchaddress.startsWith("http://")) { // a relative path .. this addresses the local peer - searchaddress = "http://" + switchboard.peers.mySeed().getPublicAddress() + ((searchaddress.length() > 0 && searchaddress.charAt(0) == '/') ? "" : "/") + searchaddress; + searchaddress = "http://" + sb.peers.mySeed().getPublicAddress(sb.peers.mySeed().getIP()) + ((searchaddress.length() > 0 && searchaddress.charAt(0) == '/') ? "" : "/") + searchaddress; } post.remove("url"); post.remove("login"); diff --git a/htroot/goto_p.java b/htroot/goto_p.java index b958742d9..4c0889378 100644 --- a/htroot/goto_p.java +++ b/htroot/goto_p.java @@ -56,7 +56,7 @@ public class goto_p { Seed seed = sb.peers.getConnected(hash); if (seed != null) { - String peersUrl = seed.getPublicAddress(); + String peersUrl = seed.getPublicAddress(seed.getIP()); if (peersUrl != null) { String path = post.get("path", "/"); if (!path.startsWith("/")) path = "/" + path; diff --git a/htroot/mediawiki_p.java b/htroot/mediawiki_p.java index 140c9e115..0a11fb7d1 100644 --- a/htroot/mediawiki_p.java +++ b/htroot/mediawiki_p.java @@ -70,7 +70,7 @@ public class mediawiki_p { page = page.substring(p, q); prop.putHTML("title", title); - prop.putWiki(sb.peers.mySeed().getPublicAddress(), "page", page); + prop.putWiki(sb.peers.mySeed().getPublicAddress(sb.peers.mySeed().getIP()), "page", page); return prop; } diff --git a/htroot/yacy/hello.java b/htroot/yacy/hello.java index 2659c14a1..e21455a9c 100644 --- a/htroot/yacy/hello.java +++ b/htroot/yacy/hello.java @@ -115,12 +115,10 @@ public final class hello { //final double clientversion = remoteSeed.getVersion(); if (remoteSeed.getPort() == sb.peers.mySeed().getPort()) { - for (String reportedip: reportedips) { - if (sb.peers.mySeed().getIPs().contains(reportedip)) { - // reject a self-ping - prop.put("message", "I am I"); - return prop; - } + if (sb.peers.mySeed().clash(reportedips)) { + // reject a self-ping + prop.put("message", "I am I"); + return prop; } } if (remoteSeed.hash.equals(sb.peers.mySeed().hash)) { diff --git a/htroot/yacy/seedlist.java b/htroot/yacy/seedlist.java index c30de0c66..9c9d1f8a4 100644 --- a/htroot/yacy/seedlist.java +++ b/htroot/yacy/seedlist.java @@ -97,7 +97,7 @@ public final class seedlist { } } prop.putJSON("peers_" + count + "_map_" + c + "_k", "Address"); - prop.putJSON("peers_" + count + "_map_" + c + "_v", seed.getPublicAddress()); + prop.putJSON("peers_" + count + "_map_" + c + "_v", seed.getPublicAddress(seed.getIP())); prop.put("peers_" + count + "_map_" + c + "_c", 0); prop.put("peers_" + count + "_map", c + 1); prop.put("peers_" + count + "_c", 1); @@ -124,7 +124,7 @@ public final class seedlist { } } prop.putXML("peers_" + count + "_map_" + c + "_k", "Address"); - prop.putXML("peers_" + count + "_map_" + c + "_v", seed.getPublicAddress()); + prop.putXML("peers_" + count + "_map_" + c + "_v", seed.getPublicAddress(seed.getIP())); prop.put("peers_" + count + "_map_" + c + "_c", 0); prop.put("peers_" + count + "_map", c + 1); count++; diff --git a/htroot/yacysearch_location.java b/htroot/yacysearch_location.java index 7a08205fd..145c34b0a 100644 --- a/htroot/yacysearch_location.java +++ b/htroot/yacysearch_location.java @@ -81,7 +81,7 @@ public class yacysearch_location { prop.put("kml_placemark_" + placemarkCounter + "_subject", ""); prop.put("kml_placemark_" + placemarkCounter + "_description", ""); prop.put("kml_placemark_" + placemarkCounter + "_date", ""); - prop.putXML("kml_placemark_" + placemarkCounter + "_url", "http://" + sb.peers.mySeed().getPublicAddress() + "/yacysearch.html?query=" + location.getName()); + prop.putXML("kml_placemark_" + placemarkCounter + "_url", "http://" + sb.peers.mySeed().getPublicAddress(sb.peers.mySeed().getIP()) + "/yacysearch.html?query=" + location.getName()); prop.put("kml_placemark_" + placemarkCounter + "_pointname", location.getName()); prop.put("kml_placemark_" + placemarkCounter + "_lon", location.lon()); prop.put("kml_placemark_" + placemarkCounter + "_lat", location.lat()); diff --git a/source/net/yacy/peers/Network.java b/source/net/yacy/peers/Network.java index ba7d8f5ed..75aaf82e6 100644 --- a/source/net/yacy/peers/Network.java +++ b/source/net/yacy/peers/Network.java @@ -586,7 +586,7 @@ public class Network String logt; // be shure that we have something to say - if ( sb.peers.mySeed().getPublicAddress() == null ) { + if ( sb.peers.mySeed().getPublicAddress(sb.peers.mySeed().getIP()) == null ) { final String errorMsg = "We have no valid IP address until now"; log.warn("SaveSeedList: " + errorMsg); return errorMsg; diff --git a/source/net/yacy/peers/Protocol.java b/source/net/yacy/peers/Protocol.java index 2a7104c2b..58e2d771f 100644 --- a/source/net/yacy/peers/Protocol.java +++ b/source/net/yacy/peers/Protocol.java @@ -367,14 +367,15 @@ public final class Protocol { basicRequestParts(Switchboard.getSwitchboard(), target.hash, salt); parts.put("object", UTF8.StringBody("seed")); parts.put("env", UTF8.StringBody(seedHash)); - final Post post = new Post(target.getPublicAddress(target.getIP()), target.hash, "/yacy/query.html", parts, 10000); + String ip = target.getIP(); + final Post post = new Post(target.getPublicAddress(ip), target.hash, "/yacy/query.html", parts, 10000); final Map result = FileUtils.table(post.result); if ( result == null || result.isEmpty() ) { return null; } //final Date remoteTime = yacyCore.parseUniversalDate((String) result.get(yacySeed.MYTIME)); // read remote time - return Seed.genRemoteSeed(result.get("response"), false, target.getIP()); + return Seed.genRemoteSeed(result.get("response"), false, ip); } catch (final Exception e ) { Network.log.warn("yacyClient.querySeed error:" + e.getMessage()); return null; @@ -434,43 +435,42 @@ public final class Protocol { final String salt = crypt.randomSalt(); // send request - try { - /* a long time-out is needed */ - final Map parts = - basicRequestParts(Switchboard.getSwitchboard(), target.hash, salt); - parts.put("call", UTF8.StringBody("remotecrawl")); - parts.put("count", UTF8.StringBody(Integer.toString(maxCount))); - parts.put("time", UTF8.StringBody(Long.toString(maxTime))); - // final byte[] result = HTTPConnector.getConnector(MultiProtocolURI.yacybotUserAgent).post(new MultiProtocolURI("http://" + target.getClusterAddress() + "/yacy/urls.xml"), (int) maxTime, target.getHexHash() + ".yacyh", parts); - final HTTPClient httpClient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent, (int) maxTime); - String ip = target.getIP(); - final byte[] result = httpClient.POSTbytes(new MultiProtocolURL("http://" + target.getPublicAddress(ip) + "/yacy/urls.xml"), target.getHexHash() + ".yacyh", parts, false, true); - final RSSReader reader = RSSReader.parse(RSSFeed.DEFAULT_MAXSIZE, result); - if ( reader == null ) { - Network.log.warn("yacyClient.queryRemoteCrawlURLs failed asking peer '" + target.getName() + "': probably bad response from remote peer (1), reader == null"); - target.put(Seed.RCOUNT, "0"); - seedDB.peerActions.interfaceDeparture(target, ip); - //Log.logException(e); - return null; - } - final RSSFeed feed = reader.getFeed(); - if ( feed == null ) { - // case where the rss reader does not understand the content - Network.log.warn("yacyClient.queryRemoteCrawlURLs failed asking peer '" + target.getName() + "': probably bad response from remote peer (2)"); - //System.out.println("***DEBUG*** rss input = " + UTF8.String(result)); - target.put(Seed.RCOUNT, "0"); - seedDB.updateConnected(target); // overwrite number of remote-available number to avoid that this peer is called again (until update is done by peer ping) - //Log.logException(e); - return null; + /* a long time-out is needed */ + final Map parts = + basicRequestParts(Switchboard.getSwitchboard(), target.hash, salt); + parts.put("call", UTF8.StringBody("remotecrawl")); + parts.put("count", UTF8.StringBody(Integer.toString(maxCount))); + parts.put("time", UTF8.StringBody(Long.toString(maxTime))); + // final byte[] result = HTTPConnector.getConnector(MultiProtocolURI.yacybotUserAgent).post(new MultiProtocolURI("http://" + target.getClusterAddress() + "/yacy/urls.xml"), (int) maxTime, target.getHexHash() + ".yacyh", parts); + final HTTPClient httpClient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent, (int) maxTime); + RSSReader reader = null; + for (String ip: target.getIPs()) { + try { + final byte[] result = httpClient.POSTbytes(new MultiProtocolURL("http://" + target.getPublicAddress(ip) + "/yacy/urls.xml"), target.getHexHash() + ".yacyh", parts, false, true); + reader = RSSReader.parse(RSSFeed.DEFAULT_MAXSIZE, result); + } catch (final IOException e ) { + reader = null; } - // update number of remotely available links in seed - target.put(Seed.RCOUNT, Integer.toString(Math.max(0, targetCount - feed.size()))); - seedDB.updateConnected(target); - return feed; - } catch (final IOException e ) { - Network.log.warn("yacyClient.queryRemoteCrawlURLs error asking peer '" + target.getName() + "':" + e.toString()); + if (reader != null) break; + Network.log.warn("yacyClient.queryRemoteCrawlURLs failed asking peer '" + target.getName() + "': probably bad response from remote peer (1), reader == null"); + target.put(Seed.RCOUNT, "0"); + seedDB.peerActions.interfaceDeparture(target, ip); + } + + final RSSFeed feed = reader == null ? null : reader.getFeed(); + if ( feed == null ) { + // case where the rss reader does not understand the content + Network.log.warn("yacyClient.queryRemoteCrawlURLs failed asking peer '" + target.getName() + "': probably bad response from remote peer (2)"); + //System.out.println("***DEBUG*** rss input = " + UTF8.String(result)); + target.put(Seed.RCOUNT, "0"); + seedDB.updateConnected(target); // overwrite number of remote-available number to avoid that this peer is called again (until update is done by peer ping) + //Log.logException(e); return null; } + // update number of remotely available links in seed + target.put(Seed.RCOUNT, Integer.toString(Math.max(0, targetCount - feed.size()))); + seedDB.updateConnected(target); + return feed; } protected static int primarySearch( @@ -526,13 +526,15 @@ public final class Protocol { clusteraddress, secondarySearchSuperviser ); + break; } catch (final IOException e ) { Network.log.info("SEARCH failed, Peer: " + target.hash + ":" + target.getName() + " (" + e.getMessage() + ")"); event.peers.peerActions.interfaceDeparture(target, ip); return -1; - } - + } } + if (result == null) return -1; + // computation time final long totalrequesttime = System.currentTimeMillis() - timestamp; @@ -586,30 +588,35 @@ public final class Protocol { final long timestamp = System.currentTimeMillis(); event.addExpectedRemoteReferences(count); - SearchResult result; - try { - result = - new SearchResult( - event, - basicRequestParts(Switchboard.getSwitchboard(), target.hash, crypt.randomSalt()), - wordhashes, - "", - urlhashes, - "", - contentdom, - count, - time, - maxDistance, - partitions, - target.getHexHash() + ".yacyh", - target.getPublicAddress(), - null - ); - } catch (final IOException e ) { - Network.log.info("SEARCH failed, Peer: " + target.hash + ":" + target.getName() + " (" + e.getMessage() + ")"); - event.peers.peerActions.peerDeparture(target, "search request to peer created io exception: " + e.getMessage()); - return -1; + SearchResult result = null; + for (String ip: target.getIPs()) { + try { + result = + new SearchResult( + event, + basicRequestParts(Switchboard.getSwitchboard(), target.hash, crypt.randomSalt()), + wordhashes, + "", + urlhashes, + "", + contentdom, + count, + time, + maxDistance, + partitions, + target.getHexHash() + ".yacyh", + target.getPublicAddress(ip), + null + ); + break; + } catch (final IOException e ) { + Network.log.info("SEARCH failed, Peer: " + target.hash + ":" + target.getName() + " (" + e.getMessage() + ")"); + event.peers.peerActions.interfaceDeparture(target, ip); + return -1; + } } + if (result == null) return -1; + // computation time final long totalrequesttime = System.currentTimeMillis() - timestamp; @@ -660,7 +667,7 @@ public final class Protocol { } if ( blacklist.isListed(BlacklistType.SEARCH, urlEntry.url()) ) { if ( Network.log.isInfo() ) { - Network.log.info("remote search: filtered blacklisted url " + urlEntry.url() + " from peer " + target.getName()); + Network.log.info("remote search: filtered blacklisted url " + urlEntry.url().toNormalform(true) + " from peer " + target.getName()); } continue; // block with backlist } @@ -669,7 +676,7 @@ public final class Protocol { Switchboard.getSwitchboard().crawlStacker.urlInAcceptedDomain(urlEntry.url()); if ( urlRejectReason != null ) { if ( Network.log.isInfo() ) { - Network.log.info("remote search: rejected url '" + urlEntry.url() + "' (" + urlRejectReason + ") from peer " + target.getName()); + Network.log.info("remote search: rejected url '" + urlEntry.url().toNormalform(true) + "' (" + urlRejectReason + ") from peer " + target.getName()); } continue; // reject url outside of our domain } @@ -935,6 +942,7 @@ public final class Protocol { Map> snippets = new HashMap>(); // this will be a list of urlhash-snippet entries final QueryResponse[] rsp = new QueryResponse[]{null}; final SolrDocumentList[] docList = new SolrDocumentList[]{null}; + String ip = target.getIP(); {// encapsulate expensive solr QueryResponse object if (localsearch && !Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.DEBUG_SEARCH_REMOTE_SOLR_TESTLOCAL, false)) { // search the local index @@ -955,7 +963,7 @@ public final class Protocol { Network.log.info("SEARCH skip (solr), remote Solr interface not accessible, peer=" + target.getName()); return -1; } - final String address = myseed ? "localhost:" + target.getPort() : target.getPublicAddress(target.getIP()); + final String address = myseed ? "localhost:" + target.getPort() : target.getPublicAddress(ip); final int solrtimeout = Switchboard.getSwitchboard().getConfigInt(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_TIMEOUT, 6000); Thread remoteRequest = new Thread() { @Override @@ -981,20 +989,20 @@ public final class Protocol { remoteRequest.join(solrtimeout); // just wait until timeout appears if (remoteRequest.isAlive()) { try {remoteRequest.interrupt();} catch (Throwable e) {} - Network.log.info("SEARCH failed (solr), remote Peer: " + target.getName() + "/" + target.getPublicAddress() + " does not answer (time-out)"); + Network.log.info("SEARCH failed (solr), remote Peer: " + target.getName() + "/" + target.getPublicAddress(ip) + " does not answer (time-out)"); target.setFlagSolrAvailable(false || myseed); return -1; // give up, leave remoteRequest abandoned. } // no need to close this here because that sends a commit to remote solr which is not wanted here } catch (final Throwable e) { - Network.log.info("SEARCH failed (solr), remote Peer: " + target.getName() + "/" + target.getPublicAddress() + " (" + e.getMessage() + ")"); + Network.log.info("SEARCH failed (solr), remote Peer: " + target.getName() + "/" + target.getPublicAddress(ip) + " (" + e.getMessage() + ")"); target.setFlagSolrAvailable(false || localsearch); return -1; } } if (rsp[0] == null || docList[0] == null) { - Network.log.info("SEARCH failed (solr), remote Peer: " + target.getName() + "/" + target.getPublicAddress() + " returned null"); + Network.log.info("SEARCH failed (solr), remote Peer: " + target.getName() + "/" + target.getPublicAddress(ip) + " returned null"); target.setFlagSolrAvailable(false || localsearch); return -1; } @@ -1040,7 +1048,8 @@ public final class Protocol { Network.log.info("SEARCH (solr), returned 0 out of 0 documents from " + (target == null ? "shard" : ("peer " + target.hash + ":" + target.getName())) + " query = " + solrQuery.toString()) ; return 0; } - List container = new ArrayList(); + + List container = new ArrayList(); Network.log.info("SEARCH (solr), returned " + docList[0].size() + " out of " + docList[0].getNumFound() + " documents and " + facets.size() + " facets " + facets.keySet().toString() + " from " + (target == null ? "shard" : ("peer " + target.hash + ":" + target.getName()))); int term = count; Collection docs = new ArrayList(docList[0].size()); @@ -1156,8 +1165,7 @@ public final class Protocol { // send request try { - final Map parts = - basicRequestParts(Switchboard.getSwitchboard(), targetHash, salt); + final Map parts = basicRequestParts(Switchboard.getSwitchboard(), targetHash, salt); parts.put("process", UTF8.StringBody("post")); parts.put("myseed", UTF8.StringBody(seedDB.mySeed().genSeedStr(salt))); parts.put("subject", UTF8.StringBody(subject)); diff --git a/source/net/yacy/peers/Seed.java b/source/net/yacy/peers/Seed.java index bc963b0a3..f63580f27 100644 --- a/source/net/yacy/peers/Seed.java +++ b/source/net/yacy/peers/Seed.java @@ -744,16 +744,6 @@ public class Seed implements Cloneable, Comparable, Comparator public final int getRevision() { return yacyVersion.revision(get(Seed.VERSION, Seed.ZERO)); } - - /** - * deprecated, use getIPs() instead - * @return the public address of the peer as IP:port string or null if no valid values for - * either the IP or the port could be retrieved from this yacySeed object - */ - @Deprecated - public final String getPublicAddress() { - return getPublicAddress(getIP()); - } /** * generate a public address using a given ip. This combines the ip with the port and encloses the ip diff --git a/source/net/yacy/peers/SeedDB.java b/source/net/yacy/peers/SeedDB.java index a401dd83a..525fa3176 100644 --- a/source/net/yacy/peers/SeedDB.java +++ b/source/net/yacy/peers/SeedDB.java @@ -966,6 +966,11 @@ public final class SeedDB implements AlternativeDomainNames { } } + /** + * use getConnected(hash) instead and retrieve the addresses from the seed + * @param targetHash + * @return + */ @Deprecated public String targetAddress(final String targetHash) { // find target address @@ -980,19 +985,6 @@ public final class SeedDB implements AlternativeDomainNames { if (address == null) address = "localhost" + (this.mySeed.getPort() > 0 ? ":" + this.mySeed.getPort() : ""); return address; } - - public Set targetIPs(final String targetHash) { - // find target address - Set ips; - if (targetHash.equals(mySeed().hash)) { - ips = mySeed().getIPs(); - } else { - final Seed targetSeed = getConnected(targetHash); - if (targetSeed == null) { return null; } - ips = targetSeed.getIPs(); - } - return ips; - } private class seedEnum implements Iterator { diff --git a/source/net/yacy/search/snippet/ResultEntry.java b/source/net/yacy/search/snippet/ResultEntry.java index 2c20652b5..e1f361ffb 100644 --- a/source/net/yacy/search/snippet/ResultEntry.java +++ b/source/net/yacy/search/snippet/ResultEntry.java @@ -76,7 +76,7 @@ public class ResultEntry implements Comparable, Comparator