From 703f4273034ee0b1d7a687bf9ed13a7ef094c6c9 Mon Sep 17 00:00:00 2001 From: Michael Peter Christen Date: Thu, 16 Aug 2012 17:11:54 +0200 Subject: [PATCH] fixed some peer-ping connection details - larger time-out - removed too old seedlist - fixed a bug in connection test --- defaults/yacy.network.freeworld.unit | 9 +++---- htroot/ViewProfile.java | 24 ++++++++--------- source/de/anomic/crawler/CrawlQueues.java | 2 +- source/net/yacy/peers/Network.java | 4 +-- source/net/yacy/peers/Protocol.java | 33 ++++++++++++----------- 5 files changed, 36 insertions(+), 36 deletions(-) diff --git a/defaults/yacy.network.freeworld.unit b/defaults/yacy.network.freeworld.unit index cb67df042..3c4e40aab 100644 --- a/defaults/yacy.network.freeworld.unit +++ b/defaults/yacy.network.freeworld.unit @@ -76,11 +76,10 @@ network.unit.bootstrap.seedlist1 = http://home.arcor.de/hermens/yacy/seed.txt network.unit.bootstrap.seedlist2 = http://low.audioattack.de/yacy/seed.txt network.unit.bootstrap.seedlist3 = http://www.lulabad.de/seed.txt network.unit.bootstrap.seedlist4 = http://fennec.cx/yacy/seed.txt -network.unit.bootstrap.seedlist5 = http://yacy.dvz24.net/yacy/seed.txt -network.unit.bootstrap.seedlist6 = http://sixcooler.de/yacy/seed.txt -network.unit.bootstrap.seedlist7 = http://headrift.dyndns.org/yacy/seed.txt -network.unit.bootstrap.seedlist8 = http://dk5ras.dyndns.org/seed.txt -network.unit.bootstrap.seedlist9 = http://yacy.bonus-communis-bibliotheca.eu/seed.txt +network.unit.bootstrap.seedlist5 = http://sixcooler.de/yacy/seed.txt +network.unit.bootstrap.seedlist6 = http://headrift.dyndns.org/yacy/seed.txt +network.unit.bootstrap.seedlist7 = http://dk5ras.dyndns.org/seed.txt +network.unit.bootstrap.seedlist8 = http://yacy.bonus-communis-bibliotheca.eu/seed.txt # each network may use different yacy distributions. # the auto-updater can access network-specific update locations diff --git a/htroot/ViewProfile.java b/htroot/ViewProfile.java index 4b358b8d0..4e3b3adb8 100644 --- a/htroot/ViewProfile.java +++ b/htroot/ViewProfile.java @@ -1,4 +1,4 @@ -// ViewProfile_p.java +// ViewProfile_p.java // ----------------------- // (C) 2009 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany // first published 07.04.2005 on http://yacy.net @@ -43,13 +43,12 @@ import java.util.Properties; import net.yacy.cora.protocol.RequestHeader; import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.order.Base64Order; -import net.yacy.peers.Protocol; import net.yacy.peers.Network; import net.yacy.peers.NewsDB; import net.yacy.peers.NewsPool; +import net.yacy.peers.Protocol; import net.yacy.peers.Seed; import net.yacy.search.Switchboard; - import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; @@ -63,23 +62,23 @@ public class ViewProfile { prop.put("display", display); prop.put("edit", authenticated ? 1 : 0); final String hash = (post == null) ? null : post.get("hash"); - + if ((hash == null) || (sb.peers == null)) { // wrong access prop.put("success", "0"); return prop; } prop.put("hash", hash); - + // get the profile Map profile = null; - if (hash.equals("localhash")) { + if (hash.equals("localhash") || sb.peers.mySeed().hash.equals(hash)) { // read the profile from local peer final Properties p = new Properties(); FileInputStream fileIn = null; try { fileIn = new FileInputStream(new File("DATA/SETTINGS/profile.txt")); - p.load(fileIn); + p.load(fileIn); } catch(final IOException e) {} finally { if (fileIn != null) try { fileIn.close(); fileIn = null; } catch (final Exception e) {} } @@ -93,6 +92,7 @@ public class ViewProfile { // read the profile from remote peer Seed seed = sb.peers.getConnected(hash); if (seed == null) seed = sb.peers.getDisconnected(hash); + if (seed == null) seed = sb.peers.getPotential(hash); if (seed == null) { prop.put("success", "1"); // peer unknown } else { @@ -103,11 +103,11 @@ public class ViewProfile { } catch (final Exception e) { Log.logException(e); } - + // try to get the profile from remote peer if (sb.clusterhashes != null) seed.setAlternativeAddress(sb.clusterhashes.get(seed.hash.getBytes())); profile = Protocol.getProfile(seed); - + // if profile did not arrive, say that peer is disconnected if (profile == null) { prop.put("success", "2"); // peer known, but disconnected @@ -138,14 +138,14 @@ public class ViewProfile { knownKeys.add("yahoo"); knownKeys.add("msn"); knownKeys.add("skype"); - knownKeys.add("comment"); + knownKeys.add("comment"); //empty values final Iterator it = knownKeys.iterator(); while (it.hasNext()) { prop.put("success_" + it.next(), "0"); } - + //number of not explicitly recognized but displayed items int numUnknown = 0; while (i.hasNext()) { @@ -162,7 +162,7 @@ public class ViewProfile { prop.put("success_" + key, "1"); // only comments get "wikified" if(key.equals("comment")){ - prop.putWiki(sb.peers.mySeed().getClusterAddress(), + prop.putWiki(sb.peers.mySeed().getClusterAddress(), "success_" + key + "_value", entry.getValue().replaceAll("\r", "").replaceAll("\\\\n", "\n")); prop.put("success_" + key + "_b64value", Base64Order.standardCoder.encodeString(entry.getValue())); diff --git a/source/de/anomic/crawler/CrawlQueues.java b/source/de/anomic/crawler/CrawlQueues.java index d7dc86969..ec35a37fa 100644 --- a/source/de/anomic/crawler/CrawlQueues.java +++ b/source/de/anomic/crawler/CrawlQueues.java @@ -486,7 +486,7 @@ public class CrawlQueues { } // we know a peer which should provide remote crawl entries. load them now. - final RSSFeed feed = Protocol.queryRemoteCrawlURLs(this.sb.peers, seed, 60, 8000); + final RSSFeed feed = Protocol.queryRemoteCrawlURLs(this.sb.peers, seed, 60, 10000); if (feed == null || feed.isEmpty()) { // something is wrong with this provider. To prevent that we get not stuck with this peer // we remove it from the peer list diff --git a/source/net/yacy/peers/Network.java b/source/net/yacy/peers/Network.java index 936f89b5d..dcaae8c16 100644 --- a/source/net/yacy/peers/Network.java +++ b/source/net/yacy/peers/Network.java @@ -77,7 +77,7 @@ public class Network public static long magic = System.currentTimeMillis(); public static final Map amIAccessibleDB = new ConcurrentHashMap(); // Holds PeerHash / yacyAccessible Relations // constants for PeerPing behavior - private static final int PING_INITIAL = 10; + private static final int PING_INITIAL = 20; private static final int PING_MAX_RUNNING = 3; private static final int PING_MIN_RUNNING = 1; private static final int PING_MIN_DBSIZE = 5; @@ -461,7 +461,7 @@ public class Network int i = 0; while ( si.hasNext() ) { seed = si.next(); - if ( seed == null ) { + if ( seed == null || seed.hash.equals(this.sb.peers.mySeed().hash)) { sync.acquire(); continue; } diff --git a/source/net/yacy/peers/Protocol.java b/source/net/yacy/peers/Protocol.java index 9bff39417..1e561a9f5 100644 --- a/source/net/yacy/peers/Protocol.java +++ b/source/net/yacy/peers/Protocol.java @@ -175,6 +175,7 @@ public final class Protocol Map result = null; final String salt = crypt.randomSalt(); long responseTime = Long.MAX_VALUE; + byte[] content = null; try { // generate request final Map parts = @@ -186,22 +187,13 @@ public final class Protocol final long start = System.currentTimeMillis(); // final byte[] content = HTTPConnector.getConnector(MultiProtocolURI.yacybotUserAgent).post(new MultiProtocolURI("http://" + address + "/yacy/hello.html"), 30000, yacySeed.b64Hash2hexHash(otherHash) + ".yacyh", parts); final HTTPClient httpClient = new HTTPClient(ClientIdentification.getUserAgent(), 30000); - final byte[] content = + content = httpClient.POSTbytes( new MultiProtocolURI("http://" + address + "/yacy/hello.html"), Seed.b64Hash2hexHash(otherHash) + ".yacyh", parts, false); responseTime = System.currentTimeMillis() - start; - Network.log.logInfo("yacyClient.hello thread '" - + Thread.currentThread().getName() - + "' contacted peer at " - + address - + ", received " - + ((content == null) ? "null" : content.length) - + " bytes, time = " - + responseTime - + " milliseconds"); result = FileUtils.table(content); } catch ( final Exception e ) { if ( Thread.currentThread().isInterrupted() ) { @@ -220,11 +212,20 @@ public final class Protocol result = null; } - if ( result == null ) { + if (result == null || result.size() == 0) { Network.log.logInfo("yacyClient.hello result error: " + ((result == null) ? "result null" : ("result=" + result.toString()))); return -1; } + Network.log.logInfo("yacyClient.hello thread '" + + Thread.currentThread().getName() + + "' contacted peer at " + + address + + ", received " + + ((content == null) ? "null" : content.length) + + " bytes, time = " + + responseTime + + " milliseconds"); // check consistency with expectation Seed otherPeer = null; @@ -421,7 +422,7 @@ public final class Protocol parts.put("object", UTF8.StringBody("rwicount")); parts.put("ttl", UTF8.StringBody("0")); parts.put("env", UTF8.StringBody(wordHash)); - final byte[] content = postToFile(target, "query.html", parts, 5000); + final byte[] content = postToFile(target, "query.html", parts, 6000); final Map result = FileUtils.table(content); if ( result == null || result.isEmpty() ) { @@ -457,7 +458,7 @@ public final class Protocol parts.put("object", UTF8.StringBody("lurlcount")); parts.put("ttl", UTF8.StringBody("0")); parts.put("env", UTF8.StringBody("")); - final byte[] content = postToFile(target, "query.html", parts, 5000); + final byte[] content = postToFile(target, "query.html", parts, 6000); final Map result = FileUtils.table(content); if ( result == null || result.isEmpty() ) { @@ -965,7 +966,7 @@ public final class Protocol //resultMap = FileUtils.table(HTTPConnector.getConnector(MultiProtocolURI.crawlerUserAgent).post(new MultiProtocolURI("http://" + target.getClusterAddress() + "/yacy/search.html"), 60000, target.getHexHash() + ".yacyh", parts)); } - final HTTPClient httpClient = new HTTPClient(ClientIdentification.getUserAgent(), 8000); + final HTTPClient httpClient = new HTTPClient(ClientIdentification.getUserAgent(), 20000); byte[] a = httpClient.POSTbytes(new MultiProtocolURI("http://" + hostaddress + "/yacy/search.html"), hostname, parts, false); if (a != null && a.length > 200000) { // there is something wrong. This is too large, maybe a hack on the other side? @@ -1040,7 +1041,7 @@ public final class Protocol final Map parts = basicRequestParts(Switchboard.getSwitchboard(), targetHash, salt); parts.put("process", UTF8.StringBody("permission")); - final byte[] content = postToFile(seedDB, targetHash, "message.html", parts, 5000); + final byte[] content = postToFile(seedDB, targetHash, "message.html", parts, 6000); final Map result = FileUtils.table(content); return result; } catch ( final Exception e ) { @@ -1400,7 +1401,7 @@ public final class Protocol final Map parts = basicRequestParts(Switchboard.getSwitchboard(), targetSeed.hash, salt); // final byte[] content = HTTPConnector.getConnector(MultiProtocolURI.yacybotUserAgent).post(new MultiProtocolURI("http://" + address + "/yacy/profile.html"), 5000, targetSeed.getHexHash() + ".yacyh", parts); - final HTTPClient httpclient = new HTTPClient(ClientIdentification.getUserAgent(), 5000); + final HTTPClient httpclient = new HTTPClient(ClientIdentification.getUserAgent(), 15000); final byte[] content = httpclient.POSTbytes( new MultiProtocolURI("http://" + address + "/yacy/profile.html"),