From 155d55656895fdcabec85fa377a041539d209920 Mon Sep 17 00:00:00 2001 From: orbiter Date: Wed, 27 Oct 2010 13:21:18 +0000 Subject: [PATCH] - better memory protection - more logging - little bit of refactoring git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7278 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- htroot/Network.java | 4 +- htroot/SettingsAck_p.java | 4 +- htroot/Settings_p.java | 2 +- htroot/Status.java | 2 +- source/de/anomic/server/serverSwitch.java | 2 +- source/de/anomic/yacy/yacyCore.java | 4 +- source/de/anomic/yacy/yacySearch.java | 8 +++- source/de/anomic/yacy/yacySeed.java | 13 ++--- source/de/anomic/yacy/yacySeedDB.java | 2 +- .../yacy/cora/protocol/TimeoutRequest.java | 47 +++++++++++-------- 10 files changed, 51 insertions(+), 37 deletions(-) diff --git a/htroot/Network.java b/htroot/Network.java index df1db5def..71c1e1bec 100644 --- a/htroot/Network.java +++ b/htroot/Network.java @@ -145,7 +145,7 @@ public class Network { prop.putNum("table_my-qph-pubremote", Math.round(6000d * sb.averageQPMGlobal()) / 100d); prop.putNum("table_my-seeds", Long.parseLong(seed.get(yacySeed.SCOUNT, "0"))); prop.putNum("table_my-connects", Double.parseDouble(seed.get(yacySeed.CCOUNT, "0"))); - prop.put("table_my-url", seed.get(yacySeed.SEEDLIST, "")); + prop.put("table_my-url", seed.get(yacySeed.SEEDLISTURL, "")); // generating the location string prop.putHTML("table_my-location", MultiProtocolURI.generateLocation()); @@ -392,7 +392,7 @@ public class Network { } else if(seed.isPrincipal()) { prop.put(STR_TABLE_LIST + conCount + "_type", 2); } - prop.putHTML(STR_TABLE_LIST + conCount + "_type_url", seed.get(yacySeed.SEEDLIST, "http://nowhere/")); + prop.putHTML(STR_TABLE_LIST + conCount + "_type_url", seed.get(yacySeed.SEEDLISTURL, "http://nowhere/")); final long lastseen = Math.abs((System.currentTimeMillis() - seed.getLastSeenUTC()) / 1000 / 60); if (page == 1 && lastseen > 720) { diff --git a/htroot/SettingsAck_p.java b/htroot/SettingsAck_p.java index f9b3df8d3..0210df0e0 100644 --- a/htroot/SettingsAck_p.java +++ b/htroot/SettingsAck_p.java @@ -347,7 +347,7 @@ public class SettingsAck_p { // get the currently used uploading method final String oldSeedUploadMethod = env.getConfig("seedUploadMethod","none"); final String newSeedUploadMethod = post.get("seedUploadMethod"); - final String oldSeedURLStr = sb.peers.mySeed().get(yacySeed.SEEDLIST, ""); + final String oldSeedURLStr = sb.peers.mySeed().get(yacySeed.SEEDLISTURL, ""); final String newSeedURLStr = post.get("seedURL"); final boolean seedUrlChanged = !oldSeedURLStr.equals(newSeedURLStr); @@ -358,7 +358,7 @@ public class SettingsAck_p { if (seedUrlChanged || uploadMethodChanged) { env.setConfig("seedUploadMethod", newSeedUploadMethod); - sb.peers.mySeed().put(yacySeed.SEEDLIST, newSeedURLStr); + sb.peers.mySeed().put(yacySeed.SEEDLISTURL, newSeedURLStr); // try an upload String error; diff --git a/htroot/Settings_p.java b/htroot/Settings_p.java index e19aeb51a..7c46349a2 100644 --- a/htroot/Settings_p.java +++ b/htroot/Settings_p.java @@ -189,7 +189,7 @@ public final class Settings_p { } // general settings - prop.put("seedURL", sb.peers.mySeed().get(yacySeed.SEEDLIST, "")); + prop.put("seedURL", sb.peers.mySeed().get(yacySeed.SEEDLISTURL, "")); /* * Message forwarding configuration diff --git a/htroot/Status.java b/htroot/Status.java index 979a72142..94a69bcfb 100644 --- a/htroot/Status.java +++ b/htroot/Status.java @@ -231,7 +231,7 @@ public class Status { } else if (peerStatus.equals(yacySeed.PEERTYPE_PRINCIPAL)) { prop.put(PEERSTATUS, "3"); prop.put("hintStatusPrincipal", "1"); - prop.put("hintStatusPrincipal_seedURL", sb.peers.mySeed().get(yacySeed.SEEDLIST, "?")); + prop.put("hintStatusPrincipal_seedURL", sb.peers.mySeed().get(yacySeed.SEEDLISTURL, "?")); } prop.putHTML("peerName", thisName); prop.put("hash", thisHash); diff --git a/source/de/anomic/server/serverSwitch.java b/source/de/anomic/server/serverSwitch.java index 7f26392fd..14628a1e1 100644 --- a/source/de/anomic/server/serverSwitch.java +++ b/source/de/anomic/server/serverSwitch.java @@ -148,7 +148,7 @@ public class serverSwitch { public String myPublicIP() { // if a static IP was configured, we have to return it here ... final String staticIP = getConfig("staticIP", ""); - if ((!staticIP.equals(""))) { + if (staticIP.length() > 0) { return staticIP; } diff --git a/source/de/anomic/yacy/yacyCore.java b/source/de/anomic/yacy/yacyCore.java index fe5128650..180d50c6d 100644 --- a/source/de/anomic/yacy/yacyCore.java +++ b/source/de/anomic/yacy/yacyCore.java @@ -619,7 +619,7 @@ public class yacyCore { // ensure that the seed file url is configured properly DigestURI seedURL; try { - final String seedURLStr = sb.peers.mySeed().get(yacySeed.SEEDLIST, ""); + final String seedURLStr = sb.peers.mySeed().get(yacySeed.SEEDLISTURL, ""); if (seedURLStr.length() == 0) { throw new MalformedURLException("The seed-file url must not be empty."); } if (!( seedURLStr.toLowerCase().startsWith("http://") || @@ -629,7 +629,7 @@ public class yacyCore { } seedURL = new DigestURI(seedURLStr); } catch (final MalformedURLException e) { - final String errorMsg = "Malformed seed file URL '" + sb.peers.mySeed().get(yacySeed.SEEDLIST, "") + "'. " + e.getMessage(); + final String errorMsg = "Malformed seed file URL '" + sb.peers.mySeed().get(yacySeed.SEEDLISTURL, "") + "'. " + e.getMessage(); log.logWarning("SaveSeedList: " + errorMsg); return errorMsg; } diff --git a/source/de/anomic/yacy/yacySearch.java b/source/de/anomic/yacy/yacySearch.java index 74abaea26..8d23a51b4 100644 --- a/source/de/anomic/yacy/yacySearch.java +++ b/source/de/anomic/yacy/yacySearch.java @@ -288,12 +288,16 @@ public class yacySearch extends Thread { final yacySearch[] searchThreads = new yacySearch[targets]; for (int i = 0; i < targets; i++) { if (targetPeers[i] == null || targetPeers[i].hash == null) continue; - searchThreads[i] = new yacySearch( + try { + searchThreads[i] = new yacySearch( wordhashes, excludehashes, "", prefer, filter, language, sitehash, authorhash, count, maxDist, true, targets, targetPeers[i], indexSegment, peers, crawlResults, containerCache, secondarySearchSuperviser, blacklist, rankingProfile, constraint); - searchThreads[i].start(); + searchThreads[i].start(); + } catch (OutOfMemoryError e) { + break; + } } return searchThreads; } diff --git a/source/de/anomic/yacy/yacySeed.java b/source/de/anomic/yacy/yacySeed.java index 9a8cc404c..c9c062192 100644 --- a/source/de/anomic/yacy/yacySeed.java +++ b/source/de/anomic/yacy/yacySeed.java @@ -153,7 +153,7 @@ public class yacySeed implements Cloneable { public static final String CRTCNT = "CRTCnt"; public static final String IP = "IP"; public static final String PORT = "Port"; - public static final String SEEDLIST = "seedURL"; + public static final String SEEDLISTURL = "seedURL"; /** zero-value */ private static final String ZERO = "0"; @@ -789,6 +789,7 @@ public class yacySeed implements Cloneable { return resultSeed; } + // TODO: add here IP ranges to accept also intranet networks public final String isProper(final boolean checkOwnIP) { // checks if everything is ok with that seed @@ -816,7 +817,7 @@ public class yacySeed implements Cloneable { } // seedURL - final String seedURL = this.dna.get(SEEDLIST); + final String seedURL = this.dna.get(SEEDLISTURL); if (seedURL != null && seedURL.length() > 0) { if (!seedURL.startsWith("http://") && !seedURL.startsWith("https://")) return "wrong protocol for seedURL"; try { @@ -832,11 +833,11 @@ public class yacySeed implements Cloneable { public static final String isProperIP(final String ipString) { // returns null if ipString is proper, a string with the cause otherwise - if (ipString == null) return "IP is null"; - if (ipString.length() > 0 && ipString.length() < 8) return "IP is too short: " + ipString; + if (ipString == null) return ipString + " -> IP is null"; + if (ipString.length() > 0 && ipString.length() < 8) return ipString + " -> IP is too short: "; InetAddress ip = Domains.dnsResolve(ipString); - if (ip == null) return "IP is not proper: " + ipString; //this does not work with staticIP - if (ipString.equals("localhost") || ipString.startsWith("127.") || (ipString.startsWith("0:0:0:0:0:0:0:1"))) return "IP for localhost rejected"; + if (ip == null) return ipString + " -> IP is not proper"; //this does not work with staticIP + if (ipString.equals("localhost") || ipString.startsWith("127.") || ipString.startsWith("0:0:0:0:0:0:0:1")) return ipString + " - IP for localhost rejected"; return null; } diff --git a/source/de/anomic/yacy/yacySeedDB.java b/source/de/anomic/yacy/yacySeedDB.java index 264ccda61..30432731e 100644 --- a/source/de/anomic/yacy/yacySeedDB.java +++ b/source/de/anomic/yacy/yacySeedDB.java @@ -199,7 +199,7 @@ public final class yacySeedDB implements AlternativeDomainNames { if (myOwnSeedFile.length() > 0) try { // load existing identity mySeed = yacySeed.load(myOwnSeedFile); - if(mySeed == null) throw new IOException("current seed is null"); + if (mySeed == null) throw new IOException("current seed is null"); } catch (final IOException e) { // create new identity Log.logSevere("SEEDDB", "could not load stored mySeed.txt from " + myOwnSeedFile.toString() + ": " + e.getMessage() + ". creating new seed.", e); diff --git a/source/net/yacy/cora/protocol/TimeoutRequest.java b/source/net/yacy/cora/protocol/TimeoutRequest.java index 7796aa300..e53e9a9be 100644 --- a/source/net/yacy/cora/protocol/TimeoutRequest.java +++ b/source/net/yacy/cora/protocol/TimeoutRequest.java @@ -65,26 +65,35 @@ public class TimeoutRequest { */ public E call(long timeout) throws ExecutionException { ExecutorService service = Executors.newSingleThreadExecutor(); - final Future taskFuture = service.submit(this.call); - Runnable t = new Runnable() { - public void run() { taskFuture.cancel(true); } - }; - service.execute(t); - service.shutdown(); try { - return taskFuture.get(timeout, TimeUnit.MILLISECONDS); - } catch (CancellationException e) { - // callable was interrupted - throw new ExecutionException(e); - } catch (InterruptedException e) { - // service was shutdown - throw new ExecutionException(e); - } catch (ExecutionException e) { - // callable failed unexpectedly - throw e; - } catch (TimeoutException e) { - // time-out - throw new ExecutionException(e); + final Future taskFuture = service.submit(this.call); + Runnable t = new Runnable() { + public void run() { taskFuture.cancel(true); } + }; + service.execute(t); + service.shutdown(); + try { + return taskFuture.get(timeout, TimeUnit.MILLISECONDS); + } catch (CancellationException e) { + // callable was interrupted + throw new ExecutionException(e); + } catch (InterruptedException e) { + // service was shutdown + throw new ExecutionException(e); + } catch (ExecutionException e) { + // callable failed unexpectedly + throw e; + } catch (TimeoutException e) { + // time-out + throw new ExecutionException(e); + } + } catch (OutOfMemoryError e) { + // in case that no memory is there to create a new native thread + try { + return this.call.call(); + } catch (Exception e1) { + throw new ExecutionException(e1); + } } }