From d468d665c93d512170913b3ce667b09414e7763d Mon Sep 17 00:00:00 2001 From: orbiter Date: Mon, 7 Aug 2006 00:19:01 +0000 Subject: [PATCH] some changes that may help to prevent deadlocks that cause an OutOfMemoryError as described in http://www.yacy-forum.de/viewtopic.php?p=24359 git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@2353 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- build.properties | 2 +- source/de/anomic/http/httpdFileHandler.java | 27 +++++++++++++------ source/de/anomic/index/indexURLEntry.java | 2 +- .../kelondro/kelondroAbstractIOChunks.java | 3 ++- .../anomic/kelondro/kelondroAbstractRA.java | 2 ++ .../anomic/kelondro/kelondroBufferedRA.java | 9 +++++++ .../de/anomic/kelondro/kelondroCachedRA.java | 9 ++++++- source/de/anomic/kelondro/kelondroDyn.java | 4 +++ source/de/anomic/kelondro/kelondroFileRA.java | 4 +++ .../de/anomic/kelondro/kelondroNIOFileRA.java | 4 +++ source/de/anomic/kelondro/kelondroRA.java | 2 ++ .../anomic/kelondro/kelondroRAIOChunks.java | 10 ++++++- .../anomic/server/serverAbstractSwitch.java | 15 ++++++----- 13 files changed, 73 insertions(+), 20 deletions(-) diff --git a/build.properties b/build.properties index c62bc027b..575eb4452 100644 --- a/build.properties +++ b/build.properties @@ -3,7 +3,7 @@ javacSource=1.4 javacTarget=1.4 # Release Configuration -releaseVersion=0.457 +releaseVersion=0.458 releaseFile=yacy_dev_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz #releaseFile=yacy_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz releaseDir=yacy_dev_v${releaseVersion}_${DSTAMP}_${releaseNr} diff --git a/source/de/anomic/http/httpdFileHandler.java b/source/de/anomic/http/httpdFileHandler.java index ae470a435..4452a3b8f 100644 --- a/source/de/anomic/http/httpdFileHandler.java +++ b/source/de/anomic/http/httpdFileHandler.java @@ -118,7 +118,9 @@ import de.anomic.ymage.ymagePainter; public final class httpdFileHandler extends httpdAbstractHandler implements httpdHandler { - // class variables + private static final boolean safeServletsMode = false; // if true then all servlets are called synchronized + + // class variables private static final Properties mimeTable = new Properties(); private static final serverClassLoader provider; private static final HashMap templates = new HashMap(); @@ -476,7 +478,7 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http requestHeader.put("CLIENTIP", conProp.getProperty("CLIENTIP")); requestHeader.put("PATH", path); // in case that there are no args given, args = null or empty hashmap - img = rewriteMethod(targetClass).invoke(null, new Object[] {requestHeader, args, switchboard}); + img = invokeServlet(targetClass, requestHeader, args); } catch (InvocationTargetException e) { this.theLogger.logSevere("INTERNAL ERROR: " + e.toString() + ":" + e.getMessage() + @@ -560,8 +562,7 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http httpd.sendRespondHeader(this.connectionProperties, out, httpVersion, 200, null); // in case that there are no args given, args = null or empty hashmap - /* serverObjects tp = (serverObjects) */ rewriteMethod(targetClass).invoke(null, new Object[] {requestHeader, args, switchboard}); - + /* serverObjects tp = (serverObjects) */ invokeServlet(targetClass, requestHeader, args); this.forceConnectionClose(); return; } else if ((targetFile.exists()) && (targetFile.canRead())) { @@ -597,7 +598,7 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http requestHeader.put("CLIENTIP", conProp.getProperty("CLIENTIP")); requestHeader.put("PATH", path); // in case that there are no args given, args = null or empty hashmap - tp = (serverObjects) rewriteMethod(targetClass).invoke(null, new Object[] {requestHeader, args, switchboard}); + tp = (serverObjects) invokeServlet(targetClass, requestHeader, args); // if no args given , then tp will be an empty Hashtable object (not null) if (tp == null) tp = new serverObjects(); // check if the servlets requests authentification @@ -708,8 +709,8 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http //warning: o,tp and fis are reused httpTemplate.writeTemplate(fis, o, tp, "-UNRESOLVED_PATTERN-".getBytes()); - if(pageClass != null && pageClass.exists()) - tp = (serverObjects) rewriteMethod(pageClass).invoke(null, new Object[] {requestHeader, args, switchboard}); + if (pageClass != null && pageClass.exists()) + tp = (serverObjects) invokeServlet(pageClass, requestHeader, args); else tp = new serverObjects(); tp.put("page", o.toString()); @@ -910,7 +911,7 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http } } } - + private File getOverlayedClass(String path) { File targetClass; targetClass=rewriteClassFile(new File(htDefaultPath, path)); //works for default and localized files @@ -994,6 +995,16 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http return m; } + private Object invokeServlet(File targetClass, httpHeader request, serverObjects args) throws IllegalArgumentException, IllegalAccessException, InvocationTargetException { + Object result; + if (safeServletsMode) synchronized (switchboard) { + result = rewriteMethod(targetClass).invoke(null, new Object[] {request, args, switchboard}); + } else { + result = rewriteMethod(targetClass).invoke(null, new Object[] {request, args, switchboard}); + } + return result; + } + public void doConnect(Properties conProp, httpHeader requestHeader, InputStream clientIn, OutputStream clientOut) { throw new UnsupportedOperationException(); } diff --git a/source/de/anomic/index/indexURLEntry.java b/source/de/anomic/index/indexURLEntry.java index 3d6484268..f244d33a3 100644 --- a/source/de/anomic/index/indexURLEntry.java +++ b/source/de/anomic/index/indexURLEntry.java @@ -90,7 +90,7 @@ public class indexURLEntry implements Cloneable, indexEntry { ) { // more needed attributes: - // - boolean: appearance attributes: title, appears in header, anchor-descr, image-tag etc + // - boolean: appearance attributes: title, appears in header, anchor-descr, image-tag, hervorhebungen, meta-tags, word in link etc // - boolean: URL attributes if ((language == null) || (language.length() != indexURL.urlLanguageLength)) language = "uk"; diff --git a/source/de/anomic/kelondro/kelondroAbstractIOChunks.java b/source/de/anomic/kelondro/kelondroAbstractIOChunks.java index d66fc388b..9466358e5 100644 --- a/source/de/anomic/kelondro/kelondroAbstractIOChunks.java +++ b/source/de/anomic/kelondro/kelondroAbstractIOChunks.java @@ -69,8 +69,9 @@ public abstract class kelondroAbstractIOChunks { if (len < 0) throw new IndexOutOfBoundsException("length is negative:" + len); if (b.length < off + len) throw new IndexOutOfBoundsException("bounds do not fit: b.length=" + b.length + ", off=" + off + ", len=" + len); while (len > 0) { - int r = read(pos, b, off, len); + int r = read(pos, b, off, len); // blocks until at least one byte is available if (r < 0) throw new IOException("EOF"); // read exceeded EOF + if (r == 0) throw new IOException("readFully cannot read remaining " + len + " bytes"); // security exception to prevent endless loops pos += r; off += r; len -= r; diff --git a/source/de/anomic/kelondro/kelondroAbstractRA.java b/source/de/anomic/kelondro/kelondroAbstractRA.java index f80f3f933..1d27ec5ce 100644 --- a/source/de/anomic/kelondro/kelondroAbstractRA.java +++ b/source/de/anomic/kelondro/kelondroAbstractRA.java @@ -64,6 +64,8 @@ abstract class kelondroAbstractRA implements kelondroRA { } // pseudo-native methods: + abstract public long available() throws IOException; + abstract public int read() throws IOException; abstract public void write(int b) throws IOException; diff --git a/source/de/anomic/kelondro/kelondroBufferedRA.java b/source/de/anomic/kelondro/kelondroBufferedRA.java index 03bac06fc..2ed5817cb 100644 --- a/source/de/anomic/kelondro/kelondroBufferedRA.java +++ b/source/de/anomic/kelondro/kelondroBufferedRA.java @@ -45,6 +45,8 @@ import java.io.IOException; public class kelondroBufferedRA extends kelondroAbstractRA implements kelondroRA { + // FIXME: a lot of synchronization of ra is needed here + protected kelondroRA ra; protected byte[] buffer; protected int bufferPage; @@ -75,6 +77,13 @@ public class kelondroBufferedRA extends kelondroAbstractRA implements kelondroRA this.bufferWritten = true; } + public long available() throws IOException { + synchronized (ra) { + ra.seek(seekpos); + return ra.available(); + } + } + private void readBuffer(int newPageNr) throws IOException { if (newPageNr == bufferPage) return; bufferPage = newPageNr; diff --git a/source/de/anomic/kelondro/kelondroCachedRA.java b/source/de/anomic/kelondro/kelondroCachedRA.java index ccf0adb8f..87dee1bf1 100644 --- a/source/de/anomic/kelondro/kelondroCachedRA.java +++ b/source/de/anomic/kelondro/kelondroCachedRA.java @@ -64,7 +64,14 @@ public class kelondroCachedRA extends kelondroAbstractRA implements kelondroRA { this.cacheMaxElements = cachesize / cacheElementSize; this.seekpos = 0; } - + + public long available() throws IOException { + synchronized (ra) { + ra.seek(seekpos); + return ra.available(); + } + } + private int cacheElementNumber(long address) { return (int) address / cacheElementSize; } diff --git a/source/de/anomic/kelondro/kelondroDyn.java b/source/de/anomic/kelondro/kelondroDyn.java index d8ebd250d..5eaf4ba3a 100644 --- a/source/de/anomic/kelondro/kelondroDyn.java +++ b/source/de/anomic/kelondro/kelondroDyn.java @@ -334,6 +334,10 @@ public class kelondroDyn extends kelondroTree { this.filekey = filekey; } + public long available() throws IOException { + return Long.MAX_VALUE; + } + public int read() throws IOException { return getDyn(filekey, seekpos++); } diff --git a/source/de/anomic/kelondro/kelondroFileRA.java b/source/de/anomic/kelondro/kelondroFileRA.java index 0075da758..169dbdbf0 100644 --- a/source/de/anomic/kelondro/kelondroFileRA.java +++ b/source/de/anomic/kelondro/kelondroFileRA.java @@ -60,6 +60,10 @@ public final class kelondroFileRA extends kelondroAbstractRA implements kelondro RAFile = new RandomAccessFile(file, "rw"); } + public long available() throws IOException { + return RAFile.length() - RAFile.getFilePointer(); + } + // pseudo-native method read public int read() throws IOException { return RAFile.read(); diff --git a/source/de/anomic/kelondro/kelondroNIOFileRA.java b/source/de/anomic/kelondro/kelondroNIOFileRA.java index c270557a0..bfbddbcfb 100644 --- a/source/de/anomic/kelondro/kelondroNIOFileRA.java +++ b/source/de/anomic/kelondro/kelondroNIOFileRA.java @@ -113,6 +113,10 @@ public class kelondroNIOFileRA extends kelondroAbstractRA implements kelondroRA return true; } + public long available() throws IOException { + return RAFile.length() - RAFile.getFilePointer(); + } + // pseudo-native method read public int read() throws IOException { int r; diff --git a/source/de/anomic/kelondro/kelondroRA.java b/source/de/anomic/kelondro/kelondroRA.java index c0be9d9e5..bc831d3f1 100644 --- a/source/de/anomic/kelondro/kelondroRA.java +++ b/source/de/anomic/kelondro/kelondroRA.java @@ -59,6 +59,8 @@ public interface kelondroRA { public String name(); // pseudo-native methods: + public long available() throws IOException; + public int read() throws IOException; public void write(int b) throws IOException; diff --git a/source/de/anomic/kelondro/kelondroRAIOChunks.java b/source/de/anomic/kelondro/kelondroRAIOChunks.java index e91218827..9d36482d9 100644 --- a/source/de/anomic/kelondro/kelondroRAIOChunks.java +++ b/source/de/anomic/kelondro/kelondroRAIOChunks.java @@ -53,9 +53,17 @@ public final class kelondroRAIOChunks extends kelondroAbstractIOChunks implement } public int read(long pos, byte[] b, int off, int len) throws IOException { + if (len == 0) return 0; synchronized (this.ra) { this.ra.seek(pos); - return ra.read(b, off, len); + long available = ra.available(); + if (available >= len) { + return ra.read(b, off, len); + } else if (available == 0) { + return -1; + } else { + return ra.read(b, off, (int) available); + } } } diff --git a/source/de/anomic/server/serverAbstractSwitch.java b/source/de/anomic/server/serverAbstractSwitch.java index 515f12b0b..adb375606 100644 --- a/source/de/anomic/server/serverAbstractSwitch.java +++ b/source/de/anomic/server/serverAbstractSwitch.java @@ -153,12 +153,14 @@ public abstract class serverAbstractSwitch implements serverSwitch { public void setConfig(String key, String value) { // perform action before setting new value + Iterator bevore = switchActions.entrySet().iterator(); + Iterator after = switchActions.entrySet().iterator(); synchronized (configProps) { Map.Entry entry; serverSwitchAction action; - Iterator i = switchActions.entrySet().iterator(); - while (i.hasNext()) { - entry = (Map.Entry) i.next(); + + while (bevore.hasNext()) { + entry = (Map.Entry) bevore.next(); action = (serverSwitchAction) entry.getValue(); try { action.doBevoreSetConfig(key, value); @@ -172,9 +174,8 @@ public abstract class serverAbstractSwitch implements serverSwitch { saveConfig(); // perform actions afterwards - i = switchActions.entrySet().iterator(); - while (i.hasNext()) { - entry = (Map.Entry) i.next(); + while (after.hasNext()) { + entry = (Map.Entry) after.next(); action = (serverSwitchAction) entry.getValue(); try { action.doAfterSetConfig(key, value, (oldValue == null) ? null : (String) oldValue); @@ -186,6 +187,7 @@ public abstract class serverAbstractSwitch implements serverSwitch { } public String getConfig(String key, String dflt) { + Iterator i = switchActions.entrySet().iterator(); synchronized (configProps) { // get the value Object s = configProps.get(key); @@ -193,7 +195,6 @@ public abstract class serverAbstractSwitch implements serverSwitch { // do action Map.Entry entry; serverSwitchAction action; - Iterator i = switchActions.entrySet().iterator(); while (i.hasNext()) { entry = (Map.Entry) i.next(); action = (serverSwitchAction) entry.getValue();