some changes that may help to prevent deadlocks that cause an OutOfMemoryError

as described in
http://www.yacy-forum.de/viewtopic.php?p=24359

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@2353 6c8d7289-2bf4-0310-a012-ef5d649a1542

build.properties

@@ -3,7 +3,7 @@ javacSource=1.4
 javacTarget=1.4
 
 # Release Configuration
-releaseVersion=0.457
+releaseVersion=0.458
 releaseFile=yacy_dev_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz
 #releaseFile=yacy_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz
 releaseDir=yacy_dev_v${releaseVersion}_${DSTAMP}_${releaseNr}

source/de/anomic/http/httpdFileHandler.java

@@ -118,7 +118,9 @@ import de.anomic.ymage.ymagePainter;
 
 public final class httpdFileHandler extends httpdAbstractHandler implements httpdHandler {
     
-    // class variables   
+    private static final boolean safeServletsMode = false; // if true then all servlets are called synchronized
+    
+    // class variables
     private static final Properties mimeTable = new Properties();
     private static final serverClassLoader provider;
     private static final HashMap templates = new HashMap();
@@ -476,7 +478,7 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http
                     requestHeader.put("CLIENTIP", conProp.getProperty("CLIENTIP"));
                     requestHeader.put("PATH", path);
                     // in case that there are no args given, args = null or empty hashmap
-                    img = rewriteMethod(targetClass).invoke(null, new Object[] {requestHeader, args, switchboard});
+                    img = invokeServlet(targetClass, requestHeader, args);
                 } catch (InvocationTargetException e) {
                     this.theLogger.logSevere("INTERNAL ERROR: " + e.toString() + ":" +
                     e.getMessage() +
@@ -560,8 +562,7 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http
                 httpd.sendRespondHeader(this.connectionProperties, out, httpVersion, 200, null);                
                 
                 // in case that there are no args given, args = null or empty hashmap
-                /* serverObjects tp = (serverObjects) */ rewriteMethod(targetClass).invoke(null, new Object[] {requestHeader, args, switchboard});
-             
+                /* serverObjects tp = (serverObjects) */ invokeServlet(targetClass, requestHeader, args);
                 this.forceConnectionClose();
                 return;                
             } else if ((targetFile.exists()) && (targetFile.canRead())) {
@@ -597,7 +598,7 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http
                             requestHeader.put("CLIENTIP", conProp.getProperty("CLIENTIP"));
                             requestHeader.put("PATH", path);
                             // in case that there are no args given, args = null or empty hashmap
-                            tp = (serverObjects) rewriteMethod(targetClass).invoke(null, new Object[] {requestHeader, args, switchboard});
+                            tp = (serverObjects) invokeServlet(targetClass, requestHeader, args);
                             // if no args given , then tp will be an empty Hashtable object (not null)
                             if (tp == null) tp = new serverObjects();
                             // check if the servlets requests authentification
@@ -708,8 +709,8 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http
                                 //warning: o,tp and fis are reused
                                 httpTemplate.writeTemplate(fis, o, tp, "-UNRESOLVED_PATTERN-".getBytes());
                                 
-                                if(pageClass != null && pageClass.exists())
-                                    tp = (serverObjects) rewriteMethod(pageClass).invoke(null, new Object[] {requestHeader, args, switchboard});
+                                if (pageClass != null && pageClass.exists()) 
+                                    tp = (serverObjects) invokeServlet(pageClass, requestHeader, args);
                                 else
                                     tp = new serverObjects();
                                 tp.put("page", o.toString());
@@ -910,7 +911,7 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http
             }
         }
     }
-
+    
     private File getOverlayedClass(String path) {
         File targetClass;
         targetClass=rewriteClassFile(new File(htDefaultPath, path)); //works for default and localized files
@@ -994,6 +995,16 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http
         return m;
     }
     
+    private Object invokeServlet(File targetClass, httpHeader request, serverObjects args) throws IllegalArgumentException, IllegalAccessException, InvocationTargetException {
+        Object result;
+        if (safeServletsMode) synchronized (switchboard) {
+            result = rewriteMethod(targetClass).invoke(null, new Object[] {request, args, switchboard});
+        } else {
+            result = rewriteMethod(targetClass).invoke(null, new Object[] {request, args, switchboard});
+        }
+        return result;
+    }
+
     public void doConnect(Properties conProp, httpHeader requestHeader, InputStream clientIn, OutputStream clientOut) {
         throw new UnsupportedOperationException();
     }

source/de/anomic/index/indexURLEntry.java

@@ -90,7 +90,7 @@ public class indexURLEntry implements Cloneable, indexEntry {
            ) {
 
         // more needed attributes:
-        // - boolean: appearance attributes: title, appears in header, anchor-descr, image-tag etc
+        // - boolean: appearance attributes: title, appears in header, anchor-descr, image-tag, hervorhebungen, meta-tags, word in link etc
         // - boolean: URL attributes
 
         if ((language == null) || (language.length() != indexURL.urlLanguageLength)) language = "uk";

source/de/anomic/kelondro/kelondroAbstractIOChunks.java

@@ -69,8 +69,9 @@ public abstract class kelondroAbstractIOChunks {
         if (len < 0) throw new IndexOutOfBoundsException("length is negative:" + len);
         if (b.length < off + len) throw new IndexOutOfBoundsException("bounds do not fit: b.length=" + b.length + ", off=" + off + ", len=" + len);
         while (len > 0) {
-            int r = read(pos, b, off, len);
+            int r = read(pos, b, off, len); // blocks until at least one byte is available
             if (r < 0) throw new IOException("EOF"); // read exceeded EOF
+            if (r == 0) throw new IOException("readFully cannot read remaining " + len + " bytes"); // security exception to prevent endless loops
             pos += r;
             off += r;
             len -= r;

source/de/anomic/kelondro/kelondroAbstractRA.java

@@ -64,6 +64,8 @@ abstract class kelondroAbstractRA implements kelondroRA {
     }
 
     // pseudo-native methods:
+    abstract public long available() throws IOException;
+    
     abstract public int read() throws IOException;
     abstract public void write(int b) throws IOException;
 

source/de/anomic/kelondro/kelondroBufferedRA.java

@@ -45,6 +45,8 @@ import java.io.IOException;
 
 public class kelondroBufferedRA extends kelondroAbstractRA implements kelondroRA {
 
+    // FIXME: a lot of synchronization of ra is needed here
+    
     protected kelondroRA ra; 
     protected byte[] buffer;
     protected int bufferPage;
@@ -75,6 +77,13 @@ public class kelondroBufferedRA extends kelondroAbstractRA implements kelondroRA
         this.bufferWritten = true;
     }
 
+    public long available() throws IOException {
+        synchronized (ra) {
+            ra.seek(seekpos);
+            return ra.available();
+        }
+    }
+    
     private void readBuffer(int newPageNr) throws IOException {
         if (newPageNr == bufferPage) return;
         bufferPage = newPageNr;

source/de/anomic/kelondro/kelondroCachedRA.java

@@ -64,7 +64,14 @@ public class kelondroCachedRA extends kelondroAbstractRA implements kelondroRA {
         this.cacheMaxElements = cachesize / cacheElementSize;
         this.seekpos = 0;
     }
-
+    
+    public long available() throws IOException {
+        synchronized (ra) {
+            ra.seek(seekpos);
+            return ra.available();
+        }
+    }
+    
     private int cacheElementNumber(long address) {
         return (int) address / cacheElementSize;
     }

source/de/anomic/kelondro/kelondroDyn.java

@@ -334,6 +334,10 @@ public class kelondroDyn extends kelondroTree {
             this.filekey = filekey;
         }
 
+        public long available() throws IOException {
+            return Long.MAX_VALUE;
+        }
+        
         public int read() throws IOException {
             return getDyn(filekey, seekpos++);
         }

source/de/anomic/kelondro/kelondroFileRA.java

@@ -60,6 +60,10 @@ public final class kelondroFileRA extends kelondroAbstractRA implements kelondro
         RAFile = new RandomAccessFile(file, "rw");
     }
     
+    public long available() throws IOException {
+        return RAFile.length() - RAFile.getFilePointer();
+    }
+    
     // pseudo-native method read
     public int read() throws IOException {
         return RAFile.read();

source/de/anomic/kelondro/kelondroNIOFileRA.java

@@ -113,6 +113,10 @@ public class kelondroNIOFileRA extends kelondroAbstractRA implements kelondroRA
         return true;
     }
     
+    public long available() throws IOException {
+        return RAFile.length() - RAFile.getFilePointer();
+    }
+    
     // pseudo-native method read
     public int read() throws IOException {
         int r;

source/de/anomic/kelondro/kelondroRA.java

@@ -59,6 +59,8 @@ public interface kelondroRA {
     public String name();
     
     // pseudo-native methods:
+    public long available() throws IOException;
+    
     public int read() throws IOException;
     public void write(int b) throws IOException;
 

source/de/anomic/kelondro/kelondroRAIOChunks.java

@@ -53,9 +53,17 @@ public final class kelondroRAIOChunks extends kelondroAbstractIOChunks implement
     }
 
     public int read(long pos, byte[] b, int off, int len) throws IOException {
+        if (len == 0) return 0;
         synchronized (this.ra) {
             this.ra.seek(pos);
-            return ra.read(b, off, len);
+            long available = ra.available();
+            if (available >= len) {
+                return ra.read(b, off, len);
+            } else if (available == 0) {
+                return -1;
+            } else {
+                return ra.read(b, off, (int) available);
+            }
         }
     }
 

source/de/anomic/server/serverAbstractSwitch.java

@@ -153,12 +153,14 @@ public abstract class serverAbstractSwitch implements serverSwitch {
 
     public void setConfig(String key, String value) {
         // perform action before setting new value
+        Iterator bevore = switchActions.entrySet().iterator();
+        Iterator after  = switchActions.entrySet().iterator();
         synchronized (configProps) {
             Map.Entry entry;
             serverSwitchAction action;
-            Iterator i = switchActions.entrySet().iterator();
-            while (i.hasNext()) {
-                entry = (Map.Entry) i.next();
+            
+            while (bevore.hasNext()) {
+                entry = (Map.Entry) bevore.next();
                 action = (serverSwitchAction) entry.getValue();
                 try {
                     action.doBevoreSetConfig(key, value);
@@ -172,9 +174,8 @@ public abstract class serverAbstractSwitch implements serverSwitch {
             saveConfig();
 
             // perform actions afterwards
-            i = switchActions.entrySet().iterator();
-            while (i.hasNext()) {
-                entry = (Map.Entry) i.next();
+            while (after.hasNext()) {
+                entry = (Map.Entry) after.next();
                 action = (serverSwitchAction) entry.getValue();
                 try {
                     action.doAfterSetConfig(key, value, (oldValue == null) ? null : (String) oldValue);
@@ -186,6 +187,7 @@ public abstract class serverAbstractSwitch implements serverSwitch {
     }
 
     public String getConfig(String key, String dflt) {
+        Iterator i = switchActions.entrySet().iterator();
         synchronized (configProps) {
             // get the value
             Object s = configProps.get(key);
@@ -193,7 +195,6 @@ public abstract class serverAbstractSwitch implements serverSwitch {
             // do action
             Map.Entry entry;
             serverSwitchAction action;
-            Iterator i = switchActions.entrySet().iterator();
             while (i.hasNext()) {
                 entry = (Map.Entry) i.next();
                 action = (serverSwitchAction) entry.getValue();