*) adding missing calls for function close() to avoid "too many open file" bug

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@282 6c8d7289-2bf4-0310-a012-ef5d649a1542

source/de/anomic/http/httpHeader.java

@@ -276,16 +276,20 @@ public final class httpHeader extends TreeMap implements Map {
     
     // convenience methods for storing and loading to a file system
     public void store(File f) throws IOException {
-	FileOutputStream fos = new FileOutputStream(f);
+        FileOutputStream fos = null;
-	Iterator i = keySet().iterator();
+        try {
-	String key, value;
+            fos = new FileOutputStream(f);
-	while (i.hasNext()) {
+            Iterator i = keySet().iterator();
-	    key = (String) i.next();
+            String key, value;
-	    value = (String) get(key);
+            while (i.hasNext()) {
-	    fos.write((key + "=" + value + "\r\n").getBytes());
+                key = (String) i.next();
-	}
+                value = (String) get(key);
-	fos.flush();
+                fos.write((key + "=" + value + "\r\n").getBytes());
-	fos.close();
+            }
+            fos.flush();
+        } finally {
+            if (fos != null) try{fos.close();}catch(Exception e){}
+        }
     }
 
     public String toString() {

source/de/anomic/http/httpd.java

@@ -1117,8 +1117,8 @@ public final class httpd implements serverHandler {
             ByteArrayOutputStream o = new ByteArrayOutputStream();
             fis = new FileInputStream(file);
             httpTemplate.writeTemplate(fis, o, tp, "-UNRESOLVED_PATTERN-".getBytes());
-            o.close();
             result = o.toByteArray();
+            o.close(); o = null;
 
             httpHeader header = new httpHeader();            
             header.put(httpHeader.DATE, httpc.dateString(httpc.nowDate()));

source/de/anomic/http/httpdFileHandler.java

@@ -455,7 +455,7 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http
                         }                        
                     } finally {
                         if (zippedOut != null) try {zippedOut.close();} catch(Exception e) {}
-                        if (o != null) try {o.close();} catch(Exception e) {}
+                        if (o != null) try {o.close(); o = null;} catch(Exception e) {}
                         if (fis != null) try {fis.close();} catch(Exception e) {}
                     }
                     

source/de/anomic/kelondro/kelondroDyn.java

@@ -350,30 +350,41 @@ public class kelondroDyn extends kelondroTree {
     }
     
     public synchronized void writeFile(String key, File f) throws IOException {
-	// reads a file from the FS and writes it into the database
+        // reads a file from the FS and writes it into the database
-	kelondroRA kra = getRA(key);
+        kelondroRA kra = null;
-	byte[] buffer = new byte[1024];
+        FileInputStream fis = null;
-	byte[] result = new byte[(int) f.length()];
+        try {
-	FileInputStream fis = new FileInputStream(f);
+            kra = getRA(key);
-	int i;
+            byte[] buffer = new byte[1024];
-	int pos = 0;
+            byte[] result = new byte[(int) f.length()];
-	while ((i = fis.read(buffer)) > 0) {
+            fis = new FileInputStream(f);
-	    System.arraycopy(buffer, 0, result, pos, i);
+            int i;
-	    pos += i;
+            int pos = 0;
-	}
+            while ((i = fis.read(buffer)) > 0) {
-	fis.close();
+                System.arraycopy(buffer, 0, result, pos, i);
-	kra.writeArray(result);
+                pos += i;
-	kra.close();
+            }
+            fis.close();
+            kra.writeArray(result);
+        } finally {
+            if (fis != null) try{fis.close();}catch(Exception e){}
+            if (kra != null) try{kra.close();}catch(Exception e){}
+        }
     }
 
     public synchronized void readFile(String key, File f) throws IOException {
-	// reads a file from the DB and writes it to the FS
+        // reads a file from the DB and writes it to the FS
-	kelondroRA kra = getRA(key);
+        kelondroRA kra = null;
-	byte[] result = kra.readArray();
+        FileOutputStream fos = null;
-	FileOutputStream fos = new FileOutputStream(f);
+        try {
-	fos.write(result);
+            kra = getRA(key);
-	fos.close();
+            byte[] result = kra.readArray();
-	kra.close();
+            fos = new FileOutputStream(f);
+            fos.write(result);
+        } finally {
+            if (fos != null) try{fos.close();}catch(Exception e){}
+            if (kra != null) try{kra.close();}catch(Exception e){}
+        }
     }
 
     public static void main(String[] args) {

source/de/anomic/plasma/parser/AbstractParser.java

@@ -115,10 +115,12 @@ public abstract class AbstractParser implements Parser{
         BufferedInputStream contentInputStream = null;
         try {
             contentInputStream = new BufferedInputStream(new FileInputStream(sourceFile));
+            return this.parse(location, mimeType, contentInputStream);
         } catch (FileNotFoundException e) {
-            e.printStackTrace();
+            throw new ParserException(e.getMessage());
+        } finally {
+            if (contentInputStream != null) try{contentInputStream.close();}catch(Exception e){}
         }
-        return this.parse(location, mimeType, contentInputStream);
 	}
     
     /**

source/de/anomic/plasma/plasmaCrawlWorker.java

@@ -283,9 +283,13 @@ public final class plasmaCrawlWorker extends Thread {
                     } else if ((profile == null) || ((profile.storeHTCache()) && ((error = htCache.shallStoreCache()) == null))) {
                         // we write the new cache entry to file system directly
                         cacheFile.getParentFile().mkdirs();
-                        FileOutputStream fos = new FileOutputStream(cacheFile);
+                        FileOutputStream fos = null;
-                        htCache.cacheArray = res.writeContent(fos); // writes in cacheArray and cache file
+                        try {
-                        fos.close();
+                            fos = new FileOutputStream(cacheFile);
+                            htCache.cacheArray = res.writeContent(fos); // writes in cacheArray and cache file
+                        } finally {
+                            if (fos!=null)try{fos.close();}catch(Exception e){}
+                        }
                         htCache.status = plasmaHTCache.CACHE_FILL;
                     } else {
                         if (error != null) log.logDebug("CRAWLER NOT STORED RESOURCE " + url.toString() + ": " + error);

source/de/anomic/plasma/plasmaParser.java

@@ -45,21 +45,18 @@
 
 package de.anomic.plasma;
 
+import java.io.BufferedInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.FileFilter;
 import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
 import java.io.FilenameFilter;
 import java.io.IOException;
 import java.io.OutputStream;
 import java.net.MalformedURLException;
 import java.net.URI;
 import java.net.URL;
-import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Collection;
 import java.util.Enumeration;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -264,11 +261,14 @@ public final class plasmaParser {
     public static String getMimeTypeByFileExt(String fileExt) {
         // loading a list of extensions from file
         Properties prop = new Properties();
-        try {
+        BufferedInputStream bufferedIn = null;
-            prop.load(new FileInputStream(new File("httpd.mime")));
+        try {            
+            prop.load(bufferedIn = new BufferedInputStream(new FileInputStream(new File("httpd.mime"))));
         } catch (IOException e) {
             System.err.println("ERROR: httpd.mime not found in settings path");
-        }            
+        } finally {
+            if (bufferedIn != null) try{bufferedIn.close();}catch(Exception e){}
+        }
         
         return prop.getProperty(fileExt,"application/octet-stream");
     }
@@ -342,11 +342,14 @@ public final class plasmaParser {
     private static void loadEnabledParserList() {
         // loading a list of availabe parser from file
     	Properties prop = new Properties();
+        BufferedInputStream bufferedIn = null;
     	try {
-    	    prop.load(new FileInputStream(new File("yacy.parser")));
+    	    prop.load(bufferedIn = new BufferedInputStream(new FileInputStream(new File("yacy.parser"))));
     	} catch (IOException e) {
     	    System.err.println("ERROR: yacy.parser not found in settings path");
-    	}    	
+    	} finally {
+            if (bufferedIn != null) try{ bufferedIn.close(); }catch(Exception e){}
+        }
 
         // enable them ...
         setEnabledParserList(prop.keySet());

source/de/anomic/plasma/plasmaStore.java

@@ -45,37 +45,46 @@ public class plasmaStore {
 
     // some static helper methods
     public static void saveGzip(File f, byte[] content) throws IOException {
-	f.getParentFile().mkdirs();
+        java.util.zip.GZIPOutputStream gzipout = null;
-	java.util.zip.GZIPOutputStream gzipout = new java.util.zip.GZIPOutputStream(new FileOutputStream(f));
+        try {
-	gzipout.write(content, 0, content.length); 
+            f.getParentFile().mkdirs();
-	gzipout.close();
+            gzipout = new java.util.zip.GZIPOutputStream(new FileOutputStream(f));
+            gzipout.write(content, 0, content.length);
+        } finally {
+            if (gzipout!=null)try{gzipout.close();}catch(Exception e){}
+        }        
     }
 
     public static byte[] loadGzip(File f) throws IOException {
-	java.util.zip.GZIPInputStream gzipin = new java.util.zip.GZIPInputStream(new FileInputStream(f));
+        java.util.zip.GZIPInputStream gzipin = null;
-	byte[] result = new byte[1024];
+        try {
-	byte[] buffer = new byte[512];
+            gzipin = new java.util.zip.GZIPInputStream(new FileInputStream(f));
-	byte[] b;
+            byte[] result = new byte[1024];
-	int len = 0;
+            byte[] buffer = new byte[512];
-	int last;
+            byte[] b;
-	while ((last = gzipin.read(buffer, 0, buffer.length)) > 0) {
+            int len = 0;
-	    // assert the buffer to the result
+            int last;
-	    while (result.length - len < last) {
+            while ((last = gzipin.read(buffer, 0, buffer.length)) > 0) {
-		// the result array is too small, increase space
+                // assert the buffer to the result
-		b = new byte[result.length * 2];
+                while (result.length - len < last) {
-		System.arraycopy(result, 0, b, 0, len); 
+                    // the result array is too small, increase space
-		result = b; b = null;
+                    b = new byte[result.length * 2];
-	    }
+                    System.arraycopy(result, 0, b, 0, len); 
-	    // copy the last read
+                    result = b; b = null;
-	    System.arraycopy(buffer, 0, result, len, last); 
+                }
-	    len += last;
+                // copy the last read
-	}
+                System.arraycopy(buffer, 0, result, len, last); 
-	gzipin.close();
+                len += last;
-	// finished with reading. now cut the result to the right size
+            }
-	b = new byte[len];
+            gzipin.close();
-	System.arraycopy(result, 0, b, 0, len); 
+            // finished with reading. now cut the result to the right size
-	result = null;
+            b = new byte[len];
-	return b;
+            System.arraycopy(result, 0, b, 0, len); 
+            result = null;
+            return b;
+        } finally {
+            if (gzipin != null) try{gzipin.close();}catch(Exception e){}
+        }
     }
 
     /*    public static void saveProperties(File f, Properties props, String comment) throws IOException {

source/de/anomic/plasma/plasmaSwitchboard.java

@@ -420,16 +420,21 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
     private static TreeSet loadList(File file) {
         TreeSet list = new TreeSet(kelondroMSetTools.fastStringComparator);
         if (!(file.exists())) return list;
-	try {
+        
-	    BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(file)));
+        BufferedReader br = null;
-	    String line;
+        try {
-	    while ((line = br.readLine()) != null) {
+            br = new BufferedReader(new InputStreamReader(new FileInputStream(file)));
-		line = line.trim();
+            String line;
-		if ((line.length() > 0) && (!(line.startsWith("#")))) list.add(line.trim().toLowerCase());
+            while ((line = br.readLine()) != null) {
-	    }
+                line = line.trim();
-	    br.close();
+                if ((line.length() > 0) && (!(line.startsWith("#")))) list.add(line.trim().toLowerCase());
-	} catch (IOException e) {}
+            }
-	return list;
+            br.close();
+        } catch (IOException e) {            
+        } finally {
+            if (br != null) try{br.close();}catch(Exception e){}
+        }
+        return list;
     }
 
     public void close() {

source/de/anomic/server/serverAbstractSwitch.java

@@ -40,6 +40,8 @@
 
 package de.anomic.server;
 
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileOutputStream;
@@ -131,31 +133,38 @@ public abstract class serverAbstractSwitch implements serverSwitch {
     }
 
     public static Map loadHashMap(File f) {
-	// load props
+        // load props
-	Properties prop = new Properties();
+        Properties prop = new Properties();
-	try {
+        BufferedInputStream bufferedIn = null;
-	    prop.load(new FileInputStream(f));
+        try {
-	} catch (IOException e1) {
+            prop.load(bufferedIn = new BufferedInputStream(new FileInputStream(f)));
-	    System.err.println("ERROR: " + f.toString() + " not found in settings path");
+        } catch (IOException e1) {
-	    prop = null;
+            System.err.println("ERROR: " + f.toString() + " not found in settings path");
-	}
+            prop = null;
-	return (Hashtable) prop;
+        } finally {
+            if (bufferedIn != null)try{bufferedIn.close();}catch(Exception e){}
+        }
+        return (Hashtable) prop;
     }
 
     public static void saveMap(File f, Map props, String comment) throws IOException {
-	PrintWriter pw = new PrintWriter(new FileOutputStream(f));
+        PrintWriter pw = null;
-	pw.println("# " + comment);
+        try {
-	Iterator i = props.entrySet().iterator();
+            pw = new PrintWriter(new BufferedOutputStream(new FileOutputStream(f)));
-	String key, value;
+            pw.println("# " + comment);
-	Map.Entry entry;
+            Iterator i = props.entrySet().iterator();
-	while (i.hasNext()) {
+            String key, value;
-	    entry  = (Map.Entry) i.next();
+            Map.Entry entry;
-	    key = (String) entry.getKey();
+            while (i.hasNext()) {
-	    value = ((String) entry.getValue()).replaceAll("\n", "\\\\n");
+                entry  = (Map.Entry) i.next();
-	    pw.println(key + "=" + value);
+                key = (String) entry.getKey();
-	}
+                value = ((String) entry.getValue()).replaceAll("\n", "\\\\n");
-	pw.println("# EOF");
+                pw.println(key + "=" + value);
-	pw.close();
+            }
+            pw.println("# EOF");
+        } finally {
+          if (pw!=null)try{pw.close();}catch(Exception e){}  
+        }
     }
 
     public void setConfig(String key, long value) {

source/de/anomic/yacy/seedUpload/yacySeedUploadScp.java

@@ -1,5 +1,6 @@
 package de.anomic.yacy.seedUpload;
 
+import java.io.BufferedInputStream;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
@@ -116,12 +117,17 @@ class sshc {
             checkAck(in);
             
             // send a content of lfile
-            FileInputStream fis=new FileInputStream(localFile);
             byte[] buf=new byte[1024];
-            while(true){
+            BufferedInputStream bufferedIn = null;
-                int len=fis.read(buf, 0, buf.length);
+            try {
-                if(len<=0) break;
+                bufferedIn=new BufferedInputStream(new FileInputStream(localFile));                
-                out.write(buf, 0, len); out.flush();
+                while(true){
+                    int len=bufferedIn.read(buf, 0, buf.length);
+                    if(len<=0) break;
+                    out.write(buf, 0, len); out.flush();
+                }
+            } finally {
+                if (bufferedIn != null) try{bufferedIn.close();}catch(Exception e){}
             }
             
             // send '\0'

source/de/anomic/yacy/yacyPeerActions.java

@@ -183,27 +183,31 @@ public class yacyPeerActions {
     }
 
     private disorderSet loadSuperseed(File local, String url) {
-	// this returns a list of locations where seed list-files can be found
+        // this returns a list of locations where seed list-files can be found
-	disorderSet supsee = new disorderSet();
+        disorderSet supsee = new disorderSet();
-	String line;
+        String line;
-	// read in local file
+        // read in local file
         int lc = 0;
+        BufferedReader br = null;
         try {
-	    BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(local)));
+            br = new BufferedReader(new InputStreamReader(new FileInputStream(local)));
-	    while ((line = br.readLine()) != null) {
+            while ((line = br.readLine()) != null) {
-		line = line.trim();
+                line = line.trim();
-		//System.out.println("one line in file:" + line);
+                //System.out.println("one line in file:" + line);
-		if (line.length() > 0) supsee.add(line);
+                if (line.length() > 0) supsee.add(line);
-	    }
+            }
-	    br.close();
+            br.close();
             lc = supsee.size();
             yacyCore.log.logInfo("BOOTSTRAP: " + lc + " seed-list urls from superseed file " + local.toString());
-	} catch (IOException e) {
+        } catch (IOException e) {
-	    //e.printStackTrace();
+            //e.printStackTrace();
-	    supsee = new disorderSet();
+            supsee = new disorderSet();
             yacyCore.log.logInfo("BOOTSTRAP: failed to load seed-list urls from superseed file " + local.toString() + ": " + e.getMessage());
-	}
+        } finally {
-	// read in remote file from url
+            if (br!=null)try{br.close();}catch(Exception e){}
+        }
+        
+        // read in remote file from url
         try {
             Vector remote = httpc.wget(new URL(url), 5000, null, null, sb.remoteProxyHost, sb.remoteProxyPort);
             if ((remote != null) && (remote.size() > 0)) {