*) adding missing calls for function close() to avoid "too many open file" bug

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@282 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
theli 20 years ago
parent 9a98988c3c
commit 9e47ba5ad6

@ -276,16 +276,20 @@ public final class httpHeader extends TreeMap implements Map {
// convenience methods for storing and loading to a file system // convenience methods for storing and loading to a file system
public void store(File f) throws IOException { public void store(File f) throws IOException {
FileOutputStream fos = new FileOutputStream(f); FileOutputStream fos = null;
Iterator i = keySet().iterator(); try {
String key, value; fos = new FileOutputStream(f);
while (i.hasNext()) { Iterator i = keySet().iterator();
key = (String) i.next(); String key, value;
value = (String) get(key); while (i.hasNext()) {
fos.write((key + "=" + value + "\r\n").getBytes()); key = (String) i.next();
} value = (String) get(key);
fos.flush(); fos.write((key + "=" + value + "\r\n").getBytes());
fos.close(); }
fos.flush();
} finally {
if (fos != null) try{fos.close();}catch(Exception e){}
}
} }
public String toString() { public String toString() {

@ -1117,8 +1117,8 @@ public final class httpd implements serverHandler {
ByteArrayOutputStream o = new ByteArrayOutputStream(); ByteArrayOutputStream o = new ByteArrayOutputStream();
fis = new FileInputStream(file); fis = new FileInputStream(file);
httpTemplate.writeTemplate(fis, o, tp, "-UNRESOLVED_PATTERN-".getBytes()); httpTemplate.writeTemplate(fis, o, tp, "-UNRESOLVED_PATTERN-".getBytes());
o.close();
result = o.toByteArray(); result = o.toByteArray();
o.close(); o = null;
httpHeader header = new httpHeader(); httpHeader header = new httpHeader();
header.put(httpHeader.DATE, httpc.dateString(httpc.nowDate())); header.put(httpHeader.DATE, httpc.dateString(httpc.nowDate()));

@ -455,7 +455,7 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http
} }
} finally { } finally {
if (zippedOut != null) try {zippedOut.close();} catch(Exception e) {} if (zippedOut != null) try {zippedOut.close();} catch(Exception e) {}
if (o != null) try {o.close();} catch(Exception e) {} if (o != null) try {o.close(); o = null;} catch(Exception e) {}
if (fis != null) try {fis.close();} catch(Exception e) {} if (fis != null) try {fis.close();} catch(Exception e) {}
} }

@ -350,30 +350,41 @@ public class kelondroDyn extends kelondroTree {
} }
public synchronized void writeFile(String key, File f) throws IOException { public synchronized void writeFile(String key, File f) throws IOException {
// reads a file from the FS and writes it into the database // reads a file from the FS and writes it into the database
kelondroRA kra = getRA(key); kelondroRA kra = null;
byte[] buffer = new byte[1024]; FileInputStream fis = null;
byte[] result = new byte[(int) f.length()]; try {
FileInputStream fis = new FileInputStream(f); kra = getRA(key);
int i; byte[] buffer = new byte[1024];
int pos = 0; byte[] result = new byte[(int) f.length()];
while ((i = fis.read(buffer)) > 0) { fis = new FileInputStream(f);
System.arraycopy(buffer, 0, result, pos, i); int i;
pos += i; int pos = 0;
} while ((i = fis.read(buffer)) > 0) {
fis.close(); System.arraycopy(buffer, 0, result, pos, i);
kra.writeArray(result); pos += i;
kra.close(); }
fis.close();
kra.writeArray(result);
} finally {
if (fis != null) try{fis.close();}catch(Exception e){}
if (kra != null) try{kra.close();}catch(Exception e){}
}
} }
public synchronized void readFile(String key, File f) throws IOException { public synchronized void readFile(String key, File f) throws IOException {
// reads a file from the DB and writes it to the FS // reads a file from the DB and writes it to the FS
kelondroRA kra = getRA(key); kelondroRA kra = null;
byte[] result = kra.readArray(); FileOutputStream fos = null;
FileOutputStream fos = new FileOutputStream(f); try {
fos.write(result); kra = getRA(key);
fos.close(); byte[] result = kra.readArray();
kra.close(); fos = new FileOutputStream(f);
fos.write(result);
} finally {
if (fos != null) try{fos.close();}catch(Exception e){}
if (kra != null) try{kra.close();}catch(Exception e){}
}
} }
public static void main(String[] args) { public static void main(String[] args) {

@ -115,10 +115,12 @@ public abstract class AbstractParser implements Parser{
BufferedInputStream contentInputStream = null; BufferedInputStream contentInputStream = null;
try { try {
contentInputStream = new BufferedInputStream(new FileInputStream(sourceFile)); contentInputStream = new BufferedInputStream(new FileInputStream(sourceFile));
return this.parse(location, mimeType, contentInputStream);
} catch (FileNotFoundException e) { } catch (FileNotFoundException e) {
e.printStackTrace(); throw new ParserException(e.getMessage());
} finally {
if (contentInputStream != null) try{contentInputStream.close();}catch(Exception e){}
} }
return this.parse(location, mimeType, contentInputStream);
} }
/** /**

@ -283,9 +283,13 @@ public final class plasmaCrawlWorker extends Thread {
} else if ((profile == null) || ((profile.storeHTCache()) && ((error = htCache.shallStoreCache()) == null))) { } else if ((profile == null) || ((profile.storeHTCache()) && ((error = htCache.shallStoreCache()) == null))) {
// we write the new cache entry to file system directly // we write the new cache entry to file system directly
cacheFile.getParentFile().mkdirs(); cacheFile.getParentFile().mkdirs();
FileOutputStream fos = new FileOutputStream(cacheFile); FileOutputStream fos = null;
htCache.cacheArray = res.writeContent(fos); // writes in cacheArray and cache file try {
fos.close(); fos = new FileOutputStream(cacheFile);
htCache.cacheArray = res.writeContent(fos); // writes in cacheArray and cache file
} finally {
if (fos!=null)try{fos.close();}catch(Exception e){}
}
htCache.status = plasmaHTCache.CACHE_FILL; htCache.status = plasmaHTCache.CACHE_FILL;
} else { } else {
if (error != null) log.logDebug("CRAWLER NOT STORED RESOURCE " + url.toString() + ": " + error); if (error != null) log.logDebug("CRAWLER NOT STORED RESOURCE " + url.toString() + ": " + error);

@ -45,21 +45,18 @@
package de.anomic.plasma; package de.anomic.plasma;
import java.io.BufferedInputStream;
import java.io.ByteArrayOutputStream; import java.io.ByteArrayOutputStream;
import java.io.File; import java.io.File;
import java.io.FileFilter; import java.io.FileFilter;
import java.io.FileInputStream; import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FilenameFilter; import java.io.FilenameFilter;
import java.io.IOException; import java.io.IOException;
import java.io.OutputStream; import java.io.OutputStream;
import java.net.MalformedURLException; import java.net.MalformedURLException;
import java.net.URI; import java.net.URI;
import java.net.URL; import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.Collection;
import java.util.Enumeration; import java.util.Enumeration;
import java.util.HashMap; import java.util.HashMap;
import java.util.HashSet; import java.util.HashSet;
@ -264,11 +261,14 @@ public final class plasmaParser {
public static String getMimeTypeByFileExt(String fileExt) { public static String getMimeTypeByFileExt(String fileExt) {
// loading a list of extensions from file // loading a list of extensions from file
Properties prop = new Properties(); Properties prop = new Properties();
try { BufferedInputStream bufferedIn = null;
prop.load(new FileInputStream(new File("httpd.mime"))); try {
prop.load(bufferedIn = new BufferedInputStream(new FileInputStream(new File("httpd.mime"))));
} catch (IOException e) { } catch (IOException e) {
System.err.println("ERROR: httpd.mime not found in settings path"); System.err.println("ERROR: httpd.mime not found in settings path");
} } finally {
if (bufferedIn != null) try{bufferedIn.close();}catch(Exception e){}
}
return prop.getProperty(fileExt,"application/octet-stream"); return prop.getProperty(fileExt,"application/octet-stream");
} }
@ -342,11 +342,14 @@ public final class plasmaParser {
private static void loadEnabledParserList() { private static void loadEnabledParserList() {
// loading a list of availabe parser from file // loading a list of availabe parser from file
Properties prop = new Properties(); Properties prop = new Properties();
BufferedInputStream bufferedIn = null;
try { try {
prop.load(new FileInputStream(new File("yacy.parser"))); prop.load(bufferedIn = new BufferedInputStream(new FileInputStream(new File("yacy.parser"))));
} catch (IOException e) { } catch (IOException e) {
System.err.println("ERROR: yacy.parser not found in settings path"); System.err.println("ERROR: yacy.parser not found in settings path");
} } finally {
if (bufferedIn != null) try{ bufferedIn.close(); }catch(Exception e){}
}
// enable them ... // enable them ...
setEnabledParserList(prop.keySet()); setEnabledParserList(prop.keySet());

@ -45,37 +45,46 @@ public class plasmaStore {
// some static helper methods // some static helper methods
public static void saveGzip(File f, byte[] content) throws IOException { public static void saveGzip(File f, byte[] content) throws IOException {
f.getParentFile().mkdirs(); java.util.zip.GZIPOutputStream gzipout = null;
java.util.zip.GZIPOutputStream gzipout = new java.util.zip.GZIPOutputStream(new FileOutputStream(f)); try {
gzipout.write(content, 0, content.length); f.getParentFile().mkdirs();
gzipout.close(); gzipout = new java.util.zip.GZIPOutputStream(new FileOutputStream(f));
gzipout.write(content, 0, content.length);
} finally {
if (gzipout!=null)try{gzipout.close();}catch(Exception e){}
}
} }
public static byte[] loadGzip(File f) throws IOException { public static byte[] loadGzip(File f) throws IOException {
java.util.zip.GZIPInputStream gzipin = new java.util.zip.GZIPInputStream(new FileInputStream(f)); java.util.zip.GZIPInputStream gzipin = null;
byte[] result = new byte[1024]; try {
byte[] buffer = new byte[512]; gzipin = new java.util.zip.GZIPInputStream(new FileInputStream(f));
byte[] b; byte[] result = new byte[1024];
int len = 0; byte[] buffer = new byte[512];
int last; byte[] b;
while ((last = gzipin.read(buffer, 0, buffer.length)) > 0) { int len = 0;
// assert the buffer to the result int last;
while (result.length - len < last) { while ((last = gzipin.read(buffer, 0, buffer.length)) > 0) {
// the result array is too small, increase space // assert the buffer to the result
b = new byte[result.length * 2]; while (result.length - len < last) {
System.arraycopy(result, 0, b, 0, len); // the result array is too small, increase space
result = b; b = null; b = new byte[result.length * 2];
} System.arraycopy(result, 0, b, 0, len);
// copy the last read result = b; b = null;
System.arraycopy(buffer, 0, result, len, last); }
len += last; // copy the last read
} System.arraycopy(buffer, 0, result, len, last);
gzipin.close(); len += last;
// finished with reading. now cut the result to the right size }
b = new byte[len]; gzipin.close();
System.arraycopy(result, 0, b, 0, len); // finished with reading. now cut the result to the right size
result = null; b = new byte[len];
return b; System.arraycopy(result, 0, b, 0, len);
result = null;
return b;
} finally {
if (gzipin != null) try{gzipin.close();}catch(Exception e){}
}
} }
/* public static void saveProperties(File f, Properties props, String comment) throws IOException { /* public static void saveProperties(File f, Properties props, String comment) throws IOException {

@ -420,16 +420,21 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
private static TreeSet loadList(File file) { private static TreeSet loadList(File file) {
TreeSet list = new TreeSet(kelondroMSetTools.fastStringComparator); TreeSet list = new TreeSet(kelondroMSetTools.fastStringComparator);
if (!(file.exists())) return list; if (!(file.exists())) return list;
try {
BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(file))); BufferedReader br = null;
String line; try {
while ((line = br.readLine()) != null) { br = new BufferedReader(new InputStreamReader(new FileInputStream(file)));
line = line.trim(); String line;
if ((line.length() > 0) && (!(line.startsWith("#")))) list.add(line.trim().toLowerCase()); while ((line = br.readLine()) != null) {
} line = line.trim();
br.close(); if ((line.length() > 0) && (!(line.startsWith("#")))) list.add(line.trim().toLowerCase());
} catch (IOException e) {} }
return list; br.close();
} catch (IOException e) {
} finally {
if (br != null) try{br.close();}catch(Exception e){}
}
return list;
} }
public void close() { public void close() {

@ -40,6 +40,8 @@
package de.anomic.server; package de.anomic.server;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.File; import java.io.File;
import java.io.FileInputStream; import java.io.FileInputStream;
import java.io.FileOutputStream; import java.io.FileOutputStream;
@ -131,31 +133,38 @@ public abstract class serverAbstractSwitch implements serverSwitch {
} }
public static Map loadHashMap(File f) { public static Map loadHashMap(File f) {
// load props // load props
Properties prop = new Properties(); Properties prop = new Properties();
try { BufferedInputStream bufferedIn = null;
prop.load(new FileInputStream(f)); try {
} catch (IOException e1) { prop.load(bufferedIn = new BufferedInputStream(new FileInputStream(f)));
System.err.println("ERROR: " + f.toString() + " not found in settings path"); } catch (IOException e1) {
prop = null; System.err.println("ERROR: " + f.toString() + " not found in settings path");
} prop = null;
return (Hashtable) prop; } finally {
if (bufferedIn != null)try{bufferedIn.close();}catch(Exception e){}
}
return (Hashtable) prop;
} }
public static void saveMap(File f, Map props, String comment) throws IOException { public static void saveMap(File f, Map props, String comment) throws IOException {
PrintWriter pw = new PrintWriter(new FileOutputStream(f)); PrintWriter pw = null;
pw.println("# " + comment); try {
Iterator i = props.entrySet().iterator(); pw = new PrintWriter(new BufferedOutputStream(new FileOutputStream(f)));
String key, value; pw.println("# " + comment);
Map.Entry entry; Iterator i = props.entrySet().iterator();
while (i.hasNext()) { String key, value;
entry = (Map.Entry) i.next(); Map.Entry entry;
key = (String) entry.getKey(); while (i.hasNext()) {
value = ((String) entry.getValue()).replaceAll("\n", "\\\\n"); entry = (Map.Entry) i.next();
pw.println(key + "=" + value); key = (String) entry.getKey();
} value = ((String) entry.getValue()).replaceAll("\n", "\\\\n");
pw.println("# EOF"); pw.println(key + "=" + value);
pw.close(); }
pw.println("# EOF");
} finally {
if (pw!=null)try{pw.close();}catch(Exception e){}
}
} }
public void setConfig(String key, long value) { public void setConfig(String key, long value) {

@ -1,5 +1,6 @@
package de.anomic.yacy.seedUpload; package de.anomic.yacy.seedUpload;
import java.io.BufferedInputStream;
import java.io.File; import java.io.File;
import java.io.FileInputStream; import java.io.FileInputStream;
import java.io.IOException; import java.io.IOException;
@ -116,12 +117,17 @@ class sshc {
checkAck(in); checkAck(in);
// send a content of lfile // send a content of lfile
FileInputStream fis=new FileInputStream(localFile);
byte[] buf=new byte[1024]; byte[] buf=new byte[1024];
while(true){ BufferedInputStream bufferedIn = null;
int len=fis.read(buf, 0, buf.length); try {
if(len<=0) break; bufferedIn=new BufferedInputStream(new FileInputStream(localFile));
out.write(buf, 0, len); out.flush(); while(true){
int len=bufferedIn.read(buf, 0, buf.length);
if(len<=0) break;
out.write(buf, 0, len); out.flush();
}
} finally {
if (bufferedIn != null) try{bufferedIn.close();}catch(Exception e){}
} }
// send '\0' // send '\0'

@ -183,27 +183,31 @@ public class yacyPeerActions {
} }
private disorderSet loadSuperseed(File local, String url) { private disorderSet loadSuperseed(File local, String url) {
// this returns a list of locations where seed list-files can be found // this returns a list of locations where seed list-files can be found
disorderSet supsee = new disorderSet(); disorderSet supsee = new disorderSet();
String line; String line;
// read in local file // read in local file
int lc = 0; int lc = 0;
BufferedReader br = null;
try { try {
BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(local))); br = new BufferedReader(new InputStreamReader(new FileInputStream(local)));
while ((line = br.readLine()) != null) { while ((line = br.readLine()) != null) {
line = line.trim(); line = line.trim();
//System.out.println("one line in file:" + line); //System.out.println("one line in file:" + line);
if (line.length() > 0) supsee.add(line); if (line.length() > 0) supsee.add(line);
} }
br.close(); br.close();
lc = supsee.size(); lc = supsee.size();
yacyCore.log.logInfo("BOOTSTRAP: " + lc + " seed-list urls from superseed file " + local.toString()); yacyCore.log.logInfo("BOOTSTRAP: " + lc + " seed-list urls from superseed file " + local.toString());
} catch (IOException e) { } catch (IOException e) {
//e.printStackTrace(); //e.printStackTrace();
supsee = new disorderSet(); supsee = new disorderSet();
yacyCore.log.logInfo("BOOTSTRAP: failed to load seed-list urls from superseed file " + local.toString() + ": " + e.getMessage()); yacyCore.log.logInfo("BOOTSTRAP: failed to load seed-list urls from superseed file " + local.toString() + ": " + e.getMessage());
} } finally {
// read in remote file from url if (br!=null)try{br.close();}catch(Exception e){}
}
// read in remote file from url
try { try {
Vector remote = httpc.wget(new URL(url), 5000, null, null, sb.remoteProxyHost, sb.remoteProxyPort); Vector remote = httpc.wget(new URL(url), 5000, null, null, sb.remoteProxyHost, sb.remoteProxyPort);
if ((remote != null) && (remote.size() > 0)) { if ((remote != null) && (remote.size() > 0)) {

Loading…
Cancel
Save