boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

*) adding many missing (File)?(Input|Output)Stream.close() calls to avoid "Too many open files bug".

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@90 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
theli 20 years ago
parent ca93d02cca
commit cbdc499ba6
4 changed files with 205 additions and 163 deletions
Show Stats Download Patch File Download Diff File

29
source/de/anomic/http/httpc.java
Unescape View File

@ -470,12 +470,15 @@ public final class httpc {
// this writes the input stream to either another output stream or
// a file or both.
FileOutputStream bufferOS = null;
if (file != null) bufferOS = new FileOutputStream(file);
writeContentX(procOS, bufferOS, httpc.this.clientInput);
if (bufferOS != null) {
bufferOS.close();
if (file.length() == 0) file.delete();
}
try {
if (file != null) bufferOS = new FileOutputStream(file);
writeContentX(procOS, bufferOS, httpc.this.clientInput);
} finally {
if (bufferOS != null) {
bufferOS.close();
if (file.length() == 0) file.delete();
}
}
}
public void writeContentX(OutputStream procOS, OutputStream bufferOS, InputStream clientInput) throws IOException {
@ -558,15 +561,15 @@ public final class httpc {
serverLog.logInfo("HTTPC", "RESPONSE: status=" + status + ", header=" + responseHeader.toString());
}
}
}
public void close() {
// closes the connection
try {
clientInput.close();
clientOutput.close();
socket.close();
} catch (IOException e) {}
// closes the connection
try {
this.clientInput.close();
this.clientOutput.close();
this.socket.close();
} catch (IOException e) {}
}
// method is either GET, HEAD or POST

139
source/de/anomic/http/httpdFileHandler.java
Unescape View File

@ -114,57 +114,60 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http
private String adminAccountBase64MD5;
public httpdFileHandler(serverSwitch switchboard) {
this.switchboard = switchboard;
if (mimeTable == null) {
// load the mime table
mimeTable = new Properties();
String mimeTablePath = switchboard.getConfig("mimeConfig","");
try {
serverLog.logSystem("HTTPDFiles", "Loading mime mapping file " + mimeTablePath);
mimeTable.load(new FileInputStream(new File(switchboard.getRootPath(), mimeTablePath)));
} catch (Exception e) {
serverLog.logError("HTTPDFiles", "ERROR: path to configuration file or configuration invalid\n" + e);
System.exit(1);
}
}
this.switchboard = switchboard;
if (this.mimeTable == null) {
// load the mime table
this.mimeTable = new Properties();
String mimeTablePath = switchboard.getConfig("mimeConfig","");
FileInputStream mimeTableInputStream = null;
try {
serverLog.logSystem("HTTPDFiles", "Loading mime mapping file " + mimeTablePath);
mimeTableInputStream = new FileInputStream(new File(switchboard.getRootPath(), mimeTablePath));
this.mimeTable.load(mimeTableInputStream);
} catch (Exception e) {
if (mimeTableInputStream != null) try { mimeTableInputStream.close(); } catch (Exception e1) {}
serverLog.logError("HTTPDFiles", "ERROR: path to configuration file or configuration invalid\n" + e);
System.exit(1);
}
}
// create default files array
defaultFiles = switchboard.getConfig("defaultFiles","index.html").split(",");
if (defaultFiles.length == 0) defaultFiles = new String[] {"index.html"};
// create default files array
defaultFiles = switchboard.getConfig("defaultFiles","index.html").split(",");
if (defaultFiles.length == 0) defaultFiles = new String[] {"index.html"};
// create a htRootPath: system pages
if (htRootPath == null) {
htRootPath = new File(switchboard.getRootPath(), switchboard.getConfig("htRootPath","htroot"));
if (!(htRootPath.exists())) htRootPath.mkdir();
}
// create a htRootPath: system pages
if (htRootPath == null) {
htRootPath = new File(switchboard.getRootPath(), switchboard.getConfig("htRootPath","htroot"));
if (!(htRootPath.exists())) htRootPath.mkdir();
}
// create a htDocsPath: user defined pages
if (htDocsPath == null) {
htDocsPath = new File(switchboard.getRootPath(), switchboard.getConfig("htDocsPath", "htdocs"));
if (!(htDocsPath.exists())) htDocsPath.mkdir();
}
// create a htDocsPath: user defined pages
if (htDocsPath == null) {
htDocsPath = new File(switchboard.getRootPath(), switchboard.getConfig("htDocsPath", "htdocs"));
if (!(htDocsPath.exists())) htDocsPath.mkdir();
}
// create a htTemplatePath
if (htTemplatePath == null) {
htTemplatePath = new File(switchboard.getRootPath(), switchboard.getConfig("htTemplatePath","htroot/env/templates"));
if (!(htTemplatePath.exists())) htTemplatePath.mkdir();
}
// create a htTemplatePath
if (htTemplatePath == null) {
htTemplatePath = new File(switchboard.getRootPath(), switchboard.getConfig("htTemplatePath","htroot/env/templates"));
if (!(htTemplatePath.exists())) htTemplatePath.mkdir();
}
if (templates == null) templates = loadTemplates(htTemplatePath);
// create a class loader
if (provider == null) {
provider = new serverClassLoader(/*this.getClass().getClassLoader()*/);
// debug
/*
Package[] ps = ((cachedClassLoader) provider).packages();
for (int i = 0; i < ps.length; i++) System.out.println("PACKAGE IN PROVIDER: " + ps[i].toString());
*/
}
adminAccountBase64MD5 = null;
// create a class loader
if (provider == null) {
provider = new serverClassLoader(/*this.getClass().getClassLoader()*/);
// debug
/*
Package[] ps = ((cachedClassLoader) provider).packages();
for (int i = 0; i < ps.length; i++) System.out.println("PACKAGE IN PROVIDER: " + ps[i].toString());
*/
}
adminAccountBase64MD5 = null;
serverLog.logSystem("HTTPDFileHandler", "File Handler Initialized");
serverLog.logSystem("HTTPDFileHandler", "File Handler Initialized");
}
private void respondHeader(OutputStream out, int retcode,
@ -416,11 +419,18 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http
// read templates
tp.putAll(templates);
// rewrite the file
ByteArrayOutputStream o = new ByteArrayOutputStream();
FileInputStream fis = new FileInputStream(file);
httpTemplate.writeTemplate(fis, o, tp, "-UNRESOLVED_PATTERN-".getBytes());
o.close();
result = o.toByteArray();
ByteArrayOutputStream o = null;
FileInputStream fis = null;
try {
o = new ByteArrayOutputStream();
fis = new FileInputStream(file);
httpTemplate.writeTemplate(fis, o, tp, "-UNRESOLVED_PATTERN-".getBytes());
result = o.toByteArray();
} finally {
if (o != null) try {o.close();} catch(Exception e) {}
if (fis != null) try {fis.close();} catch(Exception e) {}
}
} else { // no html
// write the file to the client
result = serverFileUtils.read(file);
@ -450,22 +460,23 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http
}
private static HashMap loadTemplates(File path) {
// reads all templates from a path
// we use only the folder from the given file path
HashMap result = new HashMap();
if (path == null) return result;
if (!(path.isDirectory())) path = path.getParentFile();
if ((path == null) || (!(path.isDirectory()))) return result;
String[] templates = path.list();
int c;
for (int i = 0; i < templates.length; i++) {
if (templates[i].endsWith(".template")) try {
//System.out.println("TEMPLATE " + templates[i].substring(0, templates[i].length() - 9) + ": " + new String(buf, 0, c));
result.put(templates[i].substring(0, templates[i].length() - 9),
new String(serverFileUtils.read(new File(path, templates[i]))));
} catch (Exception e) {}
}
return result;
// reads all templates from a path
// we use only the folder from the given file path
HashMap result = new HashMap();
if (path == null) return result;
if (!(path.isDirectory())) path = path.getParentFile();
if ((path == null) || (!(path.isDirectory()))) return result;
String[] templates = path.list();
int c;
for (int i = 0; i < templates.length; i++) {
if (templates[i].endsWith(".template"))
try {
//System.out.println("TEMPLATE " + templates[i].substring(0, templates[i].length() - 9) + ": " + new String(buf, 0, c));
result.put(templates[i].substring(0, templates[i].length() - 9),
new String(serverFileUtils.read(new File(path, templates[i]))));
} catch (Exception e) {}
}
return result;
}
private File rewriteClassFile(File template) {

140
source/de/anomic/http/httpdProxyHandler.java
Unescape View File

@ -164,47 +164,54 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
private static HashSet loadSet(String setname, String filename) {
HashSet set = new HashSet();
try {
BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(filename)));
String line;
while ((line = br.readLine()) != null) {
line = line.trim();
if ((line.length() > 0) && (!(line.startsWith("#")))) set.add(line.trim().toLowerCase());
}
br.close();
serverLog.logInfo("PROXY", "read " + setname + " set from file " + filename);
} catch (IOException e) {}
return set;
HashSet set = new HashSet();
BufferedReader br = null;
try {
br = new BufferedReader(new InputStreamReader(new FileInputStream(filename)));
String line;
while ((line = br.readLine()) != null) {
line = line.trim();
if ((line.length() > 0) && (!(line.startsWith("#")))) set.add(line.trim().toLowerCase());
}
br.close();
serverLog.logInfo("PROXY", "read " + setname + " set from file " + filename);
} catch (IOException e) {
} finally {
if (br != null) try { br.close(); } catch (Exception e) {}
}
return set;
}
private static TreeMap loadMap(String mapname, String filename, String sep) {
TreeMap map = new TreeMap();
try {
BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(filename)));
String line;
int pos;
while ((line = br.readLine()) != null) {
line = line.trim();
if ((line.length() > 0) && (!(line.startsWith("#"))) && ((pos = line.indexOf(sep)) > 0))
map.put(line.substring(0, pos).trim().toLowerCase(), line.substring(pos + sep.length()).trim());
}
br.close();
serverLog.logInfo("PROXY", "read " + mapname + " map from file " + filename);
} catch (IOException e) {}
return map;
TreeMap map = new TreeMap();
BufferedReader br = null;
try {
br = new BufferedReader(new InputStreamReader(new FileInputStream(filename)));
String line;
int pos;
while ((line = br.readLine()) != null) {
line = line.trim();
if ((line.length() > 0) && (!(line.startsWith("#"))) && ((pos = line.indexOf(sep)) > 0))
map.put(line.substring(0, pos).trim().toLowerCase(), line.substring(pos + sep.length()).trim());
}
serverLog.logInfo("PROXY", "read " + mapname + " map from file " + filename);
} catch (IOException e) {
} finally {
if (br != null) try { br.close(); } catch (Exception e) {}
}
return map;
}
public static TreeMap loadBlacklist(String mapname, String filenames, String sep) {
TreeMap map = new TreeMap();
if (switchboard == null) return map; // not initialized yet
File listsPath = new File(switchboard.getRootPath(), switchboard.getConfig("listsPath", "DATA/LISTS"));
String filenamesarray[] = filenames.split(",");
String filename = "";
if(filenamesarray.length >0)
for(int i = 0; i < filenamesarray.length; i++)
map.putAll(loadMap(mapname, (new File(listsPath, filenamesarray[i])).toString(), sep));
return map;
TreeMap map = new TreeMap();
if (switchboard == null) return map; // not initialized yet
File listsPath = new File(switchboard.getRootPath(), switchboard.getConfig("listsPath", "DATA/LISTS"));
String filenamesarray[] = filenames.split(",");
if(filenamesarray.length >0)
for(int i = 0; i < filenamesarray.length; i++)
map.putAll(loadMap(mapname, (new File(listsPath, filenamesarray[i])).toString(), sep));
return map;
}
private static String domain(String host) {
@ -445,11 +452,15 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
// send also the complete body now from the cache
// simply read the file and transfer to out socket
InputStream is = new FileInputStream(cacheFile);
byte[] buffer = new byte[2048];
int l;
while ((l = is.read(buffer)) > 0) {hfos.write(buffer, 0, l);}
is.close();
InputStream is = null;
try {
is = new FileInputStream(cacheFile);
byte[] buffer = new byte[2048];
int l;
while ((l = is.read(buffer)) > 0) {hfos.write(buffer, 0, l);}
} finally {
if (is != null) try { is.close(); } catch (Exception e) {}
}
if (hfos instanceof htmlFilterOutputStream) ((htmlFilterOutputStream) hfos).finalize();
}
// that's it!
@ -493,16 +504,16 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
// handle file types
if (((ext == null) || (!(switchboard.extensionBlack.contains(ext)))) &&
(httpd.isTextMime(res.responseHeader.mime(), switchboard.mimeWhite))) {
// this is a file that is a possible candidate for parsing by the indexer
// this is a file that is a possible candidate for parsing by the indexer
if (transformer.isIdentityTransformer()) {
log.logDebug("create passthrough (parse candidate) for url " + url);
// no transformation, only passthrough
// this is especially the case if the bluelist is empty
// in that case, the content is not scraped here but later
log.logDebug("create passthrough (parse candidate) for url " + url);
// no transformation, only passthrough
// this is especially the case if the bluelist is empty
// in that case, the content is not scraped here but later
hfos = respond;
} else {
// make a scraper and transformer
log.logDebug("create scraper for url " + url);
log.logDebug("create scraper for url " + url);
scraper = new htmlFilterContentScraper(url);
hfos = new htmlFilterOutputStream(respond, scraper, transformer, (ext.length() == 0));
if (((htmlFilterOutputStream) hfos).binarySuspect()) {
@ -531,7 +542,7 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
(contentLength < 1048576)) {// 1 MB
// ok, we don't write actually into a file, only to RAM, and schedule writing the file.
byte[] cacheArray = res.writeContent(hfos);
log.logDebug("writeContent of " + url + " produced cacheArray = " + ((cacheArray == null) ? "null" : ("size=" + cacheArray.length)));
log.logDebug("writeContent of " + url + " produced cacheArray = " + ((cacheArray == null) ? "null" : ("size=" + cacheArray.length)));
if (hfos instanceof htmlFilterOutputStream) ((htmlFilterOutputStream) hfos).finalize();
@ -551,11 +562,11 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
}
} else {
// the file is too big to cache it in the ram, or the size is unknown
// write to file right here.
// write to file right here.
cacheFile.getParentFile().mkdirs();
res.writeContent(hfos, cacheFile);
if (hfos instanceof htmlFilterOutputStream) ((htmlFilterOutputStream) hfos).finalize();
log.logDebug("for write-file of " + url + ": contentLength = " + contentLength + ", sizeBeforeDelete = " + sizeBeforeDelete);
log.logDebug("for write-file of " + url + ": contentLength = " + contentLength + ", sizeBeforeDelete = " + sizeBeforeDelete);
if (sizeBeforeDelete == -1) {
// totally fresh file
cacheEntry.status = plasmaHTCache.CACHE_FILL; // it's an insert
@ -569,8 +580,8 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
cacheEntry.status = plasmaHTCache.CACHE_STALE_RELOAD_GOOD;
cacheManager.stackProcess(cacheEntry); // necessary update, write response header to cache
}
// beware! all these writings will not fill the cacheEntry.cacheArray
// that means they are not available for the indexer (except they are scraped before)
// beware! all these writings will not fill the cacheEntry.cacheArray
// that means they are not available for the indexer (except they are scraped before)
}
} else {
// no caching
@ -596,18 +607,17 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
if (cacheFile.exists()) cacheFile.delete();
respondHeader(respond,"404 client unexpectedly closed connection", new httpHeader(null));
} catch (IOException e) {
// can have various reasons
// can have various reasons
if (cacheFile.exists()) cacheFile.delete();
if (e.getMessage().indexOf("Corrupt GZIP trailer") >= 0) {
// just do nothing, we leave it this way
log.logDebug("ignoring bad gzip trail for URL " + url + " (" + e.getMessage() + ")");
} else {
respondHeader(respond,"404 client unexpectedly closed connection", new httpHeader(null));
log.logDebug("IOError for URL " + url + " (" + e.getMessage() + ") - responded 404");
e.printStackTrace();
}
if (e.getMessage().indexOf("Corrupt GZIP trailer") >= 0) {
// just do nothing, we leave it this way
log.logDebug("ignoring bad gzip trail for URL " + url + " (" + e.getMessage() + ")");
} else {
respondHeader(respond,"404 client unexpectedly closed connection", new httpHeader(null));
log.logDebug("IOError for URL " + url + " (" + e.getMessage() + ") - responded 404");
e.printStackTrace();
}
}
remote.close();
} catch (Exception e) {
// this may happen if the targeted host does not exist or anything with the
// remote server was wrong.
@ -632,6 +642,7 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
private void respondError(OutputStream respond, String origerror, int errorcase, String url) {
FileInputStream fis = null;
try {
// set rewrite values
serverObjects tp = new serverObjects();
@ -643,7 +654,7 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
File file = new File(htRootPath, "/proxymsg/error.html");
byte[] result;
ByteArrayOutputStream o = new ByteArrayOutputStream();
FileInputStream fis = new FileInputStream(file);
fis = new FileInputStream(file);
httpTemplate.writeTemplate(fis, o, tp, "-UNRESOLVED_PATTERN-".getBytes());
o.close();
result = o.toByteArray();
@ -659,8 +670,9 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
respondHeader(respond, origerror, header);
serverFileUtils.write(result, respond);
respond.flush();
} catch (IOException e) {
} catch (IOException e) {
} finally {
if (fis != null) try { fis.close(); } catch (Exception e) {}
}
}

60
source/de/anomic/server/serverFileUtils.java
Unescape View File

@ -52,30 +52,43 @@ import java.io.OutputStream;
public final class serverFileUtils {
public static void copy(InputStream source, OutputStream dest) throws IOException {
byte[] buffer = new byte[4096];
int c;
while ((c = source.read(buffer)) > 0) dest.write(buffer, 0, c);
dest.flush();
byte[] buffer = new byte[4096];
int c;
while ((c = source.read(buffer)) > 0) dest.write(buffer, 0, c);
dest.flush();
}
public static void copy(InputStream source, File dest) throws IOException {
FileOutputStream fos = new FileOutputStream(dest);
copy(source, fos);
fos.close();
FileOutputStream fos = null;
try {
fos = new FileOutputStream(dest);
copy(source, fos);
} finally {
if (fos != null) try {fos.close();} catch (Exception e) {}
}
}
public static void copy(File source, OutputStream dest) throws IOException {
InputStream fis = new FileInputStream(source);
copy(fis, dest);
fis.close();
InputStream fis = null;
try {
fis = new FileInputStream(source);
copy(fis, dest);
} finally {
if (fis != null) try { fis.close(); } catch (Exception e) {}
}
}
public static void copy(File source, File dest) throws IOException {
FileInputStream fis = new FileInputStream(source);
FileOutputStream fos = new FileOutputStream(dest);
copy(fis, fos);
fis.close();
fos.close();
FileInputStream fis = null;
FileOutputStream fos = null;
try {
fis = new FileInputStream(source);
fos = new FileOutputStream(dest);
copy(fis, fos);
} finally {
if (fis != null) try {fis.close();} catch (Exception e) {}
if (fos != null) try {fos.close();} catch (Exception e) {}
}
}
public static byte[] read(InputStream source) throws IOException {
@ -86,13 +99,16 @@ public final class serverFileUtils {
}
public static byte[] read(File source) throws IOException {
byte[] buffer = new byte[(int) source.length()];
InputStream fis = new FileInputStream(source);
int p = 0;
int c;
while ((c = fis.read(buffer, p, buffer.length - p)) > 0) p += c;
fis.close();
return buffer;
byte[] buffer = new byte[(int) source.length()];
InputStream fis = null;
try {
fis = new FileInputStream(source);
int p = 0, c;
while ((c = fis.read(buffer, p, buffer.length - p)) > 0) p += c;
} finally {
if (fis != null) try { fis.close(); } catch (Exception e) {}
}
return buffer;
}
public static void write(byte[] source, OutputStream dest) throws IOException {

Write Preview
Loading…
Cancel
Save