boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

Ensure file input streams proper closing in both success and failures

Browse Source 
Also add when possible a warning level log message on input stream
closing error instead of failing silently. This could help understanding
some IO exceptions such as "too many files open".
pull/122/head
luccioman 8 years ago
parent d98c04853d
commit a04feac064
23 changed files with 336 additions and 126 deletions
Show Stats Download Patch File Download Diff File

8
source/net/yacy/document/parser/images/bmpParser.java
Unescape View File

@ -268,6 +268,14 @@ public class bmpParser {
ConcurrentLog.logException(e); ConcurrentLog.logException(e);
} catch (final IOException e) { } catch (final IOException e) {
ConcurrentLog.logException(e); ConcurrentLog.logException(e);
} finally {
if(fis != null) {
try {
fis.close();
} catch(IOException ioe) {
ConcurrentLog.logException(ioe);
}
}
} }
try { try {

13
source/net/yacy/document/parser/images/genericImageParser.java
Unescape View File

@ -301,9 +301,11 @@ public class genericImageParser extends AbstractParser implements Parser {
final File image = new File(args[0]); final File image = new File(args[0]);
final genericImageParser parser = new genericImageParser(); final genericImageParser parser = new genericImageParser();
AnchorURL uri; AnchorURL uri;
FileInputStream inStream = null;
try { try {
uri = new AnchorURL("http://localhost/" + image.getName()); uri = new AnchorURL("http://localhost/" + image.getName());
final Document[] document = parser.parse(uri, "image/" + MultiProtocolURL.getFileExtension(uri.getFileName()), StandardCharsets.UTF_8.name(), new VocabularyScraper(), 0, new FileInputStream(image)); inStream = new FileInputStream(image);
final Document[] document = parser.parse(uri, "image/" + MultiProtocolURL.getFileExtension(uri.getFileName()), StandardCharsets.UTF_8.name(), new VocabularyScraper(), 0, inStream);
System.out.println(document[0].toString()); System.out.println(document[0].toString());
} catch (final MalformedURLException e) { } catch (final MalformedURLException e) {
e.printStackTrace(); e.printStackTrace();
@ -313,6 +315,15 @@ public class genericImageParser extends AbstractParser implements Parser {
e.printStackTrace(); e.printStackTrace();
} catch (final InterruptedException e) { } catch (final InterruptedException e) {
e.printStackTrace(); e.printStackTrace();
} finally {
if(inStream != null) {
try {
inStream.close();
} catch (IOException e) {
e.printStackTrace();
}
}
ConcurrentLog.shutdown();
} }
} }

52
source/net/yacy/document/parser/images/icoParser.java
Unescape View File

@ -104,29 +104,41 @@ public class icoParser {
} }
public static void main(final String[] args) { public static void main(final String[] args) {
// read a ICO and write it as png try {
System.setProperty("java.awt.headless", "true"); // read a ICO and write it as png
final File in = new File(args[0]); System.setProperty("java.awt.headless", "true");
final File out = new File(args[1]); final File in = new File(args[0]);
final File out = new File(args[1]);
final byte[] file = new byte[(int) in.length()]; final byte[] file = new byte[(int) in.length()];
FileInputStream fis = null; FileInputStream fis = null;
try { try {
fis = new FileInputStream(in); fis = new FileInputStream(in);
fis.read(file); fis.read(file);
} catch (final FileNotFoundException e) { } catch (final FileNotFoundException e) {
ConcurrentLog.logException(e); ConcurrentLog.logException(e);
} catch (final IOException e) { } catch (final IOException e) {
ConcurrentLog.logException(e); ConcurrentLog.logException(e);
} } finally {
if(fis != null) {
try {
fis.close();
} catch (IOException e) {
ConcurrentLog.logException(e);
}
}
}
final icoParser parser = new icoParser(file); final icoParser parser = new icoParser(file);
try { try {
ImageIO.write(parser.getImage(0), "PNG", out); ImageIO.write(parser.getImage(0), "PNG", out);
} catch (final IOException e) { } catch (final IOException e) {
ConcurrentLog.logException(e); ConcurrentLog.logException(e);
} }
} finally {
ConcurrentLog.shutdown();
}
} }
} }

6
source/net/yacy/gui/framework/Switchboard.java
Unescape View File

@ -174,7 +174,11 @@ public class Switchboard {
log.info("error: file dispatcher.properties cannot be readed. Exit"); log.info("error: file dispatcher.properties cannot be readed. Exit");
System.exit(-1); System.exit(-1);
} finally { } finally {
if (fis != null) try { fis.close(); } catch (IOException ex) { } if (fis != null) try {
fis.close();
} catch (IOException ex) {
log.warn("Could not close input stream on file " + propFile);
}
} }
} }

39
source/net/yacy/kelondro/blob/Gap.java
Unescape View File

@ -62,23 +62,36 @@ public class Gap extends TreeMap<Long, Integer> {
super(); super();
// read the index dump and fill the index // read the index dump and fill the index
DataInputStream is; DataInputStream is;
FileInputStream fis = null;
try { try {
is = new DataInputStream(new BufferedInputStream(new FileInputStream(file), (Integer.SIZE + Long.SIZE) * 1024)); // equals 16*1024*recordsize fis = new FileInputStream(file);
is = new DataInputStream(new BufferedInputStream(fis, (Integer.SIZE + Long.SIZE) * 1024)); // equals 16*1024*recordsize
} catch (final OutOfMemoryError e) { } catch (final OutOfMemoryError e) {
is = new DataInputStream(new FileInputStream(file)); if(fis != null) {
/* Reuse if possible the already created FileInputStream */
is = new DataInputStream(fis);
} else {
is = new DataInputStream(new FileInputStream(file));
}
} }
long p; try {
int l; long p;
while (true) { int l;
try { while (true) {
p = is.readLong(); try {
l = is.readInt(); p = is.readLong();
this.put(Long.valueOf(p), Integer.valueOf(l)); l = is.readInt();
} catch (final IOException e) { this.put(Long.valueOf(p), Integer.valueOf(l));
break; } catch (final IOException e) {
} break;
}
}
} finally {
if(is != null) {
is.close();
}
} }
is.close();
is = null; is = null;
} }

11
source/net/yacy/kelondro/blob/HeapReader.java
Unescape View File

@ -738,10 +738,17 @@ public class HeapReader {
public entries(final File blobFile, final int keylen) throws IOException { public entries(final File blobFile, final int keylen) throws IOException {
if (!(blobFile.exists())) throw new IOException("file " + blobFile + " does not exist"); if (!(blobFile.exists())) throw new IOException("file " + blobFile + " does not exist");
FileInputStream fis = null;
try { try {
this.is = new DataInputStream(new BufferedInputStream(new FileInputStream(blobFile), 256 * 1024)); fis = new FileInputStream(blobFile);
this.is = new DataInputStream(new BufferedInputStream(fis, 256 * 1024));
} catch (final OutOfMemoryError e) { } catch (final OutOfMemoryError e) {
this.is = new DataInputStream(new FileInputStream(blobFile)); if(fis != null) {
/* Reuse if possible the already created FileInputStream */
this.is = new DataInputStream(fis);
} else {
this.is = new DataInputStream(new FileInputStream(blobFile));
}
} }
this.keylen = keylen; this.keylen = keylen;
this.blobFile = blobFile; this.blobFile = blobFile;

34
source/net/yacy/kelondro/index/RowHandleMap.java
Unescape View File

@ -88,22 +88,32 @@ public final class RowHandleMap implements HandleMap, Iterable<Map.Entry<byte[],
this(keylength, objectOrder, idxbytes, (int) (file.length() / (keylength + idxbytes)), file.getAbsolutePath()); this(keylength, objectOrder, idxbytes, (int) (file.length() / (keylength + idxbytes)), file.getAbsolutePath());
// read the index dump and fill the index // read the index dump and fill the index
InputStream is; InputStream is;
FileInputStream fis = null;
try { try {
is = new BufferedInputStream(new FileInputStream(file), 1024 * 1024); fis = new FileInputStream(file);
is = new BufferedInputStream(fis, 1024 * 1024);
} catch (final OutOfMemoryError e) { } catch (final OutOfMemoryError e) {
is = new FileInputStream(file); if(fis != null) {
/* Reuse if possible the already created FileInputStream */
is = fis;
} else {
is = new FileInputStream(file);
}
} }
if (file.getName().endsWith(".gz")) is = new GZIPInputStream(is); try {
final byte[] a = new byte[keylength + idxbytes]; if (file.getName().endsWith(".gz")) is = new GZIPInputStream(is);
int c; final byte[] a = new byte[keylength + idxbytes];
Row.Entry entry; int c;
while (true) { Row.Entry entry;
c = is.read(a); while (true) {
if (c <= 0) break; c = is.read(a);
entry = this.rowdef.newEntry(a); // may be null if a is not well-formed if (c <= 0) break;
if (entry != null) this.index.addUnique(entry); entry = this.rowdef.newEntry(a); // may be null if a is not well-formed
if (entry != null) this.index.addUnique(entry);
}
} finally {
is.close();
} }
is.close();
is = null; is = null;
assert this.index.size() == file.length() / (keylength + idxbytes); assert this.index.size() == file.length() / (keylength + idxbytes);
optimize(); optimize();

39
source/net/yacy/kelondro/index/RowHandleSet.java
Unescape View File

@ -107,15 +107,24 @@ public final class RowHandleSet implements HandleSet, Iterable<byte[]>, Cloneabl
public RowHandleSet(final int keylength, final ByteOrder objectOrder, final File file) throws IOException, SpaceExceededException { public RowHandleSet(final int keylength, final ByteOrder objectOrder, final File file) throws IOException, SpaceExceededException {
this(keylength, objectOrder, (int) (file.length() / (keylength + 8))); this(keylength, objectOrder, (int) (file.length() / (keylength + 8)));
// read the index dump and fill the index // read the index dump and fill the index
final InputStream is = new BufferedInputStream(new FileInputStream(file), 1024 * 1024); FileInputStream fis = new FileInputStream(file);
final byte[] a = new byte[keylength]; InputStream is = null;
int c; try {
while (true) { is = new BufferedInputStream(fis, 1024 * 1024);
c = is.read(a); final byte[] a = new byte[keylength];
if (c <= 0) break; int c;
this.index.addUnique(this.rowdef.newEntry(a)); while (true) {
c = is.read(a);
if (c <= 0) break;
this.index.addUnique(this.rowdef.newEntry(a));
}
} finally {
if(is != null) {
is.close();
} else if(fis != null) {
fis.close();
}
} }
is.close();
assert this.index.size() == file.length() / keylength; assert this.index.size() == file.length() / keylength;
} }
@ -388,13 +397,15 @@ public final class RowHandleSet implements HandleSet, Iterable<byte[]>, Cloneabl
// read from file // read from file
ObjectInputStream in = new ObjectInputStream(new FileInputStream(f)); ObjectInputStream in = new ObjectInputStream(new FileInputStream(f));
RowHandleSet s1 = (RowHandleSet) in.readObject(); try {
in.close(); RowHandleSet s1 = (RowHandleSet) in.readObject();
for (byte[] b: s1) {
for (byte[] b: s1) { System.out.println(UTF8.String(b));
System.out.println(UTF8.String(b)); }
s1.close();
} finally {
in.close();
} }
s1.close();
} catch(IOException e) { } catch(IOException e) {
e.printStackTrace(); e.printStackTrace();
} catch (final ClassNotFoundException e) { } catch (final ClassNotFoundException e) {

9
source/net/yacy/kelondro/util/FileUtils.java
Unescape View File

@ -292,6 +292,7 @@ public final class FileUtils {
try { try {
fis.close(); fis.close();
} catch (final Exception e ) { } catch (final Exception e ) {
ConcurrentLog.warn("FileUtils", "Could not close input stream on file " + source);
} }
} }
} }
@ -319,6 +320,7 @@ public final class FileUtils {
try { try {
fis.close(); fis.close();
} catch (final Exception e ) { } catch (final Exception e ) {
ConcurrentLog.warn("FileUtils", "Could not close input stream on file " + source);
} }
} }
} }
@ -411,6 +413,7 @@ public final class FileUtils {
try { try {
fis.close(); fis.close();
} catch (final Exception e ) { } catch (final Exception e ) {
ConcurrentLog.warn("FileUtils", "Could not close input stream on file " + source);
} }
} }
fis = null; fis = null;
@ -478,13 +481,13 @@ public final class FileUtils {
set.add(line.trim().toLowerCase()); set.add(line.trim().toLowerCase());
} }
} }
br.close();
} catch (final IOException e ) { } catch (final IOException e ) {
} finally { } finally {
if ( br != null ) { if ( br != null ) {
try { try {
br.close(); br.close();
} catch (final Exception e ) { } catch (final Exception e ) {
ConcurrentLog.warn("FileUtils", "Could not close input stream on file " + file);
} }
} }
} }
@ -622,7 +625,6 @@ public final class FileUtils {
while ( (line = br.readLine()) != null ) { while ( (line = br.readLine()) != null ) {
if (!line.isEmpty()) list.add(line); if (!line.isEmpty()) list.add(line);
} }
br.close();
} catch (final IOException e ) { } catch (final IOException e ) {
// list is empty // list is empty
} finally { } finally {
@ -630,6 +632,7 @@ public final class FileUtils {
try { try {
br.close(); br.close();
} catch (final Exception e ) { } catch (final Exception e ) {
ConcurrentLog.warn("FileUtils", "Could not close input stream on file " + listFile);
} }
} }
} }
@ -692,13 +695,13 @@ public final class FileUtils {
temp.append(line).append(CR).append(LF); temp.append(line).append(CR).append(LF);
} }
} }
br.close();
} catch (final IOException e ) { } catch (final IOException e ) {
} finally { } finally {
if ( br != null ) { if ( br != null ) {
try { try {
br.close(); br.close();
} catch (final Exception e ) { } catch (final Exception e ) {
ConcurrentLog.warn("FileUtils", "Could not close input stream on file " + listFile);
} }
} }
} }

4
source/net/yacy/kelondro/util/SetTools.java
Unescape View File

@ -529,7 +529,9 @@ public final class SetTools {
} }
} catch (final IOException e) { } catch (final IOException e) {
} finally { } finally {
if (br != null) try{br.close();}catch(final Exception e){} if (br != null) try{br.close();}catch(final Exception e){
ConcurrentLog.warn("SetTools", "Could not close input stream on file " + file);
}
} }
return list; return list;
} }

7
source/net/yacy/kelondro/util/XMLTables.java
Unescape View File

@ -57,8 +57,11 @@ public class XMLTables {
this.timestamp = System.currentTimeMillis(); this.timestamp = System.currentTimeMillis();
if (propFile.exists()) { if (propFile.exists()) {
final XMLDecoder xmldec = new XMLDecoder(new FileInputStream(propFile)); final XMLDecoder xmldec = new XMLDecoder(new FileInputStream(propFile));
tables = (HashMap<String, Map<String, String>>) xmldec.readObject(); try {
xmldec.close(); tables = (HashMap<String, Map<String, String>>) xmldec.readObject();
} finally {
xmldec.close();
}
} else { } else {
tables = new HashMap<String, Map<String, String>>(); tables = new HashMap<String, Map<String, String>>();
} }

26
source/net/yacy/repository/Blacklist.java
Unescape View File

@ -795,14 +795,32 @@ public class Blacklist {
private final void loadDHTCache(final BlacklistType type) { private final void loadDHTCache(final BlacklistType type) {
File cachefile = DHTCacheFile(type); File cachefile = DHTCacheFile(type);
if (cachefile.exists()) { if (cachefile.exists()) {
FileInputStream fileInStream = null;
ObjectInputStream in = null;
try { try {
ObjectInputStream in = new ObjectInputStream(new FileInputStream(cachefile)); fileInStream = new FileInputStream(cachefile);
RowHandleSet rhs = (RowHandleSet) in.readObject(); in = new ObjectInputStream(fileInStream);
this.cachedUrlHashs.put(type, rhs == null ? new RowHandleSet(Word.commonHashLength, Word.commonHashOrder, 0) : rhs); RowHandleSet rhs = (RowHandleSet) in.readObject();
in.close(); this.cachedUrlHashs.put(type, rhs == null ? new RowHandleSet(Word.commonHashLength, Word.commonHashOrder, 0) : rhs);
return; return;
} catch (final Throwable e) { } catch (final Throwable e) {
ConcurrentLog.logException(e); ConcurrentLog.logException(e);
} finally {
if(in != null) {
try {
in.close();
} catch(IOException ioe) {
log.warn("Could not close object input stream on file " + cachefile);
}
} else if(fileInStream != null){
/* An error may have been thrown while constructing the ObjectInputStream :
* by the way the file input stream still has to be closed properly */
try {
fileInStream.close();
} catch(IOException ioe) {
log.warn("Could not close input stream on file " + cachefile);
}
}
} }
} }
this.cachedUrlHashs.put(type, new RowHandleSet(Word.commonHashLength, Word.commonHashOrder, 0)); this.cachedUrlHashs.put(type, new RowHandleSet(Word.commonHashLength, Word.commonHashOrder, 0));

11
source/net/yacy/search/AutoSearch.java
Unescape View File

@ -88,7 +88,15 @@ public class AutoSearch extends AbstractBusyThread {
ConcurrentLog.info(AutoSearch.class.getName(), "read queries from file " + pfile.getAbsolutePath()); ConcurrentLog.info(AutoSearch.class.getName(), "read queries from file " + pfile.getAbsolutePath());
Properties prop = new Properties(); Properties prop = new Properties();
FileInputStream fileIn = new FileInputStream(pfile); FileInputStream fileIn = new FileInputStream(pfile);
prop.load(fileIn); try {
prop.load(fileIn);
} finally {
try {
fileIn.close();
} catch(IOException ioe) {
ConcurrentLog.warn(AutoSearch.class.getName(), "Could not close input stream on file " + pfile);
}
}
if (prop.size() > 0) { if (prop.size() > 0) {
Set<Object> all = prop.keySet(); Set<Object> all = prop.keySet();
for (Object s : all) { for (Object s : all) {
@ -98,7 +106,6 @@ public class AutoSearch extends AbstractBusyThread {
} }
} }
} }
fileIn.close();
} }
} catch (final IOException e) { } catch (final IOException e) {
ConcurrentLog.warn(AutoSearch.class.getName(), "Error reading config file"); ConcurrentLog.warn(AutoSearch.class.getName(), "Error reading config file");

74
source/net/yacy/search/Switchboard.java
Unescape View File

@ -87,11 +87,15 @@ import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream; import java.util.zip.GZIPOutputStream;
import java.util.zip.ZipEntry; import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream; import java.util.zip.ZipInputStream;
import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletRequest;
import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.SolrInputField;
import com.cybozu.labs.langdetect.DetectorFactory;
import com.cybozu.labs.langdetect.LangDetectException;
import com.google.common.io.Files;
import net.yacy.contentcontrol.ContentControlFilterUpdateThread; import net.yacy.contentcontrol.ContentControlFilterUpdateThread;
import net.yacy.contentcontrol.SMWListSyncThread; import net.yacy.contentcontrol.SMWListSyncThread;
@ -136,9 +140,9 @@ import net.yacy.crawler.data.Cache;
import net.yacy.crawler.data.CrawlProfile; import net.yacy.crawler.data.CrawlProfile;
import net.yacy.crawler.data.CrawlQueues; import net.yacy.crawler.data.CrawlQueues;
import net.yacy.crawler.data.NoticedURL; import net.yacy.crawler.data.NoticedURL;
import net.yacy.crawler.data.NoticedURL.StackType;
import net.yacy.crawler.data.ResultImages; import net.yacy.crawler.data.ResultImages;
import net.yacy.crawler.data.ResultURLs; import net.yacy.crawler.data.ResultURLs;
import net.yacy.crawler.data.NoticedURL.StackType;
import net.yacy.crawler.data.ResultURLs.EventOrigin; import net.yacy.crawler.data.ResultURLs.EventOrigin;
import net.yacy.crawler.data.Transactions; import net.yacy.crawler.data.Transactions;
import net.yacy.crawler.retrieval.Request; import net.yacy.crawler.retrieval.Request;
@ -161,11 +165,11 @@ import net.yacy.document.Condenser;
import net.yacy.document.Document; import net.yacy.document.Document;
import net.yacy.document.LibraryProvider; import net.yacy.document.LibraryProvider;
import net.yacy.document.Parser; import net.yacy.document.Parser;
import net.yacy.document.Parser.Failure;
import net.yacy.document.ProbabilisticClassifier; import net.yacy.document.ProbabilisticClassifier;
import net.yacy.document.TextParser; import net.yacy.document.TextParser;
import net.yacy.document.VocabularyScraper;
import net.yacy.document.Parser.Failure;
import net.yacy.document.Tokenizer; import net.yacy.document.Tokenizer;
import net.yacy.document.VocabularyScraper;
import net.yacy.document.content.DCEntry; import net.yacy.document.content.DCEntry;
import net.yacy.document.content.SurrogateReader; import net.yacy.document.content.SurrogateReader;
import net.yacy.document.importer.OAIListFriendsLoader; import net.yacy.document.importer.OAIListFriendsLoader;
@ -190,11 +194,11 @@ import net.yacy.kelondro.workflow.BusyThread;
import net.yacy.kelondro.workflow.InstantBusyThread; import net.yacy.kelondro.workflow.InstantBusyThread;
import net.yacy.kelondro.workflow.WorkflowProcessor; import net.yacy.kelondro.workflow.WorkflowProcessor;
import net.yacy.kelondro.workflow.WorkflowThread; import net.yacy.kelondro.workflow.WorkflowThread;
import net.yacy.peers.DHTSelection;
import net.yacy.peers.Dispatcher; import net.yacy.peers.Dispatcher;
import net.yacy.peers.EventChannel; import net.yacy.peers.EventChannel;
import net.yacy.peers.Network; import net.yacy.peers.Network;
import net.yacy.peers.NewsPool; import net.yacy.peers.NewsPool;
import net.yacy.peers.DHTSelection;
import net.yacy.peers.Protocol; import net.yacy.peers.Protocol;
import net.yacy.peers.Seed; import net.yacy.peers.Seed;
import net.yacy.peers.SeedDB; import net.yacy.peers.SeedDB;
@ -226,10 +230,6 @@ import net.yacy.utils.crypt;
import net.yacy.utils.upnp.UPnP; import net.yacy.utils.upnp.UPnP;
import net.yacy.visualization.CircleTool; import net.yacy.visualization.CircleTool;
import com.cybozu.labs.langdetect.DetectorFactory;
import com.cybozu.labs.langdetect.LangDetectException;
import com.google.common.io.Files;
public final class Switchboard extends serverSwitch { public final class Switchboard extends serverSwitch {
@ -2005,7 +2005,9 @@ public final class Switchboard extends serverSwitch {
ConcurrentLog.logException(e); ConcurrentLog.logException(e);
} finally { } finally {
moved = infile.renameTo(outfile); moved = infile.renameTo(outfile);
if (zis != null) try {zis.close();} catch (final IOException e) {} if (zis != null) try {zis.close();} catch (final IOException e) {
log.warn("Could not close zip input stream on file " + infile);
}
} }
return moved; return moved;
} else if (s.endsWith(".warc") || s.endsWith(".warc.gz")) { } else if (s.endsWith(".warc") || s.endsWith(".warc.gz")) {
@ -2025,9 +2027,12 @@ public final class Switchboard extends serverSwitch {
} else if (s.endsWith(".jsonlist") || s.endsWith(".flatjson")) { } else if (s.endsWith(".jsonlist") || s.endsWith(".flatjson")) {
// parse a file that can be generated with yacy_grid_parser // parse a file that can be generated with yacy_grid_parser
// see https://github.com/yacy/yacy_grid_parser/blob/master/README.md // see https://github.com/yacy/yacy_grid_parser/blob/master/README.md
FileInputStream fis = null;
BufferedReader br = null;
try { try {
InputStream is = new BufferedInputStream(new FileInputStream(infile)); fis = new FileInputStream(infile);
BufferedReader br = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8)); InputStream is = new BufferedInputStream(fis);
br = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8));
String line; String line;
while ((line = br.readLine()) != null) { while ((line = br.readLine()) != null) {
JSONTokener jt = new JSONTokener(line); JSONTokener jt = new JSONTokener(line);
@ -2098,10 +2103,29 @@ public final class Switchboard extends serverSwitch {
} }
Switchboard.this.index.putDocument(surrogate); Switchboard.this.index.putDocument(surrogate);
} }
is.close(); br.close();
br = null;
fis = null;
moved = infile.renameTo(outfile); moved = infile.renameTo(outfile);
} catch (IOException ex) { } catch (IOException ex) {
log.warn("IO Error processing flatjson file " + infile); log.warn("IO Error processing flatjson file " + infile);
} finally {
/* Properly release file system resources even in failure cases */
if(br != null) {
/* buffered reader was successfully created : close it and its underlying streams */
try {
br.close();
} catch (IOException e) {
log.warn("Could not close reader on file " + infile);
}
} else if(fis != null) {
/* no buffered reader : maybe a case of exhausted memory. Anyway file input stream has to be closed. */
try {
fis.close();
} catch (IOException e) {
log.warn("Could not close input stream on file " + infile);
}
}
} }
return moved; return moved;
} }
@ -2123,9 +2147,19 @@ public final class Switchboard extends serverSwitch {
try { try {
final OutputStream os = new BufferedOutputStream(new GZIPOutputStream(new FileOutputStream(gzfile), 65536){{def.setLevel(Deflater.BEST_COMPRESSION);}}); final OutputStream os = new BufferedOutputStream(new GZIPOutputStream(new FileOutputStream(gzfile), 65536){{def.setLevel(Deflater.BEST_COMPRESSION);}});
BufferedInputStream bis = new BufferedInputStream(new FileInputStream(outfile)); BufferedInputStream bis = new BufferedInputStream(new FileInputStream(outfile));
FileUtils.copy(bis, os); try {
os.close(); FileUtils.copy(bis, os);
bis.close(); } finally {
try {
os.close();
} finally {
try {
bis.close();
} catch(IOException ignored) {
log.warn("Could not close input stream on file " + outfile);
}
}
}
if ( gzfile.exists() ) { if ( gzfile.exists() ) {
FileUtils.deletedelete(outfile); FileUtils.deletedelete(outfile);
} }
@ -2138,7 +2172,9 @@ public final class Switchboard extends serverSwitch {
log.info("processed surrogate " + infile); log.info("processed surrogate " + infile);
} }
} }
if (is != null) try {is.close();} catch (IOException e) {} if (is != null) try {is.close();} catch (IOException e) {
log.warn("Could not close input stream on file " + infile);
}
} }
return moved; return moved;
} }
@ -2610,9 +2646,10 @@ public final class Switchboard extends serverSwitch {
if ( !isRobinsonMode() && this.peers.newsPool.size(NewsPool.OUTGOING_DB) == 0 ) { if ( !isRobinsonMode() && this.peers.newsPool.size(NewsPool.OUTGOING_DB) == 0 ) {
// read profile // read profile
final Properties profile = new Properties(); final Properties profile = new Properties();
final File profileFile = new File(this.dataPath, "DATA/SETTINGS/profile.txt");
FileInputStream fileIn = null; FileInputStream fileIn = null;
try { try {
fileIn = new FileInputStream(new File(this.dataPath, "DATA/SETTINGS/profile.txt")); fileIn = new FileInputStream(profileFile);
profile.load(fileIn); profile.load(fileIn);
} catch (final IOException e ) { } catch (final IOException e ) {
} finally { } finally {
@ -2620,6 +2657,7 @@ public final class Switchboard extends serverSwitch {
try { try {
fileIn.close(); fileIn.close();
} catch (final Exception e ) { } catch (final Exception e ) {
log.warn("Could not close input stream on file " + profileFile);
} }
} }
} }

4
source/net/yacy/server/http/TemplateEngine.java
Unescape View File

@ -427,7 +427,9 @@ public final class TemplateEngine {
//file not found? //file not found?
ConcurrentLog.severe("FILEHANDLER","Include Error with file " + UTF8.String(filename) + ": " + e.getMessage()); ConcurrentLog.severe("FILEHANDLER","Include Error with file " + UTF8.String(filename) + ": " + e.getMessage());
} finally { } finally {
if (br != null) try { br.close(); br=null; } catch (final Exception e) {} if (br != null) try { br.close(); br=null; } catch (final Exception e) {
ConcurrentLog.warn("FILEHANDLER","Could not close buffered reader on file " + UTF8.String(filename));
}
} }
final PushbackInputStream pis2 = new PushbackInputStream(new ByteArrayInputStream(include.getBytes())); final PushbackInputStream pis2 = new PushbackInputStream(new ByteArrayInputStream(include.getBytes()));
structure.append(ASCII.getBytes("<fileinclude file=\"")).append(filename).append(close_tagn); structure.append(ASCII.getBytes("<fileinclude file=\"")).append(filename).append(close_tagn);

13
source/net/yacy/utils/PKCS12Tool.java
Unescape View File

@ -95,8 +95,17 @@ public class PKCS12Tool {
} else{ } else{
System.err.println("Creating new java keystore '" + jksFile + "'"); System.err.println("Creating new java keystore '" + jksFile + "'");
} }
jks.load(jksFileIn,(jksPassword!=null)?jksPassword.toCharArray():null); try {
if (jksFileIn != null) jksFileIn.close(); jks.load(jksFileIn,(jksPassword!=null)?jksPassword.toCharArray():null);
} finally {
if (jksFileIn != null) {
try {
jksFileIn.close();
} catch(IOException ioe) {
System.err.println("Error while closing input stream on file " + jksFile);
}
}
}
final Enumeration<String> pkcs12Aliases = aliases(); final Enumeration<String> pkcs12Aliases = aliases();
while (pkcs12Aliases.hasMoreElements()) { while (pkcs12Aliases.hasMoreElements()) {

22
source/net/yacy/utils/cryptbig.java
Unescape View File

@ -231,6 +231,8 @@ public class cryptbig {
<compressed-after-encryption-flag> <compressed-after-encryption-flag>
<binary> <binary>
*/ */
InputStream fin = null;
OutputStream fout = null;
try { try {
final File inFile = new File(inFileName); final File inFile = new File(inFileName);
final String inFileDate = dateFormatter.format(new Date(inFile.lastModified())); // 17 byte final String inFileDate = dateFormatter.format(new Date(inFile.lastModified())); // 17 byte
@ -248,8 +250,8 @@ public class cryptbig {
System.out.println("TEST: preserving X-String : " + X); System.out.println("TEST: preserving X-String : " + X);
// start encryption // start encryption
final InputStream fin = new CipherInputStream(new FileInputStream(inFile), this.ecipher); fin = new CipherInputStream(new FileInputStream(inFile), this.ecipher);
final OutputStream fout = new FileOutputStream(outFileName); fout = new FileOutputStream(outFileName);
// write magic and properties of original file // write magic and properties of original file
// - we encrypt the original date, the encryption date, the file size, the flag // - we encrypt the original date, the encryption date, the file size, the flag
@ -272,13 +274,23 @@ public class cryptbig {
} }
catch (final javax.crypto.IllegalBlockSizeException e) {System.err.println("ERROR:" + e.getMessage());} catch (final javax.crypto.IllegalBlockSizeException e) {System.err.println("ERROR:" + e.getMessage());}
catch (final javax.crypto.BadPaddingException e) {System.err.println("ERROR:" + e.getMessage());} catch (final javax.crypto.BadPaddingException e) {System.err.println("ERROR:" + e.getMessage());}
// finished files
fin.close();
fout.close();
} catch (final FileNotFoundException e) { } catch (final FileNotFoundException e) {
System.err.println("ERROR: file '" + inFileName + "' not found"); System.err.println("ERROR: file '" + inFileName + "' not found");
} catch (final IOException e) { } catch (final IOException e) {
System.err.println("ERROR: IO trouble"); System.err.println("ERROR: IO trouble");
} finally {
try {
if(fin != null) {
fin.close();
}
} catch (IOException ignored) {
}
try {
if(fout != null) {
fout.close();
}
} catch(IOException ignored) {
}
} }
} }

13
source/net/yacy/utils/tarTools.java
Unescape View File

@ -59,8 +59,10 @@ public class tarTools {
*/ */
public static InputStream getInputStream(final String tarPath) throws FileNotFoundException { public static InputStream getInputStream(final String tarPath) throws FileNotFoundException {
if (tarPath.endsWith(".gz")) { if (tarPath.endsWith(".gz")) {
FileInputStream fileInStream = null;
try { try {
return new GZIPInputStream(new FileInputStream(new File(tarPath))); fileInStream = new FileInputStream(new File(tarPath));
return new GZIPInputStream(fileInStream);
} catch (FileNotFoundException e) { } catch (FileNotFoundException e) {
/* /*
* FileNotFoundException is is a subClass of IOException but the * FileNotFoundException is is a subClass of IOException but the
@ -68,6 +70,15 @@ public class tarTools {
*/ */
throw e; throw e;
} catch (final IOException e) { } catch (final IOException e) {
if(fileInStream != null) {
try {
/* release the now useless firstly opened file input stream
* (we can not reuse it as the header has been read by the GZIPInputStream) */
fileInStream.close();
} catch (IOException e1) {
ConcurrentLog.warn("UNTAR", "Could not close input stream on file " + tarPath);
}
}
// this might happen if the stream is not in gzip format. // this might happen if the stream is not in gzip format.
// there may be a 'gz' extension, but it may still be a raw tar file // there may be a 'gz' extension, but it may still be a raw tar file
// this can be caused by 'one too much gzip-content header' that was attached // this can be caused by 'one too much gzip-content header' that was attached

2
source/net/yacy/utils/translation/TranslationManager.java
Unescape View File

@ -192,13 +192,13 @@ public class TranslationManager extends TranslatorXliff {
while ((line = br.readLine()) != null) { while ((line = br.readLine()) != null) {
content.append(line).append(net.yacy.server.serverCore.CRLF_STRING); content.append(line).append(net.yacy.server.serverCore.CRLF_STRING);
} }
br.close();
} catch (final IOException e) { } catch (final IOException e) {
} finally { } finally {
if (br != null) { if (br != null) {
try { try {
br.close(); br.close();
} catch (final Exception e) { } catch (final Exception e) {
ConcurrentLog.fine("TRANSLATOR", "Could not close buffered reader on file " + checkfile);
} }
} }
} }

20
test/java/net/yacy/document/parser/htmlParserTest.java
Unescape View File

@ -77,12 +77,22 @@ public class htmlParserTest extends TestCase {
System.out.println("parse file: " + filename); System.out.println("parse file: " + filename);
htmlParser p = new htmlParser(); htmlParser p = new htmlParser();
final Document[] docs = p.parse(url, mimetype, null, new VocabularyScraper(), 0, new FileInputStream(file)); FileInputStream inStream = null;
try {
inStream = new FileInputStream(file);
final Document[] docs = p.parse(url, mimetype, null, new VocabularyScraper(), 0, inStream);
Document doc = docs[0];
String txt = doc.getCharset();
assertTrue("get Charset", txt != null);
System.out.println("detected charset = " + txt);
} finally {
if(inStream != null) {
System.out.println("Could not close input stream on file " + file);
}
}
Document doc = docs[0];
String txt = doc.getCharset();
assertTrue("get Charset", txt != null);
System.out.println("detected charset = " + txt);
} }
} }

14
test/java/net/yacy/document/parser/images/genericImageParserTest.java
Unescape View File

@ -29,11 +29,15 @@ public class genericImageParserTest {
System.out.println("parse file: " + filename); System.out.println("parse file: " + filename);
genericImageParser p = new genericImageParser(); genericImageParser p = new genericImageParser();
final Document[] docs = p.parse(url, mimetype, charset, new VocabularyScraper(), 0, new FileInputStream(file)); FileInputStream inStream = new FileInputStream(file);
try {
Document doc = docs[0]; final Document[] docs = p.parse(url, mimetype, charset, new VocabularyScraper(), 0, inStream);
assertEquals("YaCy Logo",doc.dc_title()); Document doc = docs[0];
System.out.println(doc.toString()); assertEquals("YaCy Logo",doc.dc_title());
System.out.println(doc.toString());
} finally {
inStream.close();
}
} }
} }

15
test/java/net/yacy/document/parser/images/metadataImageParserTest.java
Unescape View File

@ -30,11 +30,16 @@ public class metadataImageParserTest {
System.out.println("parse file: " + filename); System.out.println("parse file: " + filename);
metadataImageParser p = new metadataImageParser(); metadataImageParser p = new metadataImageParser();
final Document[] docs = p.parse(url, mimetype, charset, new VocabularyScraper(), 0, new FileInputStream(file)); FileInputStream inStream = new FileInputStream(file);
try {
Document doc = docs[0]; final Document[] docs = p.parse(url, mimetype, charset, new VocabularyScraper(), 0, inStream);
assertEquals("YaCy Logo",doc.dc_title());
System.out.println(doc.toString()); Document doc = docs[0];
assertEquals("YaCy Logo",doc.dc_title());
System.out.println(doc.toString());
} finally {
inStream.close();
}
} }
} }

26
test/java/net/yacy/document/parser/pdfParserTest.java
Unescape View File

@ -2,6 +2,7 @@ package net.yacy.document.parser;
import java.io.File; import java.io.File;
import java.io.FileInputStream; import java.io.FileInputStream;
import java.io.IOException;
import java.util.Collection; import java.util.Collection;
import static junit.framework.TestCase.assertEquals; import static junit.framework.TestCase.assertEquals;
import net.yacy.cora.document.id.AnchorURL; import net.yacy.cora.document.id.AnchorURL;
@ -30,16 +31,25 @@ public class pdfParserTest {
System.out.println("parse file: " + filename); System.out.println("parse file: " + filename);
pdfParser p = new pdfParser(); pdfParser p = new pdfParser();
final Document[] docs = p.parse(url, mimetype, charset, new VocabularyScraper(), 0, new FileInputStream(file)); FileInputStream inStream = new FileInputStream(file);
try {
final Document[] docs = p.parse(url, mimetype, charset, new VocabularyScraper(), 0, inStream);
Document doc = docs[0]; Document doc = docs[0];
int ilinks = doc.getAnchors().size(); int ilinks = doc.getAnchors().size();
assertEquals("number of links in pdf", 1, ilinks); assertEquals("number of links in pdf", 1, ilinks);
Collection<AnchorURL> links = doc.getAnchors(); Collection<AnchorURL> links = doc.getAnchors();
System.out.println("number of links detected = " + ilinks); System.out.println("number of links detected = " + ilinks);
for (AnchorURL aurl : links) { for (AnchorURL aurl : links) {
System.out.println(" found: " + aurl.toString()); System.out.println(" found: " + aurl.toString());
}
} finally {
try {
inStream.close();
} catch(IOException ioe) {
System.out.println("Could not close input stream on file " + file);
}
} }
} }

Write Preview
Loading…
Cancel
Save