boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

Ensure proper closing of file input streams.

Browse Source
pull/122/head
luccioman 8 years ago
parent c53c58fa85
commit d98c04853d
17 changed files with 444 additions and 151 deletions
Show Stats Download Patch File Download Diff File

27
source/net/yacy/cora/document/id/MultiProtocolURL.java
Unescape View File

@ -2352,14 +2352,27 @@ public class MultiProtocolURL implements Serializable, Comparable<MultiProtocolU
return null;
}
/**
* Read fully the source, close it and return its content as a bytes array.
* @param source the source to read
* @return return the content of the source stream
* @throws IOException when an erro occured
*/
public static byte[] read(final InputStream source) throws IOException {
final ByteArrayOutputStream baos = new ByteArrayOutputStream();
final byte[] buffer = new byte[2048];
int c;
while ((c = source.read(buffer, 0, 2048)) > 0) baos.write(buffer, 0, c);
baos.flush();
baos.close();
return baos.toByteArray();
try {
final ByteArrayOutputStream baos = new ByteArrayOutputStream();
final byte[] buffer = new byte[2048];
int c;
while ((c = source.read(buffer, 0, 2048)) > 0) baos.write(buffer, 0, c);
baos.flush();
baos.close();
return baos.toByteArray();
} finally {
try {
source.close();
} catch(IOException ignored) {
}
}
}
public Locale getLocale() {

11
source/net/yacy/cora/geo/OpenGeoDBLocation.java
Unescape View File

@ -74,9 +74,10 @@ public class OpenGeoDBLocation implements Locations
if ( file == null || !file.exists() ) {
return;
}
InputStream is = null;
BufferedReader reader = null;
try {
InputStream is = new FileInputStream(file);
is = new FileInputStream(file);
if ( file.getName().endsWith(".gz") ) {
is = new GZIPInputStream(is);
}
@ -162,14 +163,18 @@ public class OpenGeoDBLocation implements Locations
}
continue;
}
reader.close();
} catch (final IOException e ) {
ConcurrentLog.logException(e);
} finally {
if ( reader != null ) {
try {
reader.close();
} catch (final Exception e ) {
} catch (final Exception ignored ) {
}
} else if(is != null) {
try {
is.close();
} catch (final IOException ignored ) {
}
}
}

10
source/net/yacy/cora/protocol/ftp/FTPClient.java
Unescape View File

@ -247,15 +247,21 @@ public class FTPClient {
final File f = new File(System.getProperty("user.dir"), classname + ".class");
final int length = (int) f.length();
final byte[] classbytes = new byte[length];
DataInputStream in = null;
try {
final DataInputStream in = new DataInputStream(new FileInputStream(f));
in = new DataInputStream(new FileInputStream(f));
in.readFully(classbytes);
in.close();
c = defineClass(classname, classbytes, 0, classbytes.length);
} catch (final FileNotFoundException ee) {
throw new ClassNotFoundException();
} catch (final IOException ee) {
throw new ClassNotFoundException();
} finally {
try {
in.close();
} catch (IOException ioe) {
log.warn("Could not close input stream on file " + f);
}
}
}
}

24
source/net/yacy/cora/storage/Files.java
Unescape View File

@ -139,13 +139,25 @@ public class Files {
if (to.exists()) to.delete();
final byte[] buffer = new byte[4096];
int bytesRead;
final InputStream in = new BufferedInputStream(new FileInputStream(from));
final OutputStream out = new BufferedOutputStream(new FileOutputStream (to));
while ((bytesRead = in.read(buffer)) >= 0) {
out.write(buffer,0,bytesRead);
InputStream in = null;
OutputStream out = null;
try {
in = new BufferedInputStream(new FileInputStream(from));
out = new BufferedOutputStream(new FileOutputStream (to));
while ((bytesRead = in.read(buffer)) >= 0) {
out.write(buffer,0,bytesRead);
}
} finally {
if(in != null) {
try {
in.close();
} catch(IOException ignored) {
}
}
if(out != null) {
out.close();
}
}
in.close();
out.close();
}
}
}

17
source/net/yacy/crawler/data/Snapshots.java
Unescape View File

@ -43,6 +43,7 @@ import org.apache.solr.common.SolrDocument;
import net.yacy.cora.date.GenericFormatter;
import net.yacy.cora.document.encoding.ASCII;
import net.yacy.cora.document.id.DigestURL;
import net.yacy.cora.util.ConcurrentLog;
import net.yacy.search.index.Fulltext;
import net.yacy.search.schema.CollectionSchema;
@ -148,8 +149,9 @@ public class Snapshots {
this.pathtoxml[0] = new File(pathToShard(hostport, urlhash, depth), this.urlhash + "." + datestring + ".xml");
String u = null;
if (this.pathtoxml[0].exists()) {
BufferedReader reader = null;
try {
BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(this.pathtoxml[0])));
reader = new BufferedReader(new InputStreamReader(new FileInputStream(this.pathtoxml[0])));
String line;
while ((line = reader.readLine()) != null) {
if (line.startsWith("<str name=\"sku\">")) {
@ -157,8 +159,17 @@ public class Snapshots {
break;
}
}
reader.close();
} catch (IOException e) {}
} catch (IOException e) {
ConcurrentLog.warn("SNAPSHOTS", "Error while reading file " + this.pathtoxml[0]);
} finally {
if(reader != null) {
try {
reader.close();
} catch (IOException ignored) {
ConcurrentLog.warn("SNAPSHOTS", "Could not close input stream on file " + this.pathtoxml[0]);
}
}
}
}
this.url = u;
}

1
source/net/yacy/data/Translator.java
Unescape View File

@ -180,7 +180,6 @@ public class Translator {
while( (line = br.readLine()) != null){
content.append(line).append(net.yacy.server.serverCore.CRLF_STRING);
}
br.close();
} catch(final IOException e) {
return false;
} finally {

12
source/net/yacy/document/Condenser.java
Unescape View File

@ -282,10 +282,12 @@ public final class Condenser extends Tokenizer {
public static void main(final String[] args) {
// read a property file and convert them into configuration lines
FileInputStream inStream = null;
try {
final File f = new File(args[0]);
final Properties p = new Properties();
p.load(new FileInputStream(f));
inStream = new FileInputStream(f);
p.load(inStream);
final StringBuilder sb = new StringBuilder();
sb.append("{\n");
for (int i = 0; i <= 15; i++) {
@ -303,6 +305,14 @@ public final class Condenser extends Tokenizer {
ConcurrentLog.logException(e);
} catch (final IOException e) {
ConcurrentLog.logException(e);
} finally {
if(inStream != null) {
try {
inStream.close();
} catch (IOException e) {
ConcurrentLog.logException(e);
}
}
}
}

26
source/net/yacy/document/Document.java
Unescape View File

@ -767,7 +767,19 @@ dc_rights
if (!(this.text instanceof ByteArrayOutputStream)) {
this.text = new ByteArrayOutputStream();
}
FileUtils.copy(doc.getTextStream(), (ByteArrayOutputStream) this.text);
InputStream textStream = doc.getTextStream();
try {
FileUtils.copy(textStream, (ByteArrayOutputStream) this.text);
} finally {
try {
if(textStream != null) {
/* textStream can be a FileInputStream : we must close it to ensure releasing system resource */
textStream.close();
}
} catch(IOException e) {
ConcurrentLog.warn("DOCUMENT", "Could not close text input stream");
}
}
this.anchors.addAll(doc.getAnchors());
this.rss.putAll(doc.getRSS());
@ -958,10 +970,20 @@ dc_rights
if (doc.getTextLength() > 0) {
if (docTextLength > 0) content.write('\n');
InputStream textStream = doc.getTextStream();
try {
docTextLength += FileUtils.copy(doc.getTextStream(), content);
docTextLength += FileUtils.copy(textStream, content);
} catch (final IOException e) {
ConcurrentLog.logException(e);
} finally {
try {
if(textStream != null) {
/* textStream can be a FileInputStream : we must close it to ensure releasing system resource */
textStream.close();
}
} catch (IOException e) {
ConcurrentLog.warn("DOCUMENT", "Could not close text input stream");
}
}
}
anchors.addAll(doc.getAnchors());

24
source/net/yacy/document/parser/pdfParser.java
Unescape View File

@ -361,8 +361,10 @@ public class pdfParser extends AbstractParser implements Parser {
// parse
final AbstractParser parser = new pdfParser();
Document document = null;
FileInputStream inStream = null;
try {
document = Document.mergeDocuments(null, "application/pdf", parser.parse(null, "application/pdf", null, new VocabularyScraper(), 0, new FileInputStream(pdfFile)));
inStream = new FileInputStream(pdfFile);
document = Document.mergeDocuments(null, "application/pdf", parser.parse(null, "application/pdf", null, new VocabularyScraper(), 0, inStream));
} catch (final Parser.Failure e) {
System.err.println("Cannot parse file " + pdfFile.getAbsolutePath());
ConcurrentLog.logException(e);
@ -373,6 +375,14 @@ public class pdfParser extends AbstractParser implements Parser {
System.err.println("class not found: " + e.getMessage());
} catch (final FileNotFoundException e) {
ConcurrentLog.logException(e);
} finally {
if(inStream != null) {
try {
inStream.close();
} catch(IOException e) {
System.err.println("Could not close input stream on file " + pdfFile);
}
}
}
// statistics
@ -383,12 +393,22 @@ public class pdfParser extends AbstractParser implements Parser {
System.out.println("\t!!!Parsing without result!!!");
} else {
System.out.println("\tParsed text with " + document.getTextLength() + " chars of text and " + document.getAnchors().size() + " anchors");
InputStream textStream = document.getTextStream();
try {
// write file
FileUtils.copy(document.getTextStream(), new File("parsedPdf.txt"));
FileUtils.copy(textStream, new File("parsedPdf.txt"));
} catch (final IOException e) {
System.err.println("error saving parsed document");
ConcurrentLog.logException(e);
} finally {
try {
if(textStream != null) {
/* textStream can be a FileInputStream : we must close it to ensure releasing system resource */
textStream.close();
}
} catch (IOException e) {
ConcurrentLog.warn("PDFPARSER", "Could not close text input stream");
}
}
}
} else {

11
source/net/yacy/http/Jetty9HttpServerImpl.java
Unescape View File

@ -442,8 +442,15 @@ public class Jetty9HttpServerImpl implements YaCyHttpServer {
// loading keystore data from file
if (ConcurrentLog.isFine("SERVER")) ConcurrentLog.fine("SERVER","Loading keystore file " + keyStoreFileName);
final FileInputStream stream = new FileInputStream(keyStoreFileName);
ks.load(stream, keyStorePwd.toCharArray());
stream.close();
try {
ks.load(stream, keyStorePwd.toCharArray());
} finally {
try {
stream.close();
} catch(IOException ioe) {
ConcurrentLog.warn("SERVER", "Could not close input stream on file " + keyStoreFileName);
}
}
// creating a keystore factory
if (ConcurrentLog.isFine("SERVER")) ConcurrentLog.fine("SERVER","Initializing key manager factory ...");

39
source/net/yacy/utils/CryptoLib.java
Unescape View File

@ -104,25 +104,32 @@ public class CryptoLib {
}
public byte[] getSignature(PrivateKey privKey, InputStream dataStream) throws InvalidKeyException, SignatureException, IOException {
this.sign.initSign(privKey);
byte[] buffer = new byte[1024];
int count = 0;
while((count = dataStream.read(buffer)) != -1) {
this.sign.update(buffer, 0, count);
}
dataStream.close();
return this.sign.sign();
try {
this.sign.initSign(privKey);
byte[] buffer = new byte[1024];
int count = 0;
while((count = dataStream.read(buffer)) != -1) {
this.sign.update(buffer, 0, count);
}
} finally {
dataStream.close();
}
return this.sign.sign();
}
public boolean verifySignature(PublicKey pubKey, InputStream dataStream, byte[] signBuffer) throws InvalidKeyException, SignatureException, IOException {
this.sign.initVerify(pubKey);
byte[] buffer = new byte[1024];
int count = 0;
while((count = dataStream.read(buffer)) != -1) {
this.sign.update(buffer, 0, count);
}
dataStream.close();
try {
this.sign.initVerify(pubKey);
byte[] buffer = new byte[1024];
int count = 0;
while((count = dataStream.read(buffer)) != -1) {
this.sign.update(buffer, 0, count);
}
} finally {
dataStream.close();
}
return this.sign.verify(signBuffer);
}

37
source/net/yacy/utils/PKCS12Tool.java
Unescape View File

@ -53,12 +53,21 @@ public class PKCS12Tool {
// creating PKCS12 keystore
this.kspkcs12 = KeyStore.getInstance("PKCS12");
// load pkcs12 file into keystore object
final FileInputStream fileIn = new FileInputStream(pkcs12FileName);
this.kspkcs12.load(fileIn,(pkcs12Pwd!=null)?pkcs12Pwd.toCharArray():null);
// close stream
fileIn.close();
FileInputStream fileIn = null;
try {
// load pkcs12 file into keystore object
fileIn = new FileInputStream(pkcs12FileName);
this.kspkcs12.load(fileIn,(pkcs12Pwd!=null)?pkcs12Pwd.toCharArray():null);
} finally {
if(fileIn != null) {
try {
// close stream
fileIn.close();
} catch(IOException ioe) {
System.err.println("Could not close file " + pkcs12FileName);
}
}
}
}
public Enumeration<String> aliases() throws KeyStoreException {
@ -108,9 +117,19 @@ public class PKCS12Tool {
// storing jdk into file
System.err.print("Storing java keystore");
final FileOutputStream jksFileOut = new FileOutputStream(jksName);
jks.store(jksFileOut,(jksPassword!=null)?jksPassword.toCharArray():null);
jksFileOut.close();
FileOutputStream jksFileOut = null;
try {
jksFileOut = new FileOutputStream(jksName);
jks.store(jksFileOut,(jksPassword!=null)?jksPassword.toCharArray():null);
} finally {
if(jksFileOut != null) {
try {
jksFileOut.close();
} catch(IOException ioe) {
System.err.println("Could not close file " + jksFileOut);
}
}
}
System.err.print("Import finished.");
}

5
source/net/yacy/utils/cryptbig.java
Unescape View File

@ -327,10 +327,6 @@ public class cryptbig {
// read and decrypt the file
copy(fout, fin, 512);
// close the files
fin.close();
fout.close();
// do postprocessing
} catch (final BadPaddingException e) {
System.err.println("ERROR: decryption of '" + inFileName + "' not possible: " + e.getMessage());
@ -340,6 +336,7 @@ public class cryptbig {
System.err.println("ERROR: file '" + inFileName + "' not found");
} catch (final IOException e) {
System.err.println("ERROR: IO trouble");
} finally {
try { if(fin != null) {
fin.close();
}} catch (final Exception ee) {}

36
source/net/yacy/utils/gzip.java
Unescape View File

@ -48,17 +48,33 @@ public class gzip {
private final static ConcurrentLog logger = new ConcurrentLog("GZIP");
public static void gzipFile(final String inFile, final String outFile) {
try {
final InputStream fin = new BufferedInputStream(new FileInputStream(inFile));
final OutputStream fout = new GZIPOutputStream(new BufferedOutputStream(new FileOutputStream(outFile)), 65536){{def.setLevel(Deflater.BEST_COMPRESSION);}};
copy(fout, fin, 1024);
fin.close();
fout.close();
} catch (final FileNotFoundException e) {
logger.warn("ERROR: file '" + inFile + "' not found", e);
} catch (final IOException e) {
InputStream fin = null;
OutputStream fout = null;
try {
fin = new BufferedInputStream(new FileInputStream(inFile));
fout = new GZIPOutputStream(new BufferedOutputStream(new FileOutputStream(outFile)), 65536){{def.setLevel(Deflater.BEST_COMPRESSION);}};
copy(fout, fin, 1024);
} catch (final FileNotFoundException e) {
logger.warn("ERROR: file '" + inFile + "' not found", e);
} catch (final IOException e) {
logger.warn("ERROR: IO trouble ",e);
}
} finally {
if(fin != null) {
try {
fin.close();
} catch(IOException e) {
logger.warn("ERROR: Could not close file " + inFile);
}
}
if(fout != null) {
try {
fout.close();
} catch(IOException e) {
logger.warn("ERROR: Could not close file " + outFile);
}
}
}
}
public static File gunzipFile(final File in) {

20
source/net/yacy/yacy.java
Unescape View File

@ -345,12 +345,18 @@ public final class yacy {
for (String tmplang : langlist) {
if (!tmplang.equals("") && !tmplang.equals("default") && !tmplang.equals("browser")) { //locale is used
String currentRev = null;
BufferedReader br = null;
try {
final BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(new File(sb.getDataPath("locale.translated_html", "DATA/LOCALE/htroot"), tmplang + "/version"))));
br = new BufferedReader(new InputStreamReader(new FileInputStream(new File(sb.getDataPath("locale.translated_html", "DATA/LOCALE/htroot"), tmplang + "/version"))));
currentRev = br.readLine(); // may return null
br.close();
} catch (final IOException e) {
//Error
} finally {
try {
br.close();
} catch(IOException ioe) {
ConcurrentLog.warn("STARTUP", "Could not close " + tmplang + " version file");
}
}
if (currentRev == null || !currentRev.equals(sb.getConfig(Seed.VERSION, ""))) {
@ -696,10 +702,10 @@ public final class yacy {
if (configFile.exists()) {
Properties p = new Properties();
FileInputStream fis = null;
try {
FileInputStream fis = new FileInputStream(configFile);
fis = new FileInputStream(configFile);
p.load(fis);
fis.close();
// Test for server access restriction (is implemented using Jetty IPaccessHandler which does not support IPv6
// try to disable IPv6
String teststr = p.getProperty("serverClient", "*");
@ -736,6 +742,12 @@ public final class yacy {
System.err.println(configFile.getAbsolutePath());
ConcurrentLog.logException(ex);
ConcurrentLog.severe("Startup", "cannot read " + configFile.toString() + ", please delete the corrupted file if problem persits");
} finally {
try {
fis.close();
} catch (IOException e) {
ConcurrentLog.warn("Startup", "Could not close file " + configFile);
}
}
}
}

243
test/java/net/yacy/document/ParserTest.java
Unescape View File

@ -29,28 +29,51 @@ public class ParserTest {
};
for (final String[] testFile : testFiles) {
FileInputStream inStream = null;
final String filename = "test/parsertest/" + testFile[0];
try {
final String filename = "test/parsertest/" + testFile[0];
final File file = new File(filename);
final String mimetype = testFile[1];
final AnchorURL url = new AnchorURL("http://localhost/"+filename);
AbstractParser p = new ooxmlParser();
final Document[] docs = p.parse(url, mimetype, null, new VocabularyScraper(), 0, new FileInputStream(file));
inStream = new FileInputStream(file);
final Document[] docs = p.parse(url, mimetype, null, new VocabularyScraper(), 0, inStream);
for (final Document doc: docs) {
final Reader content = new InputStreamReader(doc.getTextStream(), doc.getCharset());
final StringBuilder str = new StringBuilder();
int c;
while( (c = content.read()) != -1 )
str.append((char)c);
Reader content = null;
try {
content = new InputStreamReader(doc.getTextStream(), doc.getCharset());
final StringBuilder str = new StringBuilder();
int c;
while( (c = content.read()) != -1 )
str.append((char)c);
System.out.println("Parsed " + filename + ": " + str);
assertThat(str.toString(), containsString("In München steht ein Hofbräuhaus, dort gibt es Bier in Maßkrügen"));
assertThat(doc.dc_title(), containsString(testFile[2]));
assertThat(doc.dc_creator(), containsString(testFile[3]));
if (testFile[4].length() > 0) assertThat(doc.dc_description()[0], containsString(testFile[4]));
System.out.println("Parsed " + filename + ": " + str);
assertThat(str.toString(), containsString("In München steht ein Hofbräuhaus, dort gibt es Bier in Maßkrügen"));
assertThat(doc.dc_title(), containsString(testFile[2]));
assertThat(doc.dc_creator(), containsString(testFile[3]));
if (testFile[4].length() > 0) assertThat(doc.dc_description()[0], containsString(testFile[4]));
} finally {
if(content != null) {
try {
content.close();
} catch(IOException ioe) {
System.out.println("Could not close text input stream");
}
}
}
}
} catch (final InterruptedException ex) {}
} catch (final InterruptedException ex) {
} finally {
if(inStream != null) {
try {
inStream.close();
} catch(IOException ioe) {
System.out.println("Could not close input stream on file " + filename);
}
}
}
}
}
@ -63,28 +86,50 @@ public class ParserTest {
};
for (final String[] testFile : testFiles) {
FileInputStream inStream = null;
final String filename = "test/parsertest/" + testFile[0];
try {
final String filename = "test/parsertest/" + testFile[0];
final File file = new File(filename);
final String mimetype = testFile[1];
final AnchorURL url = new AnchorURL("http://localhost/"+filename);
AbstractParser p = new odtParser();
final Document[] docs = p.parse(url, mimetype, null, new VocabularyScraper(), 0, new FileInputStream(file));
inStream = new FileInputStream(file);
final Document[] docs = p.parse(url, mimetype, null, new VocabularyScraper(), 0, inStream);
for (final Document doc: docs) {
final Reader content = new InputStreamReader(doc.getTextStream(), doc.getCharset());
final StringBuilder str = new StringBuilder();
int c;
while( (c = content.read()) != -1 )
Reader content = null;
try {
content = new InputStreamReader(doc.getTextStream(), doc.getCharset());
final StringBuilder str = new StringBuilder();
int c;
while( (c = content.read()) != -1 )
str.append((char)c);
System.out.println("Parsed " + filename + ": " + str);
assertThat(str.toString(), containsString("In München steht ein Hofbräuhaus, dort gibt es Bier in Maßkrügen"));
assertThat(doc.dc_title(), containsString(testFile[2]));
assertThat(doc.dc_creator(), containsString(testFile[3]));
if (testFile[4].length() > 0) assertThat(doc.dc_description()[0], containsString(testFile[4]));
System.out.println("Parsed " + filename + ": " + str);
assertThat(str.toString(), containsString("In München steht ein Hofbräuhaus, dort gibt es Bier in Maßkrügen"));
assertThat(doc.dc_title(), containsString(testFile[2]));
assertThat(doc.dc_creator(), containsString(testFile[3]));
if (testFile[4].length() > 0) assertThat(doc.dc_description()[0], containsString(testFile[4]));
} finally {
if(content != null) {
try {
content.close();
} catch(IOException ioe) {
System.out.println("Could not close text input stream");
}
}
}
}
} catch (final InterruptedException ex) {}
} catch (final InterruptedException ex) {
} finally {
if(inStream != null) {
try {
inStream.close();
} catch(IOException ioe) {
System.out.println("Could not close input stream on file " + filename);
}
}
}
}
}
@ -95,28 +140,50 @@ public class ParserTest {
};
for (final String[] testFile : testFiles) {
final String filename = "test/parsertest/" + testFile[0];
FileInputStream inStream = null;
try {
final String filename = "test/parsertest/" + testFile[0];
final File file = new File(filename);
final String mimetype = testFile[1];
final AnchorURL url = new AnchorURL("http://localhost/"+filename);
AbstractParser p = new pdfParser();
final Document[] docs = p.parse(url, mimetype, null, new VocabularyScraper(), 0, new FileInputStream(file));
inStream = new FileInputStream(file);
final Document[] docs = p.parse(url, mimetype, null, new VocabularyScraper(), 0, inStream);
for (final Document doc: docs) {
final Reader content = new InputStreamReader(doc.getTextStream(), doc.getCharset());
final StringBuilder str = new StringBuilder();
int c;
while( (c = content.read()) != -1 )
Reader content = null;
try {
content = new InputStreamReader(doc.getTextStream(), doc.getCharset());
final StringBuilder str = new StringBuilder();
int c;
while( (c = content.read()) != -1 )
str.append((char)c);
System.out.println("Parsed " + filename + ": " + str);
assertThat(str.toString(), containsString("In München steht ein Hofbräuhaus, dort gibt es Bier in Maßkrügen"));
assertThat(doc.dc_title(), containsString(testFile[2]));
assertThat(doc.dc_creator(), containsString(testFile[3]));
if (testFile[4].length() > 0) assertThat(doc.dc_description()[0], containsString(testFile[4]));
System.out.println("Parsed " + filename + ": " + str);
assertThat(str.toString(), containsString("In München steht ein Hofbräuhaus, dort gibt es Bier in Maßkrügen"));
assertThat(doc.dc_title(), containsString(testFile[2]));
assertThat(doc.dc_creator(), containsString(testFile[3]));
if (testFile[4].length() > 0) assertThat(doc.dc_description()[0], containsString(testFile[4]));
} finally {
if(content != null) {
try {
content.close();
} catch(IOException ioe) {
System.out.println("Could not close text input stream");
}
}
}
}
} catch (final InterruptedException ex) {}
} catch (final InterruptedException ex) {
} finally {
if(inStream != null) {
try {
inStream.close();
} catch(IOException ioe) {
System.out.println("Could not close input stream on file " + filename);
}
}
}
}
}
@ -127,30 +194,52 @@ public class ParserTest {
};
for (final String[] testFile : testFiles) {
final String filename = "test/parsertest/" + testFile[0];
FileInputStream inStream = null;
try {
final String filename = "test/parsertest/" + testFile[0];
final File file = new File(filename);
final String mimetype = testFile[1];
final AnchorURL url = new AnchorURL("http://localhost/"+filename);
AbstractParser p = new docParser();
final Document[] docs = p.parse(url, mimetype, null, new VocabularyScraper(), 0, new FileInputStream(file));
inStream = new FileInputStream(file);
final Document[] docs = p.parse(url, mimetype, null, new VocabularyScraper(), 0, inStream);
for (final Document doc: docs) {
final Reader content = new InputStreamReader(doc.getTextStream(), doc.getCharset());
final StringBuilder str = new StringBuilder();
int c;
while( (c = content.read()) != -1 )
Reader content = null;
try {
content = new InputStreamReader(doc.getTextStream(), doc.getCharset());
final StringBuilder str = new StringBuilder();
int c;
while( (c = content.read()) != -1 )
str.append((char)c);
System.out.println("Parsed " + filename + ": " + str);
assertThat(str.toString(), containsString("In München steht ein Hofbräuhaus, dort gibt es Bier in Maßkrügen"));
assertThat(doc.dc_title(), containsString(testFile[2]));
assertThat(doc.dc_creator(), containsString(testFile[3]));
if (testFile[4].length() > 0) assertThat(doc.dc_description()[0], containsString(testFile[4]));
System.out.println("Parsed " + filename + ": " + str);
assertThat(str.toString(), containsString("In München steht ein Hofbräuhaus, dort gibt es Bier in Maßkrügen"));
assertThat(doc.dc_title(), containsString(testFile[2]));
assertThat(doc.dc_creator(), containsString(testFile[3]));
if (testFile[4].length() > 0) assertThat(doc.dc_description()[0], containsString(testFile[4]));
} finally {
if(content != null) {
try {
content.close();
} catch(IOException ioe) {
System.out.println("Could not close text input stream");
}
}
}
}
} catch (final InterruptedException ex) {}
} catch (final InterruptedException ex) {
} finally {
if(inStream != null) {
try {
inStream.close();
} catch(IOException ioe) {
System.out.println("Could not close input stream on file " + filename);
}
}
}
}
}
/**
* Powerpoint parser test *
@ -172,22 +261,46 @@ public class ParserTest {
final AnchorURL url = new AnchorURL("http://localhost/" + filename);
AbstractParser p = new pptParser();
final Document[] docs = p.parse(url, mimetype, null, new VocabularyScraper(), 0, new FileInputStream(file));
for (final Document doc : docs) {
final Reader content = new InputStreamReader(doc.getTextStream(), doc.getCharset());
final StringBuilder str = new StringBuilder();
int c;
while ((c = content.read()) != -1) {
str.append((char) c);
}
System.out.println("Parsed " + filename + ": " + str);
assertThat(str.toString(), containsString("In München steht ein Hofbräuhaus, dort gibt es Bier in Maßkrügen"));
assertThat(doc.dc_title(), containsString(testFile[2]));
assertThat(doc.dc_creator(), containsString(testFile[3]));
if (testFile[4].length() > 0) {
assertThat(doc.dc_description()[0], containsString(testFile[4]));
}
FileInputStream inStream = null;
try {
inStream = new FileInputStream(file);
final Document[] docs = p.parse(url, mimetype, null, new VocabularyScraper(), 0, inStream);
for (final Document doc : docs) {
Reader content = null;
try {
content = new InputStreamReader(doc.getTextStream(), doc.getCharset());
final StringBuilder str = new StringBuilder();
int c;
while ((c = content.read()) != -1) {
str.append((char) c);
}
System.out.println("Parsed " + filename + ": " + str);
assertThat(str.toString(), containsString("In München steht ein Hofbräuhaus, dort gibt es Bier in Maßkrügen"));
assertThat(doc.dc_title(), containsString(testFile[2]));
assertThat(doc.dc_creator(), containsString(testFile[3]));
if (testFile[4].length() > 0) {
assertThat(doc.dc_description()[0], containsString(testFile[4]));
}
} finally {
if(content != null) {
try {
content.close();
} catch(IOException ioe) {
System.out.println("Could not close text input stream");
}
}
}
}
} finally {
if(inStream != null) {
try {
inStream.close();
} catch(IOException ioe) {
System.out.println("Could not close input stream on file " + filename);
}
}
}
}
}

52
test/java/net/yacy/document/parser/xlsParserTest.java
Unescape View File

@ -2,6 +2,7 @@ package net.yacy.document.parser;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.Reader;
import net.yacy.cora.document.id.AnchorURL;
@ -35,21 +36,44 @@ public class xlsParserTest {
final AnchorURL url = new AnchorURL("http://localhost/" + filename);
AbstractParser p = new xlsParser();
final Document[] docs = p.parse(url, mimetype, null, new VocabularyScraper(), 0, new FileInputStream(file));
for (final Document doc : docs) {
final Reader content = new InputStreamReader(doc.getTextStream(), doc.getCharset());
final StringBuilder str = new StringBuilder();
int c;
while ((c = content.read()) != -1) {
str.append((char) c);
}
System.out.println("Parsed " + filename + ": " + str);
assertThat(str.toString(), containsString("In München steht ein Hofbräuhaus, dort gibt es Bier in Maßkrügen"));
assertThat(doc.dc_creator(), containsString(testFile[3]));
if (testFile[4].length() > 0) {
assertThat(doc.dc_description()[0], containsString(testFile[4]));
FileInputStream inStream = null;
try {
inStream = new FileInputStream(file);
final Document[] docs = p.parse(url, mimetype, null, new VocabularyScraper(), 0, inStream);
for (final Document doc : docs) {
Reader content = null;
try {
content = new InputStreamReader(doc.getTextStream(), doc.getCharset());
final StringBuilder str = new StringBuilder();
int c;
while ((c = content.read()) != -1) {
str.append((char) c);
}
System.out.println("Parsed " + filename + ": " + str);
assertThat(str.toString(), containsString("In München steht ein Hofbräuhaus, dort gibt es Bier in Maßkrügen"));
assertThat(doc.dc_creator(), containsString(testFile[3]));
if (testFile[4].length() > 0) {
assertThat(doc.dc_description()[0], containsString(testFile[4]));
}
} finally {
if(content != null) {
try {
content.close();
} catch(IOException ioe) {
System.out.println("Could not close text input stream");
}
}
}
}
} finally {
if(inStream != null) {
try {
inStream.close();
} catch(IOException ioe) {
System.out.println("Could not close input stream on file " + filename);
}
}
}
}
}

Write Preview
Loading…
Cancel
Save