boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

*) adding missing calls for function close() to avoid "too many open file" bug*) adding

Browse Source 
*) bugfix in plasma/plasmaParser.java:
   - parsers with missing dependencies wehre not ignored correctly
*) passing a logger instance to the parsers modules which can be used 
   for logging purposes by the parsers (not done yet)


git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@276 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
theli 20 years ago
parent bd79c43b0b
commit 890e3f4d4a
9 changed files with 109 additions and 46 deletions
Show Stats Download Patch File Download Diff File

23
source/de/anomic/data/listManager.java
Unescape View File

@ -121,8 +121,9 @@ public class listManager {
String line; String line;
Vector list = new Vector(); Vector list = new Vector();
int count = 0; int count = 0;
BufferedReader br = null;
try{ try{
BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(listFile))); br = new BufferedReader(new InputStreamReader(new FileInputStream(listFile)));
while( (line = br.readLine()) != null){ while( (line = br.readLine()) != null){
list.add(line); list.add(line);
@ -131,20 +132,25 @@ public class listManager {
br.close(); br.close();
}catch(IOException e){ }catch(IOException e){
//list is empty //list is empty
} } finally {
if (br!=null)try{br.close();}catch(Exception e) {}
}
return list; return list;
} }
//Writes the Liststring to a file //Writes the Liststring to a file
public static boolean writeList(File listFile, String out){ public static boolean writeList(File listFile, String out){
BufferedWriter bw = null;
try{ try{
BufferedWriter bw = new BufferedWriter(new PrintWriter(new FileWriter(listFile))); bw = new BufferedWriter(new PrintWriter(new FileWriter(listFile)));
bw.write(out); bw.write(out);
bw.close(); bw.close();
return true; return true;
}catch(IOException e){ }catch(IOException e){
return false; return false;
} } finally {
if (bw!=null)try{bw.close();}catch(Exception e){}
}
} }
//overloaded function to write an array //overloaded function to write an array
@ -159,8 +165,9 @@ public class listManager {
public static String getListString(String filename, boolean withcomments){ public static String getListString(String filename, boolean withcomments){
String temp = ""; String temp = "";
String line = ""; String line = "";
BufferedReader br = null;
try{ try{
BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(new File(listsPath ,filename)))); br = new BufferedReader(new InputStreamReader(new FileInputStream(new File(listsPath ,filename))));
//Read the List //Read the List
while((line = br.readLine()) != null){ while((line = br.readLine()) != null){
if( (!line.startsWith("#") || withcomments) || (!line.equals("")) ){ if( (!line.startsWith("#") || withcomments) || (!line.equals("")) ){
@ -168,7 +175,11 @@ public class listManager {
} }
} }
br.close(); br.close();
}catch(IOException e){} } catch(IOException e){
} finally {
if (br!=null)try{br.close();}catch(Exception e){}
}
return temp; return temp;
} }

29
source/de/anomic/data/translator.java
Unescape View File

@ -77,47 +77,60 @@ public class translator {
public static boolean translateFile(File sourceFile, File destFile, File translationFile){ public static boolean translateFile(File sourceFile, File destFile, File translationFile){
Properties translationList = new Properties(); Properties translationList = new Properties();
FileInputStream fileIn = null;
try{ try{
translationList.load(new FileInputStream(translationFile)); translationList.load(fileIn = new FileInputStream(translationFile));
return translateFile(sourceFile, destFile, translationList); return translateFile(sourceFile, destFile, translationList);
}catch(IOException e){ }catch(IOException e){
return false; return false;
} } finally {
if (fileIn!=null)try{fileIn.close();}catch(Exception e){}
}
} }
public static boolean translateFile(File sourceFile, File destFile, Properties translationList){ public static boolean translateFile(File sourceFile, File destFile, Properties translationList){
String content = ""; String content = "";
String line = ""; String line = "";
BufferedReader br = null;
try{ try{
BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(sourceFile))); br = new BufferedReader(new InputStreamReader(new FileInputStream(sourceFile)));
while( (line = br.readLine()) != null){ while( (line = br.readLine()) != null){
content += line + de.anomic.server.serverCore.crlfString; content += line + de.anomic.server.serverCore.crlfString;
} }
br.close(); br.close();
}catch(IOException e){ }catch(IOException e){
return false; return false;
} } finally {
if (br!=null)try{br.close();}catch(Exception e){}
}
content = translate(content, translationList); content = translate(content, translationList);
BufferedWriter bw = null;
try{ try{
BufferedWriter bw = new BufferedWriter(new FileWriter(destFile)); bw = new BufferedWriter(new FileWriter(destFile));
bw.write(content); bw.write(content);
bw.close(); bw.close();
}catch(IOException e){ }catch(IOException e){
return false; return false;
} } finally {
if(bw!=null)try{bw.close();}catch(Exception e){}
}
return true; return true;
} }
public static boolean translateFiles(File sourceDir, File destDir, File translationFile, String extension){ public static boolean translateFiles(File sourceDir, File destDir, File translationFile, String extension){
Properties translationList = new Properties(); Properties translationList = new Properties();
FileInputStream fileIn = null;
try{ try{
translationList.load(new FileInputStream(translationFile)); translationList.load(fileIn = new FileInputStream(translationFile));
return translateFiles(sourceDir, destDir, translationList, extension); return translateFiles(sourceDir, destDir, translationList, extension);
}catch(IOException e){ }catch(IOException e){
return false; return false;
} } finally {
if (fileIn!=null)try{fileIn.close();}catch(Exception e){}
}
} }
public static boolean translateFiles(File sourceDir, File destDir, Properties translationList, String extension){ public static boolean translateFiles(File sourceDir, File destDir, Properties translationList, String extension){

7
source/de/anomic/http/httpTemplate.java
Unescape View File

@ -349,8 +349,9 @@ public final class httpTemplate {
filename= new String(replacePattern( prefix + filename.substring(1, filename.length()-1), pattern, dflt)); filename= new String(replacePattern( prefix + filename.substring(1, filename.length()-1), pattern, dflt));
} }
if ((!filename.equals("")) && (!filename.equals(dflt))) { if ((!filename.equals("")) && (!filename.equals(dflt))) {
BufferedReader br = null;
try{ try{
BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream( new File("htroot", filename) ))); br = new BufferedReader(new InputStreamReader(new FileInputStream( new File("htroot", filename) )));
//Read the Include //Read the Include
while( (line = br.readLine()) != null ){ while( (line = br.readLine()) != null ){
include+=line+de.anomic.server.serverCore.crlfString; include+=line+de.anomic.server.serverCore.crlfString;
@ -358,7 +359,9 @@ public final class httpTemplate {
}catch(IOException e){ }catch(IOException e){
//file not found? //file not found?
serverLog.logError("FILEHANDLER","Include Error with file: "+filename); serverLog.logError("FILEHANDLER","Include Error with file: "+filename);
} } finally {
if (br!=null) try{br.close(); br=null;}catch(Exception e){}
}
PushbackInputStream pis2 = new PushbackInputStream(new ByteArrayInputStream(include.getBytes())); PushbackInputStream pis2 = new PushbackInputStream(new ByteArrayInputStream(include.getBytes()));
writeTemplate(pis2, out, pattern, dflt, prefix); writeTemplate(pis2, out, pattern, dflt, prefix);
} }

6
source/de/anomic/http/httpdFileHandler.java
Unescape View File

@ -93,9 +93,6 @@ import java.util.Map;
import java.util.Properties; import java.util.Properties;
import java.util.zip.GZIPOutputStream; import java.util.zip.GZIPOutputStream;
import sun.security.provider.MD5;
import de.anomic.server.serverByteBuffer;
import de.anomic.server.serverClassLoader; import de.anomic.server.serverClassLoader;
import de.anomic.server.serverCodings; import de.anomic.server.serverCodings;
import de.anomic.server.serverCore; import de.anomic.server.serverCore;
@ -138,9 +135,10 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http
mimeTableInputStream = new FileInputStream(new File(switchboard.getRootPath(), mimeTablePath)); mimeTableInputStream = new FileInputStream(new File(switchboard.getRootPath(), mimeTablePath));
this.mimeTable.load(mimeTableInputStream); this.mimeTable.load(mimeTableInputStream);
} catch (Exception e) { } catch (Exception e) {
if (mimeTableInputStream != null) try { mimeTableInputStream.close(); } catch (Exception e1) {}
serverLog.logError("HTTPDFiles", "ERROR: path to configuration file or configuration invalid\n" + e); serverLog.logError("HTTPDFiles", "ERROR: path to configuration file or configuration invalid\n" + e);
System.exit(1); System.exit(1);
} finally {
if (mimeTableInputStream != null) try { mimeTableInputStream.close(); } catch (Exception e1) {}
} }
} }

14
source/de/anomic/plasma/parser/AbstractParser.java
Unescape View File

@ -53,6 +53,7 @@ import java.io.InputStream;
import java.net.URL; import java.net.URL;
import de.anomic.plasma.plasmaParserDocument; import de.anomic.plasma.plasmaParserDocument;
import de.anomic.server.logging.serverLog;
/** /**
* New classes implementing the {@link de.anomic.plasma.parser.Parser} interface * New classes implementing the {@link de.anomic.plasma.parser.Parser} interface
@ -67,6 +68,12 @@ public abstract class AbstractParser implements Parser{
*/ */
protected String[] libxDependencies = null; protected String[] libxDependencies = null;
/**
* the logger class that should be used by the parser module for logging
* purposes.
*/
protected serverLog theLogger = null;
/** /**
* The Constructor of this class. * The Constructor of this class.
*/ */
@ -136,4 +143,11 @@ public abstract class AbstractParser implements Parser{
return this.libxDependencies; return this.libxDependencies;
} }
/**
* Setting the logger that should be used by this parser class ...
*/
public void setLogger(serverLog log) {
this.theLogger = log;
}
} }

7
source/de/anomic/plasma/parser/Parser.java
Unescape View File

@ -50,6 +50,7 @@ import java.net.URL;
import java.util.Hashtable; import java.util.Hashtable;
import de.anomic.plasma.plasmaParserDocument; import de.anomic.plasma.plasmaParserDocument;
import de.anomic.server.logging.serverLog;
/** /**
* This interface defines a list of methods that needs to be implemented * This interface defines a list of methods that needs to be implemented
@ -115,4 +116,10 @@ public interface Parser {
*/ */
public String[] getLibxDependences(); public String[] getLibxDependences();
/**
* Can be used to set the logger that should be used by the parser module
* @param log the {@link serverLog logger} that should be used
*/
public void setLogger(serverLog log);
} }

11
source/de/anomic/plasma/parser/pdf/pdfParser.java
Unescape View File

@ -49,8 +49,6 @@ import java.io.OutputStreamWriter;
import java.net.URL; import java.net.URL;
import java.util.Hashtable; import java.util.Hashtable;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.pdfbox.pdfparser.PDFParser; import org.pdfbox.pdfparser.PDFParser;
import org.pdfbox.pdmodel.PDDocument; import org.pdfbox.pdmodel.PDDocument;
import org.pdfbox.pdmodel.PDDocumentInformation; import org.pdfbox.pdmodel.PDDocumentInformation;
@ -108,10 +106,9 @@ public class pdfParser extends AbstractParser implements Parser {
PDFTextStripper stripper = new PDFTextStripper(); PDFTextStripper stripper = new PDFTextStripper();
theDocument = parser.getPDDocument(); theDocument = parser.getPDDocument();
// extracting some metadata
PDDocumentInformation theDocInfo = theDocument.getDocumentInformation(); PDDocumentInformation theDocInfo = theDocument.getDocumentInformation();
if (theDocInfo != null) {
if (theDocInfo != null)
{
docTitle = theDocInfo.getTitle(); docTitle = theDocInfo.getTitle();
docSubject = theDocInfo.getSubject(); docSubject = theDocInfo.getSubject();
docAuthor = theDocInfo.getAuthor(); docAuthor = theDocInfo.getAuthor();
@ -157,10 +154,10 @@ public class pdfParser extends AbstractParser implements Parser {
return theDoc; return theDoc;
} }
catch (Exception e) { catch (Exception e) {
throw new ParserException("Unable to parse the pdf content. " + e.getMessage()); throw new ParserException("Unable to parse the pdf content. " + e.getMessage(),e);
} finally { } finally {
if (theDocument != null) try { theDocument.close(); } catch (Exception e) {} if (theDocument != null) try { theDocument.close(); } catch (Exception e) {}
if (writer != null) try { writer.close(); } catch (Exception e) {} if (writer != null) try { writer.close(); } catch (Exception e) {}
Thread.currentThread().setPriority(Thread.NORM_PRIORITY); Thread.currentThread().setPriority(Thread.NORM_PRIORITY);
} }
} }

19
source/de/anomic/plasma/plasmaParser.java
Unescape View File

@ -78,6 +78,7 @@ import org.apache.commons.pool.impl.GenericObjectPool;
import de.anomic.htmlFilter.htmlFilterContentScraper; import de.anomic.htmlFilter.htmlFilterContentScraper;
import de.anomic.htmlFilter.htmlFilterOutputStream; import de.anomic.htmlFilter.htmlFilterOutputStream;
import de.anomic.plasma.parser.Parser; import de.anomic.plasma.parser.Parser;
import de.anomic.plasma.parser.ParserException;
import de.anomic.server.serverFileUtils; import de.anomic.server.serverFileUtils;
import de.anomic.server.logging.serverLog; import de.anomic.server.logging.serverLog;
import de.anomic.yacy.yacySeedUploader; import de.anomic.yacy.yacySeedUploader;
@ -397,8 +398,7 @@ public final class plasmaParser {
if (neededLibx != null) { if (neededLibx != null) {
for (int libxId=0; libxId < neededLibx.length; libxId++) { for (int libxId=0; libxId < neededLibx.length; libxId++) {
if (javaClassPath.indexOf(neededLibx[libxId]) == -1) { if (javaClassPath.indexOf(neededLibx[libxId]) == -1) {
serverLog.logWarning("PARSER","Parser '" + className + "': issing dependency detected: '" + neededLibx[libxId] + "'. Parser will be ignored."); throw new ParserException("Missing dependency detected: '" + neededLibx[libxId] + "'.");
continue;
} }
} }
} }
@ -413,7 +413,7 @@ public final class plasmaParser {
} }
} catch (Exception e) { /* we can ignore this for the moment */ } catch (Exception e) { /* we can ignore this for the moment */
serverLog.logWarning("PARSER", "Parser '" + className + "' doesn't work correctly and will be ignored. " + e.getClass().getName()); serverLog.logWarning("PARSER", "Parser '" + className + "' doesn't work correctly and will be ignored.\n [" + e.getClass().getName() + "]: " + e.getMessage());
} }
} }
} }
@ -645,8 +645,19 @@ final class plasmaParserFactory implements KeyedPoolableObjectFactory {
if (!(key instanceof String)) if (!(key instanceof String))
throw new IllegalArgumentException("The object key must be of type string."); throw new IllegalArgumentException("The object key must be of type string.");
// loading class by name
Class moduleClass = Class.forName((String)key); Class moduleClass = Class.forName((String)key);
return moduleClass.newInstance();
// instantiating class
Parser theParser = (Parser) moduleClass.newInstance();
// setting logger that should by used
String parserShortName = ((String)key).substring("de.anomic.plasma.parser.".length(),((String)key).lastIndexOf("."));
serverLog theLogger = new serverLog("PARSER." + parserShortName.toUpperCase());
theParser.setLogger(theLogger);
return theParser;
} }
/** /**

35
source/de/anomic/server/logging/serverLog.java
Unescape View File

@ -154,18 +154,27 @@ public final class serverLog {
public static final void configureLogging(String homePath) throws SecurityException, FileNotFoundException, IOException { public static final void configureLogging(String homePath) throws SecurityException, FileNotFoundException, IOException {
// loading the logger configuration from file FileInputStream fileIn = null;
LogManager logManager = LogManager.getLogManager(); try {
logManager.readConfiguration(new FileInputStream(new File(homePath, "yacy.logging"))); File loggingConfigFile = new File(homePath, "yacy.logging");
System.out.print("STARTUP: Trying to load logging configuration from file " + loggingConfigFile.toString());
// creating the logging directory fileIn = new FileInputStream(loggingConfigFile);
File log = new File("./log/");
if(!log.canRead()) log.mkdir(); // loading the logger configuration from file
LogManager logManager = LogManager.getLogManager();
// generating the root logger logManager.readConfiguration(fileIn);
Logger logger = Logger.getLogger("");
// creating the logging directory
// System.setOut(new PrintStream(new LoggerOutputStream(Logger.getLogger("STDOUT"),Level.FINEST))); File log = new File("./log/");
// System.setErr(new PrintStream(new LoggerOutputStream(Logger.getLogger("STDERR"),Level.SEVERE))); if(!log.canRead()) log.mkdir();
// generating the root logger
Logger logger = Logger.getLogger("");
// System.setOut(new PrintStream(new LoggerOutputStream(Logger.getLogger("STDOUT"),Level.FINEST)));
// System.setErr(new PrintStream(new LoggerOutputStream(Logger.getLogger("STDERR"),Level.SEVERE)));
} finally {
if (fileIn != null) try {fileIn.close();}catch(Exception e){}
}
} }
} }

Write Preview
Loading…
Cancel
Save