boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

*) adding missing calls for function close() to avoid "too many open file" bug*) adding

Browse Source 
*) bugfix in plasma/plasmaParser.java:
   - parsers with missing dependencies wehre not ignored correctly
*) passing a logger instance to the parsers modules which can be used 
   for logging purposes by the parsers (not done yet)


git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@276 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
theli 20 years ago
parent bd79c43b0b
commit 890e3f4d4a
9 changed files with 109 additions and 46 deletions
Show Stats Download Patch File Download Diff File

23
source/de/anomic/data/listManager.java
Unescape View File

@ -121,8 +121,9 @@ public class listManager {
String line;
Vector list = new Vector();
int count = 0;
BufferedReader br = null;
try{
BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(listFile)));
br = new BufferedReader(new InputStreamReader(new FileInputStream(listFile)));
while( (line = br.readLine()) != null){
list.add(line);
@ -131,20 +132,25 @@ public class listManager {
br.close();
}catch(IOException e){
//list is empty
}
} finally {
if (br!=null)try{br.close();}catch(Exception e) {}
}
return list;
}
//Writes the Liststring to a file
public static boolean writeList(File listFile, String out){
BufferedWriter bw = null;
try{
BufferedWriter bw = new BufferedWriter(new PrintWriter(new FileWriter(listFile)));
bw = new BufferedWriter(new PrintWriter(new FileWriter(listFile)));
bw.write(out);
bw.close();
return true;
}catch(IOException e){
return false;
}
} finally {
if (bw!=null)try{bw.close();}catch(Exception e){}
}
}
//overloaded function to write an array
@ -159,8 +165,9 @@ public class listManager {
public static String getListString(String filename, boolean withcomments){
String temp = "";
String line = "";
BufferedReader br = null;
try{
BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(new File(listsPath ,filename))));
br = new BufferedReader(new InputStreamReader(new FileInputStream(new File(listsPath ,filename))));
//Read the List
while((line = br.readLine()) != null){
if( (!line.startsWith("#") || withcomments) || (!line.equals("")) ){
@ -168,7 +175,11 @@ public class listManager {
}
}
br.close();
}catch(IOException e){}
} catch(IOException e){
} finally {
if (br!=null)try{br.close();}catch(Exception e){}
}
return temp;
}

29
source/de/anomic/data/translator.java
Unescape View File

@ -77,47 +77,60 @@ public class translator {
public static boolean translateFile(File sourceFile, File destFile, File translationFile){
Properties translationList = new Properties();
FileInputStream fileIn = null;
try{
translationList.load(new FileInputStream(translationFile));
translationList.load(fileIn = new FileInputStream(translationFile));
return translateFile(sourceFile, destFile, translationList);
}catch(IOException e){
return false;
}
} finally {
if (fileIn!=null)try{fileIn.close();}catch(Exception e){}
}
}
public static boolean translateFile(File sourceFile, File destFile, Properties translationList){
String content = "";
String line = "";
BufferedReader br = null;
try{
BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(sourceFile)));
br = new BufferedReader(new InputStreamReader(new FileInputStream(sourceFile)));
while( (line = br.readLine()) != null){
content += line + de.anomic.server.serverCore.crlfString;
}
br.close();
}catch(IOException e){
return false;
}
} finally {
if (br!=null)try{br.close();}catch(Exception e){}
}
content = translate(content, translationList);
BufferedWriter bw = null;
try{
BufferedWriter bw = new BufferedWriter(new FileWriter(destFile));
bw = new BufferedWriter(new FileWriter(destFile));
bw.write(content);
bw.close();
}catch(IOException e){
return false;
}
} finally {
if(bw!=null)try{bw.close();}catch(Exception e){}
}
return true;
}
public static boolean translateFiles(File sourceDir, File destDir, File translationFile, String extension){
Properties translationList = new Properties();
FileInputStream fileIn = null;
try{
translationList.load(new FileInputStream(translationFile));
translationList.load(fileIn = new FileInputStream(translationFile));
return translateFiles(sourceDir, destDir, translationList, extension);
}catch(IOException e){
return false;
}
} finally {
if (fileIn!=null)try{fileIn.close();}catch(Exception e){}
}
}
public static boolean translateFiles(File sourceDir, File destDir, Properties translationList, String extension){

7
source/de/anomic/http/httpTemplate.java
Unescape View File

@ -349,8 +349,9 @@ public final class httpTemplate {
filename= new String(replacePattern( prefix + filename.substring(1, filename.length()-1), pattern, dflt));
}
if ((!filename.equals("")) && (!filename.equals(dflt))) {
BufferedReader br = null;
try{
BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream( new File("htroot", filename) )));
br = new BufferedReader(new InputStreamReader(new FileInputStream( new File("htroot", filename) )));
//Read the Include
while( (line = br.readLine()) != null ){
include+=line+de.anomic.server.serverCore.crlfString;
@ -358,7 +359,9 @@ public final class httpTemplate {
}catch(IOException e){
//file not found?
serverLog.logError("FILEHANDLER","Include Error with file: "+filename);
}
} finally {
if (br!=null) try{br.close(); br=null;}catch(Exception e){}
}
PushbackInputStream pis2 = new PushbackInputStream(new ByteArrayInputStream(include.getBytes()));
writeTemplate(pis2, out, pattern, dflt, prefix);
}

8
source/de/anomic/http/httpdFileHandler.java
Unescape View File

@ -93,9 +93,6 @@ import java.util.Map;
import java.util.Properties;
import java.util.zip.GZIPOutputStream;
import sun.security.provider.MD5;
import de.anomic.server.serverByteBuffer;
import de.anomic.server.serverClassLoader;
import de.anomic.server.serverCodings;
import de.anomic.server.serverCore;
@ -137,10 +134,11 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http
serverLog.logSystem("HTTPDFiles", "Loading mime mapping file " + mimeTablePath);
mimeTableInputStream = new FileInputStream(new File(switchboard.getRootPath(), mimeTablePath));
this.mimeTable.load(mimeTableInputStream);
} catch (Exception e) {
if (mimeTableInputStream != null) try { mimeTableInputStream.close(); } catch (Exception e1) {}
} catch (Exception e) {
serverLog.logError("HTTPDFiles", "ERROR: path to configuration file or configuration invalid\n" + e);
System.exit(1);
} finally {
if (mimeTableInputStream != null) try { mimeTableInputStream.close(); } catch (Exception e1) {}
}
}

14
source/de/anomic/plasma/parser/AbstractParser.java
Unescape View File

@ -53,6 +53,7 @@ import java.io.InputStream;
import java.net.URL;
import de.anomic.plasma.plasmaParserDocument;
import de.anomic.server.logging.serverLog;
/**
* New classes implementing the {@link de.anomic.plasma.parser.Parser} interface
@ -67,6 +68,12 @@ public abstract class AbstractParser implements Parser{
*/
protected String[] libxDependencies = null;
/**
* the logger class that should be used by the parser module for logging
* purposes.
*/
protected serverLog theLogger = null;
/**
* The Constructor of this class.
*/
@ -136,4 +143,11 @@ public abstract class AbstractParser implements Parser{
return this.libxDependencies;
}
/**
* Setting the logger that should be used by this parser class ...
*/
public void setLogger(serverLog log) {
this.theLogger = log;
}
}

7
source/de/anomic/plasma/parser/Parser.java
Unescape View File

@ -50,6 +50,7 @@ import java.net.URL;
import java.util.Hashtable;
import de.anomic.plasma.plasmaParserDocument;
import de.anomic.server.logging.serverLog;
/**
* This interface defines a list of methods that needs to be implemented
@ -115,4 +116,10 @@ public interface Parser {
*/
public String[] getLibxDependences();
/**
* Can be used to set the logger that should be used by the parser module
* @param log the {@link serverLog logger} that should be used
*/
public void setLogger(serverLog log);
}

13
source/de/anomic/plasma/parser/pdf/pdfParser.java
Unescape View File

@ -49,8 +49,6 @@ import java.io.OutputStreamWriter;
import java.net.URL;
import java.util.Hashtable;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.pdfbox.pdfparser.PDFParser;
import org.pdfbox.pdmodel.PDDocument;
import org.pdfbox.pdmodel.PDDocumentInformation;
@ -107,11 +105,10 @@ public class pdfParser extends AbstractParser implements Parser {
PDFTextStripper stripper = new PDFTextStripper();
theDocument = parser.getPDDocument();
PDDocumentInformation theDocInfo = theDocument.getDocumentInformation();
if (theDocInfo != null)
{
// extracting some metadata
PDDocumentInformation theDocInfo = theDocument.getDocumentInformation();
if (theDocInfo != null) {
docTitle = theDocInfo.getTitle();
docSubject = theDocInfo.getSubject();
docAuthor = theDocInfo.getAuthor();
@ -157,10 +154,10 @@ public class pdfParser extends AbstractParser implements Parser {
return theDoc;
}
catch (Exception e) {
throw new ParserException("Unable to parse the pdf content. " + e.getMessage());
throw new ParserException("Unable to parse the pdf content. " + e.getMessage(),e);
} finally {
if (theDocument != null) try { theDocument.close(); } catch (Exception e) {}
if (writer != null) try { writer.close(); } catch (Exception e) {}
if (writer != null) try { writer.close(); } catch (Exception e) {}
Thread.currentThread().setPriority(Thread.NORM_PRIORITY);
}
}

19
source/de/anomic/plasma/plasmaParser.java
Unescape View File

@ -78,6 +78,7 @@ import org.apache.commons.pool.impl.GenericObjectPool;
import de.anomic.htmlFilter.htmlFilterContentScraper;
import de.anomic.htmlFilter.htmlFilterOutputStream;
import de.anomic.plasma.parser.Parser;
import de.anomic.plasma.parser.ParserException;
import de.anomic.server.serverFileUtils;
import de.anomic.server.logging.serverLog;
import de.anomic.yacy.yacySeedUploader;
@ -397,8 +398,7 @@ public final class plasmaParser {
if (neededLibx != null) {
for (int libxId=0; libxId < neededLibx.length; libxId++) {
if (javaClassPath.indexOf(neededLibx[libxId]) == -1) {
serverLog.logWarning("PARSER","Parser '" + className + "': issing dependency detected: '" + neededLibx[libxId] + "'. Parser will be ignored.");
continue;
throw new ParserException("Missing dependency detected: '" + neededLibx[libxId] + "'.");
}
}
}
@ -413,7 +413,7 @@ public final class plasmaParser {
}
} catch (Exception e) { /* we can ignore this for the moment */
serverLog.logWarning("PARSER", "Parser '" + className + "' doesn't work correctly and will be ignored. " + e.getClass().getName());
serverLog.logWarning("PARSER", "Parser '" + className + "' doesn't work correctly and will be ignored.\n [" + e.getClass().getName() + "]: " + e.getMessage());
}
}
}
@ -645,8 +645,19 @@ final class plasmaParserFactory implements KeyedPoolableObjectFactory {
if (!(key instanceof String))
throw new IllegalArgumentException("The object key must be of type string.");
// loading class by name
Class moduleClass = Class.forName((String)key);
return moduleClass.newInstance();
// instantiating class
Parser theParser = (Parser) moduleClass.newInstance();
// setting logger that should by used
String parserShortName = ((String)key).substring("de.anomic.plasma.parser.".length(),((String)key).lastIndexOf("."));
serverLog theLogger = new serverLog("PARSER." + parserShortName.toUpperCase());
theParser.setLogger(theLogger);
return theParser;
}
/**

35
source/de/anomic/server/logging/serverLog.java
Unescape View File

@ -154,18 +154,27 @@ public final class serverLog {
public static final void configureLogging(String homePath) throws SecurityException, FileNotFoundException, IOException {
// loading the logger configuration from file
LogManager logManager = LogManager.getLogManager();
logManager.readConfiguration(new FileInputStream(new File(homePath, "yacy.logging")));
// creating the logging directory
File log = new File("./log/");
if(!log.canRead()) log.mkdir();
// generating the root logger
Logger logger = Logger.getLogger("");
// System.setOut(new PrintStream(new LoggerOutputStream(Logger.getLogger("STDOUT"),Level.FINEST)));
// System.setErr(new PrintStream(new LoggerOutputStream(Logger.getLogger("STDERR"),Level.SEVERE)));
FileInputStream fileIn = null;
try {
File loggingConfigFile = new File(homePath, "yacy.logging");
System.out.print("STARTUP: Trying to load logging configuration from file " + loggingConfigFile.toString());
fileIn = new FileInputStream(loggingConfigFile);
// loading the logger configuration from file
LogManager logManager = LogManager.getLogManager();
logManager.readConfiguration(fileIn);
// creating the logging directory
File log = new File("./log/");
if(!log.canRead()) log.mkdir();
// generating the root logger
Logger logger = Logger.getLogger("");
// System.setOut(new PrintStream(new LoggerOutputStream(Logger.getLogger("STDOUT"),Level.FINEST)));
// System.setErr(new PrintStream(new LoggerOutputStream(Logger.getLogger("STDERR"),Level.SEVERE)));
} finally {
if (fileIn != null) try {fileIn.close();}catch(Exception e){}
}
}
}

Write Preview
Loading…
Cancel
Save