boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

*) added changes which were lost

Browse Source 
*) additional annotations
*) additional svn properties
*) _no_ functional changes

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6186 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
low012 16 years ago
parent 499723891d
commit 1ee109761f
31 changed files with 254 additions and 67 deletions
Show Stats Download Patch File Download Diff File

12
source/de/anomic/document/parser/bzipParser.java
Unescape View File

@ -6,7 +6,10 @@
//Frankfurt, Germany, 2005
//
//this file is contributed by Martin Thelian
//last major change: 16.05.2005
//
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
//This program is free software; you can redistribute it and/or modify
//it under the terms of the GNU General Public License as published by
@ -46,11 +49,15 @@ public class bzipParser extends AbstractParser implements Parser {
* @see #getSupportedMimeTypes()
*/
public static final Hashtable<String, String> SUPPORTED_MIME_TYPES = new Hashtable<String, String>();
static String fileExtensions = "bz2,tbz,tbz2";
static final String fileExtensions = "bz2,tbz,tbz2";
static {
SUPPORTED_MIME_TYPES.put("application/x-bzip2",fileExtensions);
SUPPORTED_MIME_TYPES.put("application/bzip2", fileExtensions);
SUPPORTED_MIME_TYPES.put("application/x-bz2", fileExtensions);
SUPPORTED_MIME_TYPES.put("application/octet-stream",fileExtensions);
SUPPORTED_MIME_TYPES.put("application/x-bzip",fileExtensions);
SUPPORTED_MIME_TYPES.put("application/x-compressed",fileExtensions);
SUPPORTED_MIME_TYPES.put("application/x-stuffit",fileExtensions);
}
/**
@ -117,6 +124,7 @@ public class bzipParser extends AbstractParser implements Parser {
}
}
@Override
public void reset() {
// Nothing todo here at the moment
super.reset();

18
source/de/anomic/document/parser/docParser.java
Unescape View File

@ -6,7 +6,10 @@
//Frankfurt, Germany, 2005
//
//this file is contributed by Martin Thelian
//last major change: 24.04.2005
//
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
//This program is free software; you can redistribute it and/or modify
//it under the terms of the GNU General Public License as published by
@ -43,7 +46,17 @@ public class docParser extends AbstractParser implements Parser {
* @see #getSupportedMimeTypes()
*/
public static final Hashtable<String, String> SUPPORTED_MIME_TYPES = new Hashtable<String, String>();
static { SUPPORTED_MIME_TYPES.put("application/msword","doc"); }
static {
SUPPORTED_MIME_TYPES.put("application/msword","doc");
SUPPORTED_MIME_TYPES.put("application/doc","doc");
SUPPORTED_MIME_TYPES.put("appl/text","doc");
SUPPORTED_MIME_TYPES.put("application/vnd.msword","doc");
SUPPORTED_MIME_TYPES.put("application/vnd.ms-word","doc");
SUPPORTED_MIME_TYPES.put("application/winword","doc");
SUPPORTED_MIME_TYPES.put("application/word","doc");
SUPPORTED_MIME_TYPES.put("application/x-msw6","doc");
SUPPORTED_MIME_TYPES.put("application/x-msword","doc");
}
/**
* a list of library names that are needed by this parser
@ -102,6 +115,7 @@ public class docParser extends AbstractParser implements Parser {
return docParser.SUPPORTED_MIME_TYPES;
}
@Override
public void reset() {
// Nothing todo here at the moment
super.reset();

21
source/de/anomic/document/parser/gzipParser.java
Unescape View File

@ -6,7 +6,10 @@
//Frankfurt, Germany, 2005
//
//this file is contributed by Martin Thelian
//last major change: 24.04.2005
//
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
//This program is free software; you can redistribute it and/or modify
//it under the terms of the GNU General Public License as published by
@ -44,10 +47,19 @@ public class gzipParser extends AbstractParser implements Parser {
* a list of mime types that are supported by this parser class
* @see #getSupportedMimeTypes()
*/
public static final Hashtable<String, String> SUPPORTED_MIME_TYPES = new Hashtable<String, String>();
public static final Hashtable<String, String> SUPPORTED_MIME_TYPES = new Hashtable<String, String>();
static final String fileExtensions = "gz,tgz";
static {
SUPPORTED_MIME_TYPES.put("application/x-gzip","gz,tgz");
SUPPORTED_MIME_TYPES.put("application/gzip","gz,tgz");
SUPPORTED_MIME_TYPES.put("application/x-gzip",fileExtensions);
SUPPORTED_MIME_TYPES.put("application/gzip",fileExtensions);
SUPPORTED_MIME_TYPES.put("application/x-gunzip",fileExtensions);
SUPPORTED_MIME_TYPES.put("application/gzipped",fileExtensions);
SUPPORTED_MIME_TYPES.put("application/gzip-compressed",fileExtensions);
SUPPORTED_MIME_TYPES.put("application/x-compressed",fileExtensions);
SUPPORTED_MIME_TYPES.put("application/x-compress",fileExtensions);
SUPPORTED_MIME_TYPES.put("gzip/document",fileExtensions);
SUPPORTED_MIME_TYPES.put("application/octet-stream",fileExtensions);
SUPPORTED_MIME_TYPES.put("application/x-tar",fileExtensions);
}
/**
@ -102,6 +114,7 @@ public class gzipParser extends AbstractParser implements Parser {
}
}
@Override
public void reset() {
// Nothing todo here at the moment
super.reset();

8
source/de/anomic/document/parser/html/AbstractScraper.java
Unescape View File

@ -1,9 +1,12 @@
// htmlFilterAbstractScraper.java
// AbstractScraper.java
// ---------------------------
// (C) by Michael Peter Christen; mc@yacy.net
// first published on http://www.anomic.de
// Frankfurt, Germany, 2004
// last major change: 18.02.2004
//
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
// You agree that the Author(s) is (are) not responsible for cost,
// loss of data or any harm that may be caused by usage of this softare or
@ -92,6 +95,7 @@ public abstract class AbstractScraper implements Scraper {
tags1 = null;
}
@Override
protected void finalize() {
close();
}

8
source/de/anomic/document/parser/html/AbstractTransformer.java
Unescape View File

@ -1,9 +1,12 @@
// htmlFilterAbstractTransformer.java
// AbstractTransformer.java
// ----------------------------------
// (C) by Michael Peter Christen; mc@yacy.net
// first published on http://www.anomic.de
// Frankfurt, Germany, 2004
// last major change: 18.02.2004
//
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
@ -65,6 +68,7 @@ public abstract class AbstractTransformer implements Transformer {
tags1 = null;
}
@Override
protected void finalize() {
close();
}

6
source/de/anomic/document/parser/html/CharacterCoding.java
Unescape View File

@ -1,9 +1,13 @@
// htmlFilterCharacterCoding.java
// CharacterCoding.java
// ----------------------------------
// (C) 22.10.2008 by Michael Peter Christen; mc@yacy.net
// first published on http://yacy.net
// Frankfurt, Germany, 2008
//
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or

3
source/de/anomic/document/parser/html/ContentScraper.java
Unescape View File

@ -1,4 +1,4 @@
// htmlFilterContentScraper.java
// ContentScraper.java
// -----------------------------
// (C) by Michael Peter Christen; mc@yacy.net
// first published on http://www.anomic.de
@ -441,6 +441,7 @@ public class ContentScraper extends AbstractScraper implements Scraper {
* (non-Javadoc)
* @see de.anomic.htmlFilter.htmlFilterScraper#close()
*/
@Override
public void close() {
// free resources
super.close();

5
source/de/anomic/document/parser/html/ContentTransformer.java
Unescape View File

@ -1,4 +1,4 @@
// htmlFilterContentTransformer.java
// ContentTransformer.java
// ---------------------------------
// (C) by Michael Peter Christen; mc@yacy.net
// first published on http://www.anomic.de
@ -146,6 +146,7 @@ public class ContentTransformer extends AbstractTransformer implements Transform
return text;
}
@Override
public char[] transformTag0(final String tagname, final Properties tagopts, final char quotechar) {
if (tagname.equals("img")) {
// check bluelist
@ -162,12 +163,14 @@ public class ContentTransformer extends AbstractTransformer implements Transform
return TransformerWriter.genTag0(tagname, tagopts, quotechar);
}
@Override
public char[] transformTag1(final String tagname, final Properties tagopts, final char[] text, final char quotechar) {
if (bluelistHit(tagopts.getProperty("href","").toCharArray())) return genBlueLetters(text.length);
if (bluelistHit(text)) return genBlueLetters(text.length);
return TransformerWriter.genTag1(tagname, tagopts, text, quotechar);
}
@Override
public void close() {
// free resources
super.close();

10
source/de/anomic/document/parser/html/ImageEntry.java
Unescape View File

@ -1,9 +1,12 @@
// htmlFilterImageEntry.java
// ImageEntry.java
// -----------------------------
// (C) by Michael Peter Christen; mc@yacy.net
// first published on http://www.anomic.de
// Frankfurt, Germany, 2006
// created 04.04.2006
//
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
@ -52,10 +55,12 @@ public class ImageEntry implements Comparable<ImageEntry> {
return this.height;
}
@Override
public String toString() {
return "{" + url.toString() + ", " + alt + ", " + width + "/" + height + "}";
}
@Override
public int hashCode() {
// if htmlFilterImageEntry elements are stored in a TreeSet, the biggest images shall be listed first
// this hash method therefore tries to compute a 'perfect hash' based on the size of the images
@ -80,6 +85,7 @@ public class ImageEntry implements Comparable<ImageEntry> {
return this.url.toString().compareTo((h).url.toString());
}
@Override
public boolean equals(final Object o) {
if(o != null && o instanceof ImageEntry) {
return compareTo((ImageEntry) o) == 0;

7
source/de/anomic/document/parser/html/Scraper.java
Unescape View File

@ -1,9 +1,12 @@
// htmlFilterScraper.java
// Scraper.java
// ---------------------------
// (C) by Michael Peter Christen; mc@yacy.net
// first published on http://www.anomic.de
// Frankfurt, Germany, 2004
// last major change: 18.02.2004
//
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by

8
source/de/anomic/document/parser/html/ScraperInputStream.java
Unescape View File

@ -1,12 +1,12 @@
// htmlFilterInputStream.java
// ScraperInputStream.java
// (C) 2005, 2006 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
// first published 2005 on http://www.anomic.de
//
// This is a part of YaCy, a peer-to-peer based web search engine
//
// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
// $LastChangedRevision: 1986 $
// $LastChangedBy: orbiter $
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
// LICENSE
//

24
source/de/anomic/document/parser/html/ScraperListener.java
Unescape View File

@ -1,3 +1,27 @@
// ScraperListener.java
// ---------------------------
// (C) by Michael Peter Christen; mc@yacy.net
// first published on http://www.anomic.de
// Frankfurt, Germany, 2004
//
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package de.anomic.document.parser.html;
import java.util.Properties;

7
source/de/anomic/document/parser/html/Transformer.java
Unescape View File

@ -1,9 +1,12 @@
// htmlFilterTransformer.java
// Transformer.java
// ---------------------------
// (C) by Michael Peter Christen; mc@yacy.net
// first published on http://www.anomic.de
// Frankfurt, Germany, 2004
// last major change: 18.02.2004
//
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by

9
source/de/anomic/document/parser/html/TransformerWriter.java
Unescape View File

@ -4,9 +4,9 @@
// first published on http://www.anomic.de
// Frankfurt, Germany, 2004, 2005
//
// $LastChangedDate: 2006-09-15 17:01:25 +0200 (Fr, 15 Sep 2006) $
// $LastChangedRevision: 2598 $
// $LastChangedBy: theli $
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
@ -351,6 +351,7 @@ public final class TransformerWriter extends Writer {
return tag.length - 1;
}
@Override
public void write(final int c) throws IOException {
// System.out.println((char) c);
if ((binaryUnsuspect) && (binaryHint((char)c))) {
@ -508,6 +509,7 @@ public final class TransformerWriter extends Writer {
}
}
@Override
public void write(final char b[]) throws IOException {
write(b, 0, b.length);
}
@ -526,6 +528,7 @@ public final class TransformerWriter extends Writer {
// if you want to flush all, call close() at end of writing;
}
@Override
protected void finalize() throws IOException {
// if we are forced to close, we of course flush the buffer first,
// then close the connection

7
source/de/anomic/document/parser/mimeTypeParser.java
Unescape View File

@ -6,7 +6,10 @@
//Frankfurt, Germany, 2005
//
//this file is contributed by Martin Thelian
//last major change: 16.05.2005
//
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
//This program is free software; you can redistribute it and/or modify
//it under the terms of the GNU General Public License as published by
@ -107,6 +110,7 @@ public class mimeTypeParser extends AbstractParser implements Parser {
}
@SuppressWarnings("unchecked")
@Override
public Document parse(final yacyURL location, String mimeType, final String charset, final File sourceFile) throws ParserException, InterruptedException {
final String orgMimeType = mimeType;
@ -185,6 +189,7 @@ public class mimeTypeParser extends AbstractParser implements Parser {
return mimeTypeParser.SUPPORTED_MIME_TYPES;
}
@Override
public void reset() {
// Nothing todo here at the moment
super.reset();

9
source/de/anomic/document/parser/odtParser.java
Unescape View File

@ -1,4 +1,4 @@
//zipParser.java
//odtParser.java
//------------------------
//part of YaCy
//(C) by Michael Peter Christen; mc@yacy.net
@ -6,7 +6,10 @@
//Frankfurt, Germany, 2005
//
//this file is contributed by Martin Thelian
//last major change: 16.05.2005
//
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
//This program is free software; you can redistribute it and/or modify
//it under the terms of the GNU General Public License as published by
@ -81,6 +84,7 @@ public class odtParser extends AbstractParser implements Parser {
return SUPPORTED_MIME_TYPES;
}
@Override
public Document parse(final yacyURL location, final String mimeType, final String charset, final File dest) throws ParserException, InterruptedException {
Writer writer = null;
@ -225,6 +229,7 @@ public class odtParser extends AbstractParser implements Parser {
}
}
@Override
public void reset() {
// Nothing todo here at the moment
super.reset();

15
source/de/anomic/document/parser/pdfParser.java
Unescape View File

@ -6,7 +6,10 @@
//Frankfurt, Germany, 2005
//
//this file is contributed by Martin Thelian
//last major change: 24.04.2005
//
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
//This program is free software; you can redistribute it and/or modify
//it under the terms of the GNU General Public License as published by
@ -54,7 +57,14 @@ public class pdfParser extends AbstractParser implements Parser {
* @see #getSupportedMimeTypes()
*/
public static final Hashtable<String, String> SUPPORTED_MIME_TYPES = new Hashtable<String, String>();
static { SUPPORTED_MIME_TYPES.put("application/pdf","pdf"); }
static {
SUPPORTED_MIME_TYPES.put("application/pdf","pdf");
SUPPORTED_MIME_TYPES.put("application/x-pdf","pdf");
SUPPORTED_MIME_TYPES.put("application/acrobat","pdf");
SUPPORTED_MIME_TYPES.put("applications/vnd.pdf","pdf");
SUPPORTED_MIME_TYPES.put("text/pdf","pdf");
SUPPORTED_MIME_TYPES.put("text/x-pdf","pdf");
}
/**
* a list of library names that are needed by this parser
@ -190,6 +200,7 @@ public class pdfParser extends AbstractParser implements Parser {
}
}
@Override
public void reset() {
// Nothing todo here at the moment
super.reset();

17
source/de/anomic/document/parser/pptParser.java
Unescape View File

@ -11,7 +11,6 @@
// $LastChangedRevision$
// $LastChangedBy$
//
//
//This program is free software; you can redistribute it and/or modify
//it under the terms of the GNU General Public License as published by
//the Free Software Foundation; either version 2 of the License, or
@ -46,12 +45,18 @@ public class pptParser extends AbstractParser implements Parser {
* a list of mime types that are supported by this parser class
* @see #getSupportedMimeTypes()
*/
public static final Hashtable<String, String> SUPPORTED_MIME_TYPES = new Hashtable<String, String>();
public static final Hashtable<String, String> SUPPORTED_MIME_TYPES = new Hashtable<String, String>();
static final String fileExtensions = "ppt,pps";
static {
SUPPORTED_MIME_TYPES.put("application/mspowerpoint","ppt,pps");
SUPPORTED_MIME_TYPES.put("application/powerpoint","ppt,pps");
SUPPORTED_MIME_TYPES.put("application/vnd.ms-powerpoint","ppt,pps");
}
SUPPORTED_MIME_TYPES.put("application/mspowerpoint",fileExtensions);
SUPPORTED_MIME_TYPES.put("application/powerpoint",fileExtensions);
SUPPORTED_MIME_TYPES.put("application/vnd.ms-powerpoint",fileExtensions);
SUPPORTED_MIME_TYPES.put("application/ms-powerpoint",fileExtensions);
SUPPORTED_MIME_TYPES.put("application/mspowerpnt",fileExtensions);
SUPPORTED_MIME_TYPES.put("application/vnd-mspowerpoint",fileExtensions);
SUPPORTED_MIME_TYPES.put("application/x-powerpoint",fileExtensions);
SUPPORTED_MIME_TYPES.put("application/x-m",fileExtensions);
}
/**
* a list of library names that are needed by this parser

15
source/de/anomic/document/parser/psParser.java
Unescape View File

@ -6,7 +6,10 @@
//Frankfurt, Germany, 2007
//
//this file is contributed by Martin Thelian
//last major change: 15.09.2005
//
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
//This program is free software; you can redistribute it and/or modify
//it under the terms of the GNU General Public License as published by
@ -48,8 +51,10 @@ public class psParser extends AbstractParser implements Parser {
*/
public static final Hashtable<String, String> SUPPORTED_MIME_TYPES = new Hashtable<String, String>();
static {
SUPPORTED_MIME_TYPES.put("application/postscript","ps");
SUPPORTED_MIME_TYPES.put("text/postscript","ps");
SUPPORTED_MIME_TYPES.put("application/ps","ps");
SUPPORTED_MIME_TYPES.put("application/x-postscript","ps");
SUPPORTED_MIME_TYPES.put("application/x-ps","ps");
SUPPORTED_MIME_TYPES.put("application/x-postscript-not-eps","ps");
}
/**
@ -58,7 +63,7 @@ public class psParser extends AbstractParser implements Parser {
*/
private static final String[] LIBX_DEPENDENCIES = new String[] {};
private static Object modeScan = new Object();
private final static Object modeScan = new Object();
private static boolean modeScanDone = false;
private static String parserMode = "java";
@ -96,6 +101,7 @@ public class psParser extends AbstractParser implements Parser {
}
@Override
public Document parse(final yacyURL location, final String mimeType, final String charset, final File sourceFile) throws ParserException, InterruptedException {
File outputFile = null;
@ -261,6 +267,7 @@ public class psParser extends AbstractParser implements Parser {
if (execCode != 0) throw new Exception("Unable to convert ps to ascii. ps2ascii returned statuscode " + execCode + "\n" + procErr.toString());
}
@Override
public void reset() {
// Nothing todo here at the moment
super.reset();

7
source/de/anomic/document/parser/rpmParser.java
Unescape View File

@ -6,7 +6,10 @@
//Frankfurt, Germany, 2005
//
//this file is contributed by Martin Thelian
//last major change: 20.11.2005
//
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
//This program is free software; you can redistribute it and/or modify
//it under the terms of the GNU General Public License as published by
@ -90,6 +93,7 @@ public class rpmParser extends AbstractParser implements Parser {
}
}
@Override
public Document parse(final yacyURL location, final String mimeType, final String charset, final File sourceFile) throws ParserException, InterruptedException {
RPMFile rpmFile = null;
try {
@ -155,6 +159,7 @@ public class rpmParser extends AbstractParser implements Parser {
}
}
@Override
public void reset() {
// Nothing todo here at the moment
super.reset();

17
source/de/anomic/document/parser/rssParser.java
Unescape View File

@ -6,7 +6,10 @@
//Frankfurt, Germany, 2005
//
//this file is contributed by Martin Thelian
//last major change: 16.05.2005
//
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
//This program is free software; you can redistribute it and/or modify
//it under the terms of the GNU General Public License as published by
@ -56,12 +59,13 @@ public class rssParser extends AbstractParser implements Parser {
* a list of mime types that are supported by this parser class
* @see #getSupportedMimeTypes()
*/
public static final Hashtable<String, String> SUPPORTED_MIME_TYPES = new Hashtable<String, String>();
public static final Hashtable<String, String> SUPPORTED_MIME_TYPES = new Hashtable<String, String>();
static final String fileExtensions = "xml,rss,rdf";
static {
SUPPORTED_MIME_TYPES.put("text/rss","xml,rss,rdf");
SUPPORTED_MIME_TYPES.put("application/rdf+xml","xml,rss,rdf");
SUPPORTED_MIME_TYPES.put("application/rss+xml","xml,rss,rdf");
SUPPORTED_MIME_TYPES.put("application/atom+xml","xml,atom");
SUPPORTED_MIME_TYPES.put("text/rss",fileExtensions);
SUPPORTED_MIME_TYPES.put("application/rdf+xml",fileExtensions);
SUPPORTED_MIME_TYPES.put("application/rss+xml",fileExtensions);
SUPPORTED_MIME_TYPES.put("application/atom+xml",fileExtensions);
}
/**
@ -182,6 +186,7 @@ public class rssParser extends AbstractParser implements Parser {
return SUPPORTED_MIME_TYPES;
}
@Override
public void reset() {
// Nothing todo here at the moment
super.reset();

10
source/de/anomic/document/parser/rtfParser.java
Unescape View File

@ -6,7 +6,10 @@
//Frankfurt, Germany, 2005
//
//this file is contributed by Martin Thelian
//last major change: 16.05.2005
//
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
//This program is free software; you can redistribute it and/or modify
//it under the terms of the GNU General Public License as published by
@ -46,6 +49,11 @@ public class rtfParser extends AbstractParser implements Parser {
static {
SUPPORTED_MIME_TYPES.put("application/rtf","rtf");
SUPPORTED_MIME_TYPES.put("text/rtf","rtf");
SUPPORTED_MIME_TYPES.put("application/x-rtf","rtf");
SUPPORTED_MIME_TYPES.put("text/richtext","rtf");
SUPPORTED_MIME_TYPES.put("application/msword","rtf");
SUPPORTED_MIME_TYPES.put("application/doc","rtf");
SUPPORTED_MIME_TYPES.put("application/x-soffice","rtf");
}
/**

12
source/de/anomic/document/parser/sevenzipParser.java
Unescape View File

@ -6,7 +6,11 @@
// Frankfurt, Germany, 2004
//
// This file ist contributed by Franz Brausze
//
//
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
@ -98,11 +102,13 @@ public class sevenzipParser extends AbstractParser implements Parser {
}
}
@Override
public Document parse(final yacyURL location, final String mimeType, final String charset,
final byte[] source) throws ParserException, InterruptedException {
return parse(location, mimeType, charset, new ByteArrayIInStream(source), Parser.MAX_KEEP_IN_MEMORY_SIZE - source.length);
}
@Override
public Document parse(final yacyURL location, final String mimeType, final String charset,
final File sourceFile) throws ParserException, InterruptedException {
try {
@ -150,6 +156,7 @@ public class sevenzipParser extends AbstractParser implements Parser {
this.prefix = prefix;
}
@Override
public void PrepareOperation(final int arg0) {
this.extractMode = (arg0 == IInArchive.NExtract_NAskMode_kExtract);
switch (arg0) {
@ -165,6 +172,7 @@ public class sevenzipParser extends AbstractParser implements Parser {
}
}
@Override
public void SetOperationResult(final int arg0) throws IOException {
if (arg0 != IInArchive.NExtract_NOperationResult_kOK) {
this.NumErrors++;
@ -207,6 +215,7 @@ public class sevenzipParser extends AbstractParser implements Parser {
}
}
@Override
public OutputStream GetStream(final int index, final int askExtractMode) throws IOException {
final SevenZipEntry item = super.archiveHandler.getEntry(index);
super.filePath = item.getName();
@ -260,6 +269,7 @@ public class sevenzipParser extends AbstractParser implements Parser {
return this.sbais.read();
}
@Override
public int read(final byte[] b, final int off, final int len) throws IOException {
return this.sbais.read(b, off, len);
}

2
source/de/anomic/document/parser/swfParser.java
Unescape View File

@ -48,6 +48,8 @@ public class swfParser extends AbstractParser implements Parser {
static {
SUPPORTED_MIME_TYPES.put("application/x-shockwave-flash","swf");
SUPPORTED_MIME_TYPES.put("application/x-shockwave-flash2-preview","swf");
SUPPORTED_MIME_TYPES.put("application/futuresplash","swf");
SUPPORTED_MIME_TYPES.put("image/vnd.rn-realflash","swf");
}
/**

10
source/de/anomic/document/parser/tarParser.java
Unescape View File

@ -6,7 +6,10 @@
//Frankfurt, Germany, 2005
//
//this file is contributed by Martin Thelian
//last major change: 16.05.2005
//
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
//This program is free software; you can redistribute it and/or modify
//it under the terms of the GNU General Public License as published by
@ -60,6 +63,10 @@ public class tarParser extends AbstractParser implements Parser {
static {
SUPPORTED_MIME_TYPES.put("application/x-tar","tar");
SUPPORTED_MIME_TYPES.put("application/tar","tar");
SUPPORTED_MIME_TYPES.put("applicaton/x-gtar","tar");
SUPPORTED_MIME_TYPES.put("multipart/x-tar","tar");
SUPPORTED_MIME_TYPES.put("application/x-compress","tar");
SUPPORTED_MIME_TYPES.put("application/x-compressed","tar");
}
/**
@ -227,6 +234,7 @@ public class tarParser extends AbstractParser implements Parser {
}
}
@Override
public void reset() {
// Nothing todo here at the moment
super.reset();

15
source/de/anomic/document/parser/vcfParser.java
Unescape View File

@ -6,7 +6,10 @@
//Frankfurt, Germany, 2005
//
//this file is contributed by Martin Thelian
//last major change: 20.11.2005
//
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
//This program is free software; you can redistribute it and/or modify
//it under the terms of the GNU General Public License as published by
@ -61,8 +64,13 @@ public class vcfParser extends AbstractParser implements Parser {
public static final Hashtable<String, String> SUPPORTED_MIME_TYPES = new Hashtable<String, String>();
static {
SUPPORTED_MIME_TYPES.put("text/x-vcard","vcf");
SUPPORTED_MIME_TYPES.put("application/vcard","vcf");
}
SUPPORTED_MIME_TYPES.put("application/vcard","vcf");
SUPPORTED_MIME_TYPES.put("text/anytext","vcf");
SUPPORTED_MIME_TYPES.put("text/directory","vcf");
SUPPORTED_MIME_TYPES.put("application/x-versit","vcf");
SUPPORTED_MIME_TYPES.put("text/x-versit","vcf");
SUPPORTED_MIME_TYPES.put("text/x-vcalendar","vcf");
}
/**
* a list of library names that are needed by this parser
@ -244,6 +252,7 @@ public class vcfParser extends AbstractParser implements Parser {
}
}
@Override
public void reset() {
// Nothing todo here at the moment
super.reset();

3
source/de/anomic/document/parser/xlsParser.java
Unescape View File

@ -64,6 +64,9 @@ public class xlsParser extends AbstractParser implements Parser, HSSFListener {
SUPPORTED_MIME_TYPES.put("application/vnd.ms-excel","xls");
SUPPORTED_MIME_TYPES.put("application/x-excel","xls");
SUPPORTED_MIME_TYPES.put("application/x-msexcel","xls");
SUPPORTED_MIME_TYPES.put("application/x-ms-excel","xls");
SUPPORTED_MIME_TYPES.put("application/x-dos_ms_excel","xls");
SUPPORTED_MIME_TYPES.put("application/xls","xls");
}
/**

6
source/de/anomic/document/parser/xml/RSSFeed.java
Unescape View File

@ -4,9 +4,9 @@
//
// This is a part of YaCy, a peer-to-peer based web search engine
//
// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
// $LastChangedRevision: 1986 $
// $LastChangedBy: orbiter $
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
// LICENSE
//

11
source/de/anomic/document/parser/xml/RSSReader.java
Unescape View File

@ -1,12 +1,12 @@
// rssReader.java
// RSSReader.java
// (C) 2007 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
// first published 16.07.2007 on http://yacy.net
//
// This is a part of YaCy, a peer-to-peer based web search engine
//
// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
// $LastChangedRevision: 1986 $
// $LastChangedBy: orbiter $
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
// LICENSE
//
@ -122,6 +122,7 @@ public class RSSReader extends DefaultHandler {
return reader;
}
@Override
public void startElement(final String uri, final String name, final String tag, final Attributes atts) throws SAXException {
if ("channel".equals(tag)) {
item = new RSSMessage();
@ -134,6 +135,7 @@ public class RSSReader extends DefaultHandler {
}
}
@Override
public void endElement(final String uri, final String name, final String tag) {
if (tag == null) return;
if ("channel".equals(tag)) {
@ -159,6 +161,7 @@ public class RSSReader extends DefaultHandler {
}
}
@Override
public void characters(final char ch[], final int start, final int length) {
if (parsingItem || parsingChannel) {
buffer.append(ch, start, length);

9
source/de/anomic/document/parser/xml/opensearchdescriptionReader.java
Unescape View File

@ -4,9 +4,9 @@
//
// This is a part of YaCy, a peer-to-peer based web search engine
//
// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
// $LastChangedRevision: 1986 $
// $LastChangedBy: orbiter $
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
// LICENSE
//
@ -169,6 +169,7 @@ public class opensearchdescriptionReader extends DefaultHandler {
return reader;
}
@Override
public void startElement(final String uri, final String name, final String tag, final Attributes atts) throws SAXException {
if ("channel".equals(tag)) {
channel = new Item();
@ -176,6 +177,7 @@ public class opensearchdescriptionReader extends DefaultHandler {
}
}
@Override
public void endElement(final String uri, final String name, final String tag) {
if (tag == null) return;
if ("channel".equals(tag)) {
@ -187,6 +189,7 @@ public class opensearchdescriptionReader extends DefaultHandler {
}
}
@Override
public void characters(final char ch[], final int start, final int length) {
if (parsingChannel) {
buffer.append(ch, start, length);

10
source/de/anomic/document/parser/zipParser.java
Unescape View File

@ -6,7 +6,10 @@
//Frankfurt, Germany, 2005
//
//this file is contributed by Martin Thelian
//last major change: 16.05.2005
//
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
//This program is free software; you can redistribute it and/or modify
//it under the terms of the GNU General Public License as published by
@ -59,6 +62,10 @@ public class zipParser extends AbstractParser implements Parser {
SUPPORTED_MIME_TYPES.put("application/zip","zip");
SUPPORTED_MIME_TYPES.put("application/x-zip","zip");
SUPPORTED_MIME_TYPES.put("application/x-zip-compressed","zip");
SUPPORTED_MIME_TYPES.put("application/octet-stream","zip");
SUPPORTED_MIME_TYPES.put("application/x-compress","zip");
SUPPORTED_MIME_TYPES.put("application/x-compressed","zip");
SUPPORTED_MIME_TYPES.put("multipart/x-zip","zip");
SUPPORTED_MIME_TYPES.put("application/java-archive","jar");
}
@ -211,6 +218,7 @@ public class zipParser extends AbstractParser implements Parser {
}
}
@Override
public void reset() {
// Nothing todo here at the moment
super.reset();

Write Preview
Loading…
Cancel
Save