http://roar.eprints.org/index.php?action=csv - integrated the csv parser into the crawlers parser list - added an extension to the OAI-PMH import function to download and show the roar csv file using the csv parser git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6448 6c8d7289-2bf4-0310-a012-ef5d649a1542pull/1/head
orbiter
16 years ago
parent
0f63de8236
commit
9b6762ec2e
@ -0,0 +1,149 @@
|
||||
// CSVParser
|
||||
// (C) 2009 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
|
||||
// first published 02.10.2009 on http://yacy.net
|
||||
//
|
||||
// This is a part of YaCy, a peer-to-peer based web search engine
|
||||
//
|
||||
// $LastChangedDate: 2009-09-23 23:26:14 +0200 (Mi, 23 Sep 2009) $
|
||||
// $LastChangedRevision: 6340 $
|
||||
// $LastChangedBy: low012 $
|
||||
//
|
||||
// LICENSE
|
||||
//
|
||||
// This program is free software; you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation; either version 2 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with this program; if not, write to the Free Software
|
||||
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
|
||||
|
||||
package net.yacy.document.parser;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.InputStreamReader;
|
||||
import java.io.UnsupportedEncodingException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
import net.yacy.document.AbstractParser;
|
||||
import net.yacy.document.Document;
|
||||
import net.yacy.document.Idiom;
|
||||
import net.yacy.document.ParserException;
|
||||
import net.yacy.kelondro.data.meta.DigestURI;
|
||||
|
||||
/**
|
||||
* a parser for comma-separated values
|
||||
* The values may also be separated by semicolon or tab,
|
||||
* the separator character is detected automatically
|
||||
*/
|
||||
public class csvParser extends AbstractParser implements Idiom {
|
||||
|
||||
/**
|
||||
* a list of mime types that are supported by this parser class
|
||||
* @see #getSupportedMimeTypes()
|
||||
*/
|
||||
public static final Set<String> SUPPORTED_MIME_TYPES = new HashSet<String>();
|
||||
public static final Set<String> SUPPORTED_EXTENSIONS = new HashSet<String>();
|
||||
static {
|
||||
SUPPORTED_EXTENSIONS.add("csv");
|
||||
}
|
||||
|
||||
public csvParser() {
|
||||
super("Comma Separated Value Parser");
|
||||
}
|
||||
|
||||
public Set<String> supportedMimeTypes() {
|
||||
return SUPPORTED_MIME_TYPES;
|
||||
}
|
||||
|
||||
public Set<String> supportedExtensions() {
|
||||
return SUPPORTED_EXTENSIONS;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Document parse(DigestURI location, String mimeType, String charset, InputStream source) throws ParserException, InterruptedException {
|
||||
// construct a document using all cells of the document
|
||||
// the first row is used as headline
|
||||
// all lines are artificially terminated by a '.' to separate them as sentence for the condenser.
|
||||
List<String[]> table = getTable(location, mimeType, charset, source);
|
||||
if (table.size() == 0) throw new ParserException("document has no lines", location);
|
||||
StringBuilder sb = new StringBuilder();
|
||||
for (String[] row: table) sb.append(concatRow(row)).append(' ');
|
||||
try {
|
||||
return new Document(
|
||||
location,
|
||||
mimeType,
|
||||
charset,
|
||||
null,
|
||||
null,
|
||||
concatRow(table.get(0)),
|
||||
"",
|
||||
null,
|
||||
null,
|
||||
sb.toString().getBytes(charset),
|
||||
null,
|
||||
null);
|
||||
} catch (UnsupportedEncodingException e) {
|
||||
throw new ParserException("error in csvParser, getBytes: " + e.getMessage(), location);
|
||||
}
|
||||
}
|
||||
|
||||
public String concatRow(String[] column) {
|
||||
StringBuilder sb = new StringBuilder(80);
|
||||
for (int i = 0; i < column.length; i++) {
|
||||
if (i != 0) sb.append(' ');
|
||||
sb.append(column[i]);
|
||||
}
|
||||
sb.append('.');
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
public List<String[]> getTable(DigestURI location, String mimeType, String charset, InputStream source) {
|
||||
ArrayList<String[]> rows = new ArrayList<String[]>();
|
||||
BufferedReader reader;
|
||||
try {
|
||||
reader = new BufferedReader(new InputStreamReader(source, charset));
|
||||
} catch (UnsupportedEncodingException e1) {
|
||||
reader = new BufferedReader(new InputStreamReader(source));
|
||||
}
|
||||
String row;
|
||||
String separator = null;
|
||||
int columns = -1;
|
||||
try {
|
||||
while ((row = reader.readLine()) != null) {
|
||||
row = row.trim();
|
||||
if (row.length() == 0) continue;
|
||||
if (separator == null) {
|
||||
// try comma, semicolon and tab; take that one that results with more columns
|
||||
String[] colc = row.split(",");
|
||||
String[] cols = row.split(";");
|
||||
String[] colt = row.split("\t");
|
||||
if (colc.length >= cols.length && colc.length >= colt.length) separator = ",";
|
||||
if (cols.length >= colc.length && cols.length >= colt.length) separator = ";";
|
||||
if (colt.length >= cols.length && colt.length >= colc.length) separator = "\t";
|
||||
}
|
||||
String[] cols = row.split(separator);
|
||||
if (columns == -1) columns = cols.length;
|
||||
//if (cols.length != columns) continue; // skip lines that have the wrong number of columns
|
||||
rows.add(cols);
|
||||
}
|
||||
} catch (IOException e) {
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
Loading…
Reference in new issue