to make this possible, the yacyURL must be able to process file:// urls, which has also been implemented testing of the new class resulted in some bugfixes in other classes git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6313 6c8d7289-2bf4-0310-a012-ef5d649a1542pull/1/head
orbiter
16 years ago
parent
2e41e10ffd
commit
68465c37af
@ -0,0 +1,192 @@
|
||||
// DocumentIndex.java
|
||||
// (C) 2009 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
|
||||
// first published 14.09.2009 on http://yacy.net;
|
||||
//
|
||||
// This is a part of YaCy, a peer-to-peer based web search engine
|
||||
//
|
||||
// $LastChangedDate: 2009-05-28 01:51:34 +0200 (Do, 28 Mai 2009) $
|
||||
// $LastChangedRevision: 5988 $
|
||||
// $LastChangedBy: orbiter $
|
||||
//
|
||||
// LICENSE
|
||||
//
|
||||
// This program is free software; you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation; either version 2 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with this program; if not, write to the Free Software
|
||||
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
|
||||
|
||||
package de.anomic.kelondro.text;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.util.Date;
|
||||
|
||||
import de.anomic.document.Condenser;
|
||||
import de.anomic.document.Document;
|
||||
import de.anomic.document.Parser;
|
||||
import de.anomic.document.ParserException;
|
||||
import de.anomic.kelondro.text.metadataPrototype.URLMetadataRow;
|
||||
import de.anomic.search.QueryParams;
|
||||
import de.anomic.search.RankingProfile;
|
||||
import de.anomic.search.ResultEntry;
|
||||
import de.anomic.search.SearchEvent;
|
||||
import de.anomic.search.SearchEventCache;
|
||||
import de.anomic.yacy.yacyURL;
|
||||
import de.anomic.yacy.logging.Log;
|
||||
|
||||
/**
|
||||
* convenience class to access the yacycore library from outside of yacy to put files into the index
|
||||
* @author Michael Christen
|
||||
*
|
||||
*/
|
||||
public class DocumentIndex extends Segment {
|
||||
|
||||
private RankingProfile textRankingDefault = new RankingProfile(QueryParams.CONTENTDOM_TEXT);
|
||||
//private Bitfield zeroConstraint = new Bitfield(4);
|
||||
|
||||
public DocumentIndex(Log log, final File segmentPath) throws IOException {
|
||||
super(log, segmentPath, 100000, targetFileSize * 4 - 1, false, false);
|
||||
}
|
||||
|
||||
public DocumentIndex(final File segmentPath) throws IOException {
|
||||
this(new Log("DocumentIndex"), segmentPath);
|
||||
}
|
||||
|
||||
/**
|
||||
* put a single file into the index
|
||||
* @param file
|
||||
* @return a metadata object that has been generated to identify the file
|
||||
* @throws IOException in case that the file does not exist or cannot be parsed
|
||||
*/
|
||||
public URLMetadataRow add(File file) throws IOException {
|
||||
if (file == null) throw new IOException("file = null");
|
||||
if (file.isDirectory()) throw new IOException("file should be a document, not a path");
|
||||
if (!file.canRead()) throw new IOException("cannot read file");
|
||||
yacyURL url = new yacyURL("file:" + file.getAbsolutePath());
|
||||
Document document;
|
||||
try {
|
||||
document = Parser.parseSource(url, null, null, file);
|
||||
} catch (InterruptedException e) {
|
||||
throw new IOException("cannot parse " + file.toString() + ": " + e.getMessage());
|
||||
} catch (ParserException e) {
|
||||
throw new IOException("cannot parse " + file.toString() + ": " + e.getMessage());
|
||||
}
|
||||
final Condenser condenser = new Condenser(document, true, true);
|
||||
return super.storeDocument(
|
||||
url,
|
||||
null,
|
||||
new Date(file.lastModified()),
|
||||
file.length(),
|
||||
document,
|
||||
condenser
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* add a file or a directory of files to the index
|
||||
* If the given file is a path to a directory, the complete sub-tree is indexed
|
||||
* @param start
|
||||
*/
|
||||
public void addAll(File start) {
|
||||
assert (start != null);
|
||||
assert (start.canRead());
|
||||
if (!start.isDirectory()) {
|
||||
try {
|
||||
add(start);
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
return;
|
||||
}
|
||||
String[] s = start.list();
|
||||
File w;
|
||||
for (String t: s) {
|
||||
w = new File(start, t);
|
||||
if (w.canRead() && ! w.isHidden()) {
|
||||
if (w.isDirectory()) {
|
||||
addAll(w);
|
||||
} else {
|
||||
try {
|
||||
add(w);
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* do a full-text search of a given string and return a specific number of results
|
||||
* @param querystring
|
||||
* @param pos
|
||||
* @param count
|
||||
* @return a list of files that contain the given string
|
||||
*/
|
||||
public File[] find(String querystring, int pos, int count) {
|
||||
QueryParams query = new QueryParams(querystring, 100, textRankingDefault, null);
|
||||
SearchEvent se = SearchEventCache.getEvent(query, this, null, null, null, false);
|
||||
File[] result = new File[count];
|
||||
ResultEntry re;
|
||||
for (int i = 0; i < count; i++) {
|
||||
re = se.oneResult(pos + i);
|
||||
result[i] = (re == null) ? null : re.url().getLocalFile();
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* find the given string and return 20 hits
|
||||
* @param querystring
|
||||
* @return a list of files that contain the word
|
||||
*/
|
||||
public File[] find(String querystring) {
|
||||
return find(querystring, 0, 20);
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
// first argument: path to segment
|
||||
// second argument: either 'add' or 'search'
|
||||
// third and more arguments exists only in case that second argument is 'search': these are then the search words
|
||||
//
|
||||
// example:
|
||||
// DocumentIndex yacyindex add test/parsertest
|
||||
// DocumentIndex yacyindex search steht
|
||||
System.setProperty("java.awt.headless", "true");
|
||||
if (args.length < 3) return;
|
||||
File segmentPath = new File(args[0]);
|
||||
System.out.println("using index files at " + segmentPath.getAbsolutePath());
|
||||
try {
|
||||
if (args[1].equals("add")) {
|
||||
File f = new File(args[2]);
|
||||
DocumentIndex di = new DocumentIndex(segmentPath);
|
||||
di.addAll(f);
|
||||
di.close();
|
||||
} else {
|
||||
String query = "";
|
||||
for (int i = 2; i < args.length; i++) query += args[i];
|
||||
query.trim();
|
||||
DocumentIndex di = new DocumentIndex(segmentPath);
|
||||
File[] results = di.find(query);
|
||||
for (File f: results) {
|
||||
if (f != null) System.out.println(f.toString());
|
||||
}
|
||||
di.close();
|
||||
}
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
//System.exit(0);
|
||||
}
|
||||
|
||||
}
|
||||
Loading…
Reference in new issue