- it is now possible to crawl the local file system with an intranet peer - redesign of URL handling - refactoring: created LGPLed package cora: 'content retrieval api' which may be used externally by other applications without yacy core elements because it has no dependencies to other parts of yacy git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6902 6c8d7289-2bf4-0310-a012-ef5d649a1542pull/1/head
orbiter
15 years ago
parent
2fd795207c
commit
11639aef35
@ -0,0 +1,144 @@
|
||||
/**
|
||||
* FileLoader
|
||||
* Copyright 2010 by Michael Peter Christen
|
||||
* First released 25.5.2010 at http://yacy.net
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Lesser General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Lesser General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public License
|
||||
* along with this program in the file COPYING.LESSER.
|
||||
* If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package de.anomic.crawler.retrieval;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
|
||||
import de.anomic.http.server.HeaderFramework;
|
||||
import de.anomic.http.server.RequestHeader;
|
||||
import de.anomic.http.server.ResponseHeader;
|
||||
import de.anomic.net.ftpc;
|
||||
import de.anomic.search.Segments;
|
||||
import de.anomic.search.Switchboard;
|
||||
import de.anomic.data.MimeTable;
|
||||
|
||||
import net.yacy.document.TextParser;
|
||||
import net.yacy.kelondro.data.meta.DigestURI;
|
||||
import net.yacy.kelondro.logging.Log;
|
||||
import net.yacy.kelondro.util.DateFormatter;
|
||||
import net.yacy.kelondro.util.FileUtils;
|
||||
|
||||
public class FileLoader {
|
||||
|
||||
private final Switchboard sb;
|
||||
private final Log log;
|
||||
private final int maxFileSize;
|
||||
|
||||
public FileLoader(final Switchboard sb, final Log log) {
|
||||
this.sb = sb;
|
||||
this.log = log;
|
||||
maxFileSize = (int) sb.getConfigLong("crawler.file.maxFileSize", -1l);
|
||||
}
|
||||
|
||||
public Response load(final Request request, boolean acceptOnlyParseable) throws IOException {
|
||||
DigestURI url = request.url();
|
||||
if (!url.getProtocol().equals("file")) throw new IOException("wrong loader for FileLoader: " + url.getProtocol());
|
||||
|
||||
RequestHeader requestHeader = new RequestHeader();
|
||||
if (request.referrerhash() != null) {
|
||||
DigestURI ur = sb.getURL(Segments.Process.LOCALCRAWLING, request.referrerhash());
|
||||
if (ur != null) requestHeader.put(RequestHeader.REFERER, ur.toNormalform(true, false));
|
||||
}
|
||||
|
||||
// process directories: transform them to html with meta robots=noindex (using the ftpc lib)
|
||||
if (url.isDirectory()) {
|
||||
String[] l = url.list();
|
||||
if (l == null) {
|
||||
// this can only happen if there is no connection or the directory does not exist
|
||||
log.logInfo("directory listing not available. URL = " + request.url().toString());
|
||||
sb.crawlQueues.errorURL.push(request, this.sb.peers.mySeed().hash.getBytes(), new Date(), 1, "directory listing not available. URL = " + request.url().toString());
|
||||
throw new IOException("directory listing not available. URL = " + request.url().toString());
|
||||
}
|
||||
String u = url.toNormalform(true, true);
|
||||
List<String> list = new ArrayList<String>();
|
||||
for (String s: l) {
|
||||
list.add(u + ((u.endsWith("/") || u.endsWith("\\")) ? "" : "/") + s);
|
||||
}
|
||||
|
||||
StringBuilder content = ftpc.dirhtml(u, null, null, null, list, true);
|
||||
|
||||
ResponseHeader responseHeader = new ResponseHeader();
|
||||
responseHeader.put(HeaderFramework.LAST_MODIFIED, DateFormatter.formatRFC1123(new Date()));
|
||||
responseHeader.put(HeaderFramework.CONTENT_TYPE, "text/html");
|
||||
Response response = new Response(
|
||||
request,
|
||||
requestHeader,
|
||||
responseHeader,
|
||||
"200",
|
||||
sb.crawler.profilesActiveCrawls.getEntry(request.profileHandle()),
|
||||
content.toString().getBytes());
|
||||
|
||||
return response;
|
||||
}
|
||||
|
||||
// create response header
|
||||
String mime = MimeTable.ext2mime(url.getFileExtension());
|
||||
ResponseHeader responseHeader = new ResponseHeader();
|
||||
responseHeader.put(HeaderFramework.LAST_MODIFIED, DateFormatter.formatRFC1123(new Date(url.lastModified())));
|
||||
responseHeader.put(HeaderFramework.CONTENT_TYPE, mime);
|
||||
|
||||
// check mime type and availability of parsers
|
||||
// and also check resource size and limitation of the size
|
||||
long size = url.length();
|
||||
String parserError = null;
|
||||
if ((acceptOnlyParseable && (parserError = TextParser.supports(url, mime)) != null) ||
|
||||
(size > maxFileSize && maxFileSize >= 0)) {
|
||||
// we know that we cannot process that file before loading
|
||||
// only the metadata is returned
|
||||
|
||||
if (parserError != null) {
|
||||
log.logInfo("No parser available in File crawler: '" + parserError + "' for URL " + request.url().toString() + ": parsing only metadata");
|
||||
} else {
|
||||
log.logInfo("Too big file in File crawler with size = " + size + " Bytes for URL " + request.url().toString() + ": parsing only metadata");
|
||||
}
|
||||
|
||||
// create response with metadata only
|
||||
responseHeader.put(HeaderFramework.CONTENT_TYPE, "text/plain");
|
||||
Response response = new Response(
|
||||
request,
|
||||
requestHeader,
|
||||
responseHeader,
|
||||
"200",
|
||||
sb.crawler.profilesActiveCrawls.getEntry(request.profileHandle()),
|
||||
url.toNormalform(true, true).getBytes());
|
||||
return response;
|
||||
}
|
||||
|
||||
// load the resource
|
||||
InputStream is = url.getInputStream();
|
||||
byte[] b = FileUtils.read(is);
|
||||
is.close();
|
||||
|
||||
// create response with loaded content
|
||||
Response response = new Response(
|
||||
request,
|
||||
requestHeader,
|
||||
responseHeader,
|
||||
"200",
|
||||
sb.crawler.profilesActiveCrawls.getEntry(request.profileHandle()),
|
||||
b);
|
||||
return response;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,42 @@
|
||||
/**
|
||||
* Channel
|
||||
* Copyright 2010 by Michael Peter Christen
|
||||
* First released 10.5.2010 at http://yacy.net
|
||||
*
|
||||
* This file is part of YaCy Content Integration
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Lesser General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Lesser General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public License
|
||||
* along with this program in the file COPYING.LESSER.
|
||||
* If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package net.yacy.cora.document;
|
||||
|
||||
public interface Channel extends Iterable<Hit> {
|
||||
|
||||
public void setTitle(String title);
|
||||
|
||||
public void setLink(String link);
|
||||
|
||||
public void setDescription(String description);
|
||||
|
||||
public void setImageURL(String imageUrl);
|
||||
|
||||
public void setTotalResults(String totalResults);
|
||||
|
||||
public void setStartIndex(String startIndex);
|
||||
|
||||
public void setItemsPerPage(String itemsPerPage);
|
||||
|
||||
public void setSearchTerms(String searchTerms);
|
||||
}
|
||||
@ -0,0 +1,27 @@
|
||||
/**
|
||||
* Channels
|
||||
* Copyright 2010 by Michael Peter Christen
|
||||
* First released 10.5.2010 at http://yacy.net
|
||||
*
|
||||
* This file is part of YaCy Content Integration
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Lesser General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Lesser General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public License
|
||||
* along with this program in the file COPYING.LESSER.
|
||||
* If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package net.yacy.cora.document;
|
||||
|
||||
public class Channels {
|
||||
|
||||
}
|
||||
@ -0,0 +1,74 @@
|
||||
/**
|
||||
* Hit
|
||||
* Copyright 2010 by Michael Peter Christen
|
||||
* First released 10.5.2010 at http://yacy.net
|
||||
*
|
||||
* This file is part of YaCy Content Integration
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Lesser General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Lesser General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public License
|
||||
* along with this program in the file COPYING.LESSER.
|
||||
* If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package net.yacy.cora.document;
|
||||
|
||||
public interface Hit {
|
||||
|
||||
public void setAuthor(String title);
|
||||
|
||||
public void setCopyright(String title);
|
||||
|
||||
public void setCategory(String title);
|
||||
|
||||
public void setTitle(String title);
|
||||
|
||||
public void setLink(String link);
|
||||
|
||||
public void setReferrer(String title);
|
||||
|
||||
public void setLanguage(String title);
|
||||
|
||||
public void setDescription(String description);
|
||||
|
||||
public void setCreator(String pubdate);
|
||||
|
||||
public void setPubDate(String pubdate);
|
||||
|
||||
public void setGuid(String guid);
|
||||
|
||||
public void setDocs(String guid);
|
||||
|
||||
|
||||
public String getAuthor();
|
||||
|
||||
public String getCopyright();
|
||||
|
||||
public String getCategory();
|
||||
|
||||
public String getTitle();
|
||||
|
||||
public String getLink();
|
||||
|
||||
public String getReferrer();
|
||||
|
||||
public String getLanguage();
|
||||
|
||||
public String getDescription();
|
||||
|
||||
public String getPubDate();
|
||||
|
||||
public String getGuid();
|
||||
|
||||
public String getDocs();
|
||||
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,90 @@
|
||||
/**
|
||||
* HttpConnector
|
||||
* Copyright 2010 by Michael Peter Christen
|
||||
* First released 25.05.2010 at http://yacy.net
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Lesser General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Lesser General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public License
|
||||
* along with this program in the file COPYING.LESSER.
|
||||
* If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
|
||||
package net.yacy.cora.protocol;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.commons.httpclient.methods.multipart.Part;
|
||||
|
||||
import de.anomic.crawler.retrieval.HTTPLoader;
|
||||
import de.anomic.http.client.Client;
|
||||
import de.anomic.http.client.RemoteProxyConfig;
|
||||
import de.anomic.http.server.HeaderFramework;
|
||||
import de.anomic.http.server.RequestHeader;
|
||||
import de.anomic.http.server.ResponseContainer;
|
||||
|
||||
public class HttpConnector {
|
||||
|
||||
/**
|
||||
* send data to the server named by vhost
|
||||
*
|
||||
* @param address address of the server
|
||||
* @param vhost name of the server at address which should respond
|
||||
* @param post data to send (name-value-pairs)
|
||||
* @param timeout in milliseconds
|
||||
* @return response body
|
||||
* @throws IOException
|
||||
*/
|
||||
public static byte[] wput(final String url, final String vhost, final List<Part> post, final int timeout) throws IOException {
|
||||
return wput(url, vhost, post, timeout, false);
|
||||
}
|
||||
|
||||
/**
|
||||
* send data to the server named by vhost
|
||||
*
|
||||
* @param address address of the server
|
||||
* @param vhost name of the server at address which should respond
|
||||
* @param post data to send (name-value-pairs)
|
||||
* @param timeout in milliseconds
|
||||
* @param gzipBody send with content gzip encoded
|
||||
* @return response body
|
||||
* @throws IOException
|
||||
*/
|
||||
public static byte[] wput(final String url, final String vhost, final List<Part> post, final int timeout, final boolean gzipBody) throws IOException {
|
||||
final RequestHeader header = new RequestHeader();
|
||||
header.put(HeaderFramework.USER_AGENT, HTTPLoader.yacyUserAgent);
|
||||
header.put(HeaderFramework.HOST, vhost);
|
||||
final Client client = new Client(timeout, header);
|
||||
client.setProxy(proxyConfig());
|
||||
|
||||
ResponseContainer res = null;
|
||||
byte[] content = null;
|
||||
try {
|
||||
// send request/data
|
||||
res = client.POST(url, post, gzipBody);
|
||||
content = res.getData();
|
||||
} finally {
|
||||
if(res != null) {
|
||||
// release connection
|
||||
res.closeStream();
|
||||
}
|
||||
}
|
||||
return content;
|
||||
}
|
||||
|
||||
|
||||
private static final RemoteProxyConfig proxyConfig() {
|
||||
final RemoteProxyConfig p = RemoteProxyConfig.getRemoteProxyConfig();
|
||||
return ((p != null) && (p.useProxy()) && (p.useProxy4Yacy())) ? p : null;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,145 @@
|
||||
/**
|
||||
* Search
|
||||
* Copyright 2010 by Michael Peter Christen
|
||||
* First released 25.05.2010 at http://yacy.net
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Lesser General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Lesser General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public License
|
||||
* along with this program in the file COPYING.LESSER.
|
||||
* If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
|
||||
package net.yacy.cora.services;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.MalformedURLException;
|
||||
import java.nio.charset.Charset;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.BlockingQueue;
|
||||
import java.util.concurrent.LinkedBlockingQueue;
|
||||
|
||||
import net.yacy.cora.document.MultiProtocolURI;
|
||||
import net.yacy.cora.document.RSSFeed;
|
||||
import net.yacy.cora.document.RSSMessage;
|
||||
import net.yacy.cora.document.RSSReader;
|
||||
import net.yacy.cora.protocol.HttpConnector;
|
||||
|
||||
import org.apache.commons.httpclient.methods.multipart.Part;
|
||||
import org.apache.commons.httpclient.methods.multipart.StringPart;
|
||||
|
||||
public class Search {
|
||||
|
||||
public static BlockingQueue<RSSMessage> search(String rssSearchServiceURL, String query, boolean verify, boolean global, long timeout, int maximumRecords) {
|
||||
BlockingQueue<RSSMessage> queue = new LinkedBlockingQueue<RSSMessage>();
|
||||
searchJob job = new searchJob(rssSearchServiceURL, query, verify, global, timeout, maximumRecords, queue);
|
||||
job.start();
|
||||
return queue;
|
||||
}
|
||||
|
||||
private final static int recordsPerSession = 10;
|
||||
|
||||
public static class searchJob extends Thread {
|
||||
|
||||
String urlBase, query;
|
||||
boolean verify, global;
|
||||
long timeout;
|
||||
int startRecord, maximumRecords;
|
||||
BlockingQueue<RSSMessage> queue;
|
||||
|
||||
public searchJob(String urlBase, String query, boolean verify, boolean global, long timeout, int maximumRecords, BlockingQueue<RSSMessage> queue) {
|
||||
this.urlBase = urlBase;
|
||||
this.query = query;
|
||||
this.verify = verify;
|
||||
this.global = global;
|
||||
this.timeout = timeout;
|
||||
this.startRecord = 0;
|
||||
this.maximumRecords = maximumRecords;
|
||||
this.queue = queue;
|
||||
}
|
||||
|
||||
public void run() {
|
||||
RSSMessage message;
|
||||
mainloop: while (timeout > 0 && maximumRecords > 0) {
|
||||
long st = System.currentTimeMillis();
|
||||
RSSFeed feed;
|
||||
try {
|
||||
feed = search(urlBase, query, verify, global, timeout, startRecord, recordsPerSession);
|
||||
} catch (IOException e1) {
|
||||
break mainloop;
|
||||
}
|
||||
if (feed == null || feed.isEmpty()) break mainloop;
|
||||
maximumRecords -= feed.size();
|
||||
innerloop: while (!feed.isEmpty()) {
|
||||
message = feed.pollMessage();
|
||||
if (message == null) break innerloop;
|
||||
try {
|
||||
queue.put(message);
|
||||
} catch (InterruptedException e) {
|
||||
break innerloop;
|
||||
}
|
||||
}
|
||||
startRecord += recordsPerSession;
|
||||
timeout -= System.currentTimeMillis() - st;
|
||||
}
|
||||
try { queue.put(RSSMessage.POISON); } catch (InterruptedException e) {}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* send a query to a yacy public search interface
|
||||
* @param rssSearchServiceURL the target url base (everything before the ? that follows the SRU request syntax properties). can null, then the local peer is used
|
||||
* @param query the query as string
|
||||
* @param startRecord number of first record
|
||||
* @param maximumRecords maximum number of records
|
||||
* @param verify if true, result entries are verified using the snippet fetch (slow); if false simply the result is returned
|
||||
* @param global if true also search results from other peers are included
|
||||
* @param timeout milliseconds that are waited at maximum for a search result
|
||||
* @return
|
||||
*/
|
||||
public static RSSFeed search(String rssSearchServiceURL, String query, boolean verify, boolean global, long timeout, int startRecord, int maximumRecords) throws IOException {
|
||||
MultiProtocolURI uri = null;
|
||||
try {
|
||||
uri = new MultiProtocolURI(rssSearchServiceURL);
|
||||
} catch (MalformedURLException e) {
|
||||
throw new IOException("cora.Search failed asking peer '" + rssSearchServiceURL + "': bad url, " + e.getMessage());
|
||||
}
|
||||
|
||||
// prepare request
|
||||
final List<Part> post = new ArrayList<Part>();
|
||||
post.add(new StringPart("query", query, Charset.defaultCharset().name()));
|
||||
post.add(new StringPart("startRecord", Integer.toString(startRecord), Charset.defaultCharset().name()));
|
||||
post.add(new StringPart("maximumRecords", Long.toString(maximumRecords), Charset.defaultCharset().name()));
|
||||
post.add(new StringPart("verify", verify ? "true" : "false", Charset.defaultCharset().name()));
|
||||
post.add(new StringPart("resource", global ? "global" : "local", Charset.defaultCharset().name()));
|
||||
|
||||
// send request
|
||||
try {
|
||||
final byte[] result = HttpConnector.wput(rssSearchServiceURL, uri.getHost(), post, (int) timeout);
|
||||
//String debug = new String(result); System.out.println("*** DEBUG: " + debug);
|
||||
final RSSReader reader = RSSReader.parse(result);
|
||||
if (reader == null) {
|
||||
throw new IOException("cora.Search failed asking peer '" + uri.getHost() + "': probably bad response from remote peer (1), reader == null");
|
||||
}
|
||||
final RSSFeed feed = reader.getFeed();
|
||||
if (feed == null) {
|
||||
// case where the rss reader does not understand the content
|
||||
throw new IOException("cora.Search failed asking peer '" + uri.getHost() + "': probably bad response from remote peer (2)");
|
||||
}
|
||||
return feed;
|
||||
} catch (final IOException e) {
|
||||
throw new IOException("cora.Search error asking peer '" + uri.getHost() + "':" + e.toString());
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
Loading…
Reference in new issue