git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@218 6c8d7289-2bf4-0310-a012-ef5d649a1542pull/1/head
orbiter
20 years ago
parent
9244b6ad6f
commit
ca3b4ccaf4
@ -0,0 +1,234 @@
|
||||
// plasmaSnippetCache.java
|
||||
// -----------------------
|
||||
// part of YaCy
|
||||
// (C) by Michael Peter Christen; mc@anomic.de
|
||||
// first published on http://www.anomic.de
|
||||
// Frankfurt, Germany, 2005
|
||||
// last major change: 07.06.2005
|
||||
//
|
||||
// This program is free software; you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation; either version 2 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with this program; if not, write to the Free Software
|
||||
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
//
|
||||
// Using this software in any meaning (reading, learning, copying, compiling,
|
||||
// running) means that you agree that the Author(s) is (are) not responsible
|
||||
// for cost, loss of data or any harm that may be caused directly or indirectly
|
||||
// by usage of this softare or this documentation. The usage of this software
|
||||
// is on your own risk. The installation and usage (starting/running) of this
|
||||
// software may allow other people or application to access your computer and
|
||||
// any attached devices and is highly dependent on the configuration of the
|
||||
// software which must be done by the user of the software; the author(s) is
|
||||
// (are) also not responsible for proper configuration and usage of the
|
||||
// software, even if provoked by documentation provided together with
|
||||
// the software.
|
||||
//
|
||||
// Any changes to this file according to the GPL as documented in the file
|
||||
// gpl.txt aside this file in the shipment you received can be done to the
|
||||
// lines that follows this copyright notice here, but changes must not be
|
||||
// done inside the copyright notive above. A re-distribution must contain
|
||||
// the intact and unchanged copyright notice.
|
||||
// Contributions and changes to the program code must be marked as such.
|
||||
|
||||
|
||||
package de.anomic.plasma;
|
||||
|
||||
import java.util.*;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.net.URL;
|
||||
import de.anomic.htmlFilter.htmlFilterContentScraper;
|
||||
import de.anomic.kelondro.kelondroMScoreCluster;
|
||||
import de.anomic.server.serverFileUtils;
|
||||
import de.anomic.server.serverLog;
|
||||
import de.anomic.http.httpHeader;
|
||||
import de.anomic.yacy.yacySearch;
|
||||
|
||||
public class plasmaSnippetCache {
|
||||
|
||||
private static final int maxCache = 500;
|
||||
|
||||
private int snippetsScoreCounter;
|
||||
private kelondroMScoreCluster snippetsScore;
|
||||
private HashMap snippetsCache;
|
||||
private plasmaHTCache cacheManager;
|
||||
private plasmaParser parser;
|
||||
private serverLog log;
|
||||
private String remoteProxyHost;
|
||||
private int remoteProxyPort;
|
||||
private boolean remoteProxyUse;
|
||||
|
||||
public plasmaSnippetCache(plasmaHTCache cacheManager, plasmaParser parser,
|
||||
String remoteProxyHost, int remoteProxyPort, boolean remoteProxyUse,
|
||||
serverLog log) {
|
||||
this.cacheManager = cacheManager;
|
||||
this.parser = parser;
|
||||
this.log = log;
|
||||
this.remoteProxyHost = remoteProxyHost;
|
||||
this.remoteProxyPort = remoteProxyPort;
|
||||
this.remoteProxyUse = remoteProxyUse;
|
||||
this.snippetsScoreCounter = 0;
|
||||
this.snippetsScore = new kelondroMScoreCluster();
|
||||
this.snippetsCache = new HashMap();
|
||||
}
|
||||
|
||||
|
||||
public synchronized void store(String wordhashes, String urlhash, String snippet) {
|
||||
// generate key
|
||||
String key = urlhash + wordhashes;
|
||||
|
||||
// do nothing if snippet is known
|
||||
if (snippetsCache.containsKey(key)) return;
|
||||
|
||||
// learn new snippet
|
||||
snippetsScore.addScore(key, snippetsScoreCounter++);
|
||||
snippetsCache.put(key, snippet);
|
||||
|
||||
// care for counter
|
||||
if (snippetsScoreCounter == java.lang.Integer.MAX_VALUE) {
|
||||
snippetsScoreCounter = 0;
|
||||
snippetsScore = new kelondroMScoreCluster();
|
||||
snippetsCache = new HashMap();
|
||||
}
|
||||
|
||||
// flush cache if cache is full
|
||||
while (snippetsCache.size() > maxCache) {
|
||||
key = (String) snippetsScore.getMinObject();
|
||||
snippetsScore.deleteScore(key);
|
||||
snippetsCache.remove(key);
|
||||
}
|
||||
}
|
||||
|
||||
private String retrieve(String wordhashes, String urlhash) {
|
||||
// generate key
|
||||
String key = urlhash + wordhashes;
|
||||
return (String) snippetsCache.get(key);
|
||||
}
|
||||
|
||||
public String retrieve(java.net.URL url, boolean fetchOnline, Set query, boolean queryAreHashes) {
|
||||
if (query.size() == 0) return null;
|
||||
if (!(queryAreHashes)) query = plasmaSearch.words2hashes(query);
|
||||
String urlhash = plasmaURL.urlHash(url);
|
||||
|
||||
// try to get snippet from snippetCache
|
||||
String wordhashes = yacySearch.set2string(query);
|
||||
String snippet = retrieve(wordhashes, urlhash);
|
||||
if (snippet != null) return snippet;
|
||||
|
||||
// if the snippet is not in the cache, we can try to get it from the htcache
|
||||
plasmaParserDocument document = getDocument(url, fetchOnline);
|
||||
if (document == null) return null;
|
||||
String[] sentences = document.getSentences();
|
||||
//System.out.println("----" + url.toString()); for (int l = 0; l < sentences.length; l++) System.out.println(sentences[l]);
|
||||
if ((sentences == null) || (sentences.length == 0)) return null;
|
||||
|
||||
// we have found a parseable non-empty file: use the lines
|
||||
TreeMap sentencematrix = hashMatrix(sentences);
|
||||
Iterator i = query.iterator();
|
||||
String hash;
|
||||
kelondroMScoreCluster hitTable = new kelondroMScoreCluster();
|
||||
Iterator j;
|
||||
Integer sentencenumber;
|
||||
Map.Entry entry;
|
||||
while (i.hasNext()) {
|
||||
hash = (String) i.next();
|
||||
j = sentencematrix.entrySet().iterator();
|
||||
while (j.hasNext()) {
|
||||
entry = (Map.Entry) j.next();
|
||||
sentencenumber = (Integer) entry.getKey();
|
||||
if (((HashSet) entry.getValue()).contains(hash)) hitTable.addScore(sentencenumber, sentences[sentencenumber.intValue()].length());
|
||||
}
|
||||
}
|
||||
Integer maxLine = (Integer) hitTable.getMaxObject();
|
||||
if (maxLine == null) return null;
|
||||
snippet = sentences[maxLine.intValue()];
|
||||
if (snippet.length() > 140) return null;
|
||||
|
||||
// finally store this snippet in our own cache
|
||||
store(wordhashes, urlhash, snippet);
|
||||
return snippet;
|
||||
}
|
||||
|
||||
private TreeMap hashMatrix(String[] sentences) {
|
||||
TreeMap map = new TreeMap();
|
||||
HashSet set;
|
||||
Enumeration words;
|
||||
for (int i = 0; i < sentences.length; i++) {
|
||||
set = new HashSet();
|
||||
words = plasmaCondenser.wordTokenizer(sentences[i]);
|
||||
while (words.hasMoreElements()) set.add(plasmaWordIndexEntry.word2hash((String) words.nextElement()));
|
||||
map.put(new Integer(i), set);
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
private byte[] getResource(URL url, boolean fetchOnline) {
|
||||
// load the url as resource from the web
|
||||
try {
|
||||
//return httpc.singleGET(url, 5000, null, null, remoteProxyHost, remoteProxyPort);
|
||||
byte[] resource = getResourceFromCache(url);
|
||||
if ((fetchOnline) && (resource == null)) {
|
||||
loadResourceFromWeb(url, 5000);
|
||||
resource = getResourceFromCache(url);
|
||||
}
|
||||
return resource;
|
||||
} catch (IOException e) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private byte[] getResourceFromCache(URL url) {
|
||||
// load the url as resource from the cache
|
||||
String path = htmlFilterContentScraper.urlNormalform(url).substring(6);
|
||||
File cache = cacheManager.cachePath;
|
||||
File f = new File(cache, path);
|
||||
if (f.exists()) try {
|
||||
return serverFileUtils.read(f);
|
||||
} catch (IOException e) {
|
||||
return null;
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private void loadResourceFromWeb(URL url, int socketTimeout) throws IOException {
|
||||
plasmaCrawlWorker.load(
|
||||
url,
|
||||
null,
|
||||
null,
|
||||
0,
|
||||
null,
|
||||
socketTimeout,
|
||||
remoteProxyHost,
|
||||
remoteProxyPort,
|
||||
remoteProxyUse,
|
||||
cacheManager,
|
||||
log);
|
||||
}
|
||||
|
||||
public plasmaParserDocument getDocument(URL url, boolean fetchOnline) {
|
||||
byte[] resource = getResource(url, fetchOnline);
|
||||
if (resource == null) return null;
|
||||
httpHeader header = null;
|
||||
try {
|
||||
header = cacheManager.getCachedResponse(plasmaURL.urlHash(url));
|
||||
} catch (IOException e) {
|
||||
return null;
|
||||
}
|
||||
if (header == null) return null;
|
||||
if (plasmaParser.supportedMimeTypesContains(header.mime())) {
|
||||
return parser.parseSource(url, header.mime(), resource);
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Reference in new issue