boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

new crawl/proxy/cache design + fixes

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@18 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 20 years ago
parent e7d055b98e
commit c0807abd33
8 changed files with 273 additions and 386 deletions
Show Stats Download Patch File Download Diff File

4
source/de/anomic/htmlFilter/htmlFilterContentTransformer.java
Unescape View File

@ -65,7 +65,7 @@ public class htmlFilterContentTransformer extends htmlFilterAbstractTransformer
}
public void init(String initarg) {
System.out.println("Transformer init: " + initarg);
//System.out.println("Transformer init: " + initarg);
if (bluelist == null) {
// here, the initarg is used to load a list of bluelisted words
bluelist = new Vector();
@ -79,7 +79,7 @@ public class htmlFilterContentTransformer extends htmlFilterAbstractTransformer
r.close();
} catch (Exception e) {
}
if (bluelist.size() == 0) System.out.println("BLUELIST is empty");
//if (bluelist.size() == 0) System.out.println("BLUELIST is empty");
}
}

7
source/de/anomic/http/httpc.java
Unescape View File

@ -315,10 +315,9 @@ public class httpc {
}
public byte[] writeContent(OutputStream procOS) throws IOException {
ByteArrayOutputStream bos = new ByteArrayOutputStream();
writeContentX(procOS, bos);
bos.flush();
return bos.toByteArray();
serverByteBuffer sbb = new serverByteBuffer();
writeContentX(procOS, sbb);
return sbb.getBytes();
}
public void writeContent(OutputStream procOS, File file) throws IOException {

201
source/de/anomic/http/httpdProxyHandler.java
Unescape View File

@ -387,13 +387,11 @@ public class httpdProxyHandler extends httpdAbstractHandler implements httpdHand
// in two of these cases we trigger a scheduler to handle newly arrived files:
// case 1 and case 3
plasmaHTCache.Entry hpc;
if (cacheExists) {
// we respond on the request by using the cache
hpc = cacheManager.newEntry(requestDate, 0, url, requestHeader, "200 OK", cachedResponseHeader, null, switchboard.defaultProxyProfile);
if (hpc.shallUseCache()) {
// the cache is fresh
if ((cacheExists) &&
((hpc = cacheManager.newEntry(requestDate, 0, url, requestHeader, "200 OK",
cachedResponseHeader, null,
switchboard.defaultProxyProfile)).shallUseCache())) {
// we respond on the request by using the cache, the cache is fresh
try {
// replace date field in old header by actual date, this is according to RFC
@ -443,12 +441,17 @@ public class httpdProxyHandler extends httpdAbstractHandler implements httpdHand
// we do nothing here
respondError(respond, "111 socket error: " + e.getMessage(), 1, url.toString());
}
} else {
// the cache is (supposed to be) stale
respond.flush();
return;
}
// the cache does either not exist or is (supposed to be) stale
long sizeBeforeDelete = -1;
if (cacheExists) {
// delete the cache
long sizeBeforeDelete = cacheFile.length();
sizeBeforeDelete = cacheFile.length();
cacheFile.delete();
}
// take a new file from the server
httpc remote = null;
@ -470,12 +473,14 @@ public class httpdProxyHandler extends httpdAbstractHandler implements httpdHand
// reserver cache entry
hpc = cacheManager.newEntry(requestDate, 0, url, requestHeader, res.status, res.responseHeader, null, switchboard.defaultProxyProfile);
// make a scraper and transformer
// handle file types
if (((ext == null) || (!(switchboard.extensionBlack.contains(ext)))) &&
(httpd.isTextMime(res.responseHeader.mime(), switchboard.mimeWhite))) {
if (transformer.isIdentityTransformer()) {
hfos = hpc.getContentOutputStream();
// no transformation, only passthrough
hfos = respond;
} else {
// make a scraper and transformer
scraper = new htmlFilterContentScraper(url);
hfos = new htmlFilterOutputStream(respond, scraper, transformer, (ext.length() == 0));
if (((htmlFilterOutputStream) hfos).binarySuspect()) {
@ -501,42 +506,41 @@ public class httpdProxyHandler extends httpdAbstractHandler implements httpdHand
if ((storeError = hpc.shallStoreCache()) == null) {
// we write a new cache entry
if ((contentLength > 0) && // known
(contentLength < 1048576)) // 1 MB
{
(contentLength < 1048576)) {// 1 MB
// ok, we don't write actually into a file, only to RAM, and schedule writing the file.
byte[] cacheArray;
if (transformer.isIdentityTransformer()) {
res.writeContentX(hfos, respond);
cacheArray = hpc.getContentBytes();
} else {
cacheArray = res.writeContent(hfos);
}
if (hfos instanceof htmlFilterOutputStream) ((htmlFilterOutputStream) hfos).finalize();
if (sizeBeforeDelete == -1) {
// totally fresh file
hpc.status = plasmaHTCache.CACHE_FILL; // it's an insert
cacheManager.stackProcess(hpc, cacheArray);
} else if (sizeBeforeDelete == cacheArray.length) {
// before we came here we deleted a cache entry
if (sizeBeforeDelete == cacheArray.length) {
cacheArray = null;
hpc.status = plasmaHTCache.CACHE_STALE_RELOAD_BAD;
cacheManager.stackProcess(hpc); // unnecessary update
} else {
// before we came here we deleted a cache entry
hpc.status = plasmaHTCache.CACHE_STALE_RELOAD_GOOD;
cacheManager.stackProcess(hpc, cacheArray); // necessary update, write response header to cache
}
} else {
// the file is too big to cache it in the ram, write to file
// the file is too big to cache it in the ram, write to file right here
cacheFile.getParentFile().mkdirs();
if (transformer.isIdentityTransformer()) {
res.writeContent(respond, cacheFile);
if (contentLength < 10485760) { // 10 mb
serverFileUtils.copy(cacheFile, hfos);
} // else hfos is empty and that means: no work afterwards with it
} else {
res.writeContent(hfos, cacheFile);
}
if (hfos instanceof htmlFilterOutputStream) ((htmlFilterOutputStream) hfos).finalize();
if (sizeBeforeDelete == -1) {
// totally fresh file
hpc.status = plasmaHTCache.CACHE_FILL; // it's an insert
cacheManager.stackProcess(hpc);
} else if (sizeBeforeDelete == cacheFile.length()) {
// before we came here we deleted a cache entry
if (sizeBeforeDelete == cacheFile.length()) {
hpc.status = plasmaHTCache.CACHE_STALE_RELOAD_BAD;
cacheManager.stackProcess(hpc); // unnecessary update
} else {
// before we came here we deleted a cache entry
hpc.status = plasmaHTCache.CACHE_STALE_RELOAD_GOOD;
cacheManager.stackProcess(hpc); // necessary update, write response header to cache
}
@ -546,10 +550,16 @@ public class httpdProxyHandler extends httpdAbstractHandler implements httpdHand
log.logDebug(cacheFile.toString() + " not cached: " + storeError);
res.writeContent(hfos, null);
if (hfos instanceof htmlFilterOutputStream) ((htmlFilterOutputStream) hfos).finalize();
if (sizeBeforeDelete == -1) {
// no old file and no load. just data passing
hpc.status = plasmaHTCache.CACHE_PASSING;
cacheManager.stackProcess(hpc);
} else {
// before we came here we deleted a cache entry
hpc.status = plasmaHTCache.CACHE_STALE_NO_RELOAD;
cacheManager.stackProcess(hpc);
}
}
} catch (SocketException e) {
// this may happen if the client suddenly closes its connection
// maybe the user has stopped loading
@ -557,6 +567,7 @@ public class httpdProxyHandler extends httpdAbstractHandler implements httpdHand
// but we clean the cache also, since it may be only partial
// and most possible corrupted
if (cacheFile.exists()) cacheFile.delete();
respondHeader(respond,"404 client unexpectedly closed connection", new httpHeader(null));
}
remote.close();
} catch (Exception e) {
@ -576,138 +587,6 @@ public class httpdProxyHandler extends httpdAbstractHandler implements httpdHand
}
} catch (Exception ee) {}
}
}
} else {
// we take a new file from the net and respond with that
try {
// open the connection
//httpc remote = newhttpc(host, port, timeout);
httpc remote;
if (yAddress == null) {
remote = newhttpc(host, port, timeout);
} else {
remote = newhttpc(yAddress, timeout);
}
//System.out.println("HEADER: CLIENT TO PROXY = " + requestHeader.toString()); // DEBUG
// send request
httpc.response res = remote.GET(remotePath, requestHeader);
long contentLength = res.responseHeader.contentLength();
// reserve cache entry
hpc = cacheManager.newEntry(requestDate, 0, url, requestHeader, res.status, res.responseHeader, null, switchboard.defaultProxyProfile);
// make a scraper and transformer
if (((ext == null) || (!(switchboard.extensionBlack.contains(ext)))) &&
(httpd.isTextMime(res.responseHeader.mime(), switchboard.mimeWhite))) {
if (transformer.isIdentityTransformer()) {
hfos = hpc.getContentOutputStream();
} else {
scraper = new htmlFilterContentScraper(url);
hfos = new htmlFilterOutputStream(respond, scraper, transformer, (ext.length() == 0));
if (((htmlFilterOutputStream) hfos).binarySuspect()) {
scraper = null; // forget it, may be rubbish
log.logDebug("Content of " + url + " is probably binary. deleted scraper.");
}
hpc.scraper = scraper;
}
} else {
log.logDebug("Resource " + url + " has wrong extension (" + ext + ") or wrong mime-type (" + res.responseHeader.mime() + "). not scraped");
scraper = null;
hfos = respond;
hpc.scraper = scraper;
}
// handle incoming cookies
handleIncomingCookies(res.responseHeader, host, ip);
// request has been placed and result has been returned. work off response
try {
//System.out.println("HEADER: SERVER TO PROXY = [" + res.status + "] " + ((httpHeader) res.responseHeader).toString()); // DEBUG
respondHeader(respond, res.status, res.responseHeader);
String storeError;
if ((storeError = hpc.shallStoreCache()) == null) {
// we write a new cache entry
if ((contentLength > 0) && (contentLength < 1048576)) {
// write to buffer
byte[] cacheArray;
if (transformer.isIdentityTransformer()) {
res.writeContentX(hfos, respond);
cacheArray = hpc.getContentBytes();
} else {
cacheArray = res.writeContent(hfos);
}
if (hfos instanceof htmlFilterOutputStream) ((htmlFilterOutputStream) hfos).finalize();
// enQueue new entry with response header and file as byte[]
hpc.status = plasmaHTCache.CACHE_FILL;
cacheManager.stackProcess(hpc, cacheArray);
} else try {
// write to file system directly
cacheFile.getParentFile().mkdirs();
if (transformer.isIdentityTransformer()) {
res.writeContent(respond, cacheFile);
if (contentLength < 10485760) { // 10 mb
serverFileUtils.copy(cacheFile, hfos);
} // else hfos is empty and that means: no work afterwards with it
} else {
res.writeContent(hfos, cacheFile);
}
if (hfos instanceof htmlFilterOutputStream) ((htmlFilterOutputStream) hfos).finalize();
// enQueue new entry with response header
hpc.status = plasmaHTCache.CACHE_FILL;
cacheManager.stackProcess(hpc);
} catch (FileNotFoundException e) {
// this may happen if there are no write rights whatsoever
// (do nothing)
/*
Exception occurred:
java.io.FileNotFoundException:
/opt/yacy_pre_v0.314_20041219/DATA/HTCACHE/www.spiegel.de/fotostrecke/0,5538,PB64-SUQ9NDYwNyZucj0z,00.html
(Permission denied)
*/
}
} else {
// no caching
//System.out.println("DEBUG: " + res.status + " " + cacheFile.toString()); // debug
log.logDebug(cacheFile.toString() + " not cached: " + storeError);
res.writeContent(hfos, null);
if (hfos instanceof htmlFilterOutputStream) ((htmlFilterOutputStream) hfos).finalize();
// no old file and no load. just data passing
hpc.status = plasmaHTCache.CACHE_PASSING;
cacheManager.stackProcess(hpc);
}
} catch (SocketException e) {
// this may happen if the client suddenly closes its connection
// maybe the user has stopped loading
// in that case, we are not responsible and just forget it
// but we clean the cache also, since it may be only partial
// and most possible corrupted
if (cacheFile.exists()) cacheFile.delete();
respondHeader(respond,"404 client unexpectedly closed connection", new httpHeader(null));
}
remote.close();
} catch (Exception e) {
// this may happen if the targeted host does not exist or anything with the
// remote server was wrong.
// in any case, sending a 404 is appropriate
try {
if ((e.toString().indexOf("unknown host")) > 0) {
respondHeader(respond,"404 unknown host", new httpHeader(null));
} else {
respondHeader(respond,"404 resource not available (generic exception: " + e.toString() + ")", new httpHeader(null));
//respond.write(("Exception occurred:\r\n").getBytes());
//respond.write((e.toString() + "\r\n").getBytes());
//respond.write(("[TRACE: ").getBytes());
//e.printStackTrace(new PrintStream(respond));
//respond.write(("]\r\n").getBytes());
/* http://www.geocrawler.com/archives/3/201/1999/8/50/2505805/
> java.net.ConnectException: Connection refused
*/
e.printStackTrace();
}
} catch (Exception ee) {}
}
}
respond.flush();
}

6
source/de/anomic/plasma/plasmaCrawlLoader.java
Unescape View File

@ -212,12 +212,14 @@ public class plasmaCrawlLoader {
} else if ((profile.storeHTCache()) && ((error = htCache.shallStoreCache()) == null)) {
// we write the new cache entry to file system directly
cacheFile.getParentFile().mkdirs();
res.writeContent(htCache.getContentOutputStream(), cacheFile); // writes in content scraper and cache file
FileOutputStream fos = new FileOutputStream(cacheFile);
htCache.cacheArray = res.writeContent(fos); // writes in cacheArray and cache file
fos.close();
htCache.status = plasmaHTCache.CACHE_FILL;
} else {
if (error != null) log.logDebug("CRAWLER NOT STORED RESOURCE " + url.toString() + ": " + error);
// anyway, the content still lives in the content scraper
res.writeContent(htCache.getContentOutputStream(), null); // writes only into content scraper
htCache.cacheArray = res.writeContent(null); // writes only into cacheArray
htCache.status = plasmaHTCache.CACHE_PASSING;
}
// enQueue new entry with response header

9
source/de/anomic/plasma/plasmaHTCache.java
Unescape View File

@ -452,7 +452,6 @@ public class plasmaHTCache {
public String language;
public plasmaCrawlProfile.entry profile;
private String initiator;
public ByteArrayOutputStream content;
public htmlFilterContentScraper scraper;
@ -479,7 +478,6 @@ public class plasmaHTCache {
this.requestHeader = requestHeader;
this.responseStatus = responseStatus;
this.responseHeader = responseHeader;
this.content = new ByteArrayOutputStream();
this.profile = profile;
this.initiator = (initiator == null) ? null : ((initiator.length() == 0) ? null: initiator);
@ -507,13 +505,6 @@ public class plasmaHTCache {
this.scraper = null;
}
public OutputStream getContentOutputStream() {
return (OutputStream) content;
}
public byte[] getContentBytes() {
try { content.flush(); } catch (IOException e) {}
return content.toByteArray();
}
public String initiator() {
return initiator;
}

5
source/de/anomic/plasma/plasmaSwitchboard.java
Unescape View File

@ -469,8 +469,7 @@ public class plasmaSwitchboard extends serverAbstractSwitch implements serverSwi
private synchronized void processResourceStack(plasmaHTCache.Entry entry) {
// work off one stack entry with a fresh resource (scraped web page)
byte[] content;
if (((content = entry.getContentBytes()).length > 0) || (entry.scraper != null)) try {
if ((entry.cacheArray != null) || (entry.scraper != null)) try {
// we must distinguish the following cases: resource-load was initiated by
// 1) global crawling: the index is extern, not here (not possible here)
// 2) result of search queries, some indexes are here (not possible here)
@ -502,7 +501,7 @@ public class plasmaSwitchboard extends serverAbstractSwitch implements serverSwi
document = parser.transformScraper(entry.url, entry.responseHeader.mime(), entry.scraper);
} else {
log.logDebug("(Parser) '" + entry.urlString + "' is not parsed, parsing now");
document = parser.parseSource(entry.url, entry.responseHeader.mime(), content);
document = parser.parseSource(entry.url, entry.responseHeader.mime(), entry.cacheArray);
}
// put anchors on crawl stack

29
source/de/anomic/server/serverByteBuffer.java
Unescape View File

@ -43,7 +43,7 @@ package de.anomic.server;
import java.io.*;
import java.util.*;
public class serverByteBuffer {
public class serverByteBuffer extends OutputStream {
public static final byte singlequote = (byte) 39;
public static final byte doublequote = (byte) 34;
@ -119,20 +119,37 @@ public class serverByteBuffer {
offset = 0;
}
public serverByteBuffer append(byte b) {
public void write(int b) {
write((byte) (b & 0xff));
}
public void write(byte b) {
if (offset + length + 1 > buffer.length) grow();
buffer[offset + length++] = b;
return this;
}
public serverByteBuffer append(byte[] bb) {
return append(bb, 0, bb.length);
public void write(byte[] bb) {
write(bb, 0, bb.length);
}
public serverByteBuffer append(byte[] bb, int of, int le) {
public void write(byte[] bb, int of, int le) {
while (offset + length + le > buffer.length) grow();
System.arraycopy(bb, of, buffer, offset + length, le);
length += le;
}
public serverByteBuffer append(byte b) {
write(b);
return this;
}
public serverByteBuffer append(byte[] bb) {
write(bb);
return this;
}
public serverByteBuffer append(byte[] bb, int of, int le) {
write(bb, of, le);
return this;
}

Write Preview
Loading…
Cancel
Save