boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

fix for more time-related problems in proxy

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@486 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 20 years ago
parent fb6f238d70
commit 91163db52e
6 changed files with 19 additions and 18 deletions
Show Stats Download Patch File Download Diff File

1
source/de/anomic/plasma/plasmaCrawlWorker.java
Unescape View File

@ -55,6 +55,7 @@ import de.anomic.http.httpHeader;
import de.anomic.http.httpc;
import de.anomic.http.httpdProxyHandler;
import de.anomic.server.serverCore;
import de.anomic.server.serverDate;
import de.anomic.server.logging.serverLog;
import de.anomic.server.logging.serverMiniLogFormatter;

25
source/de/anomic/plasma/plasmaHTCache.java
Unescape View File

@ -485,10 +485,10 @@ public final class plasmaHTCache {
System.exit(0);
}
lastModified = new Date();
lastModified = serverDate.correctedGMTDate();
} else {
lastModified = responseHeader.lastModified();
if (lastModified == null) lastModified = new Date(); // does not exist in header
if (lastModified == null) lastModified = serverDate.correctedGMTDate(); // does not exist in header
}
this.doctype = plasmaWordIndexEntry.docType(responseHeader.mime());
if (this.doctype == plasmaWordIndexEntry.DT_UNKNOWN) this.doctype = plasmaWordIndexEntry.docType(url);
@ -598,7 +598,7 @@ public final class plasmaHTCache {
if (date == null) return "stale_no_date_given_in_response";
try {
long ttl = 1000 * Long.parseLong(cacheControl.substring(8)); // milliseconds to live
if ((new Date()).getTime() - date.getTime() > ttl) {
if (serverDate.correctedGMTDate().getTime() - date.getTime() > ttl) {
//System.out.println("***not indexed because cache-control");
return "stale_expired";
}
@ -640,8 +640,8 @@ public final class plasmaHTCache {
if (!(responseHeader.containsKey(httpHeader.LAST_MODIFIED))) return false;
// parse date
Date d1, d2;
d2 = responseHeader.lastModified(); if (d2 == null) d2 = new Date();
d1 = requestHeader.ifModifiedSince(); if (d1 == null) d1 = new Date();
d2 = responseHeader.lastModified(); if (d2 == null) d2 = serverDate.correctedGMTDate();
d1 = requestHeader.ifModifiedSince(); if (d1 == null) d1 = serverDate.correctedGMTDate();
// finally, we shall treat the cache as stale if the modification time is after the if-.. time
if (d2.after(d1)) return false;
}
@ -684,10 +684,7 @@ public final class plasmaHTCache {
// -expires in cached response
// the expires value gives us a very easy hint when the cache is stale
if (expires != null) {
//Date yesterday = new Date((new Date()).getTime() - oneday);
//long now = new GregorianCalendar(TimeZone.getTimeZone("GMT+0")).getTimeInMillis();
//System.out.println("EXPIRES-TEST: expires=" + expires.getTime()) + ", NOW=" + now + ", System.currentTimeMillis=" + System.currentTimeMillis() + ", url=" + url);
System.out.println("EXPIRES-TEST: expires=" + expires + ", NOW=" + serverDate.correctedGMTDate() + ", url=" + url);
//System.out.println("EXPIRES-TEST: expires=" + expires + ", NOW=" + serverDate.correctedGMTDate() + ", url=" + url);
if (expires.before(serverDate.correctedGMTDate())) return false;
}
@ -698,13 +695,13 @@ public final class plasmaHTCache {
// This would be a TTL factor of 100% we want no more than 10% TTL, so that a 10 month old cache
// file may only be treated as fresh for one more month, not more.
if (lastModified != null) {
if (date == null) date = new Date();
if (date == null) date = serverDate.correctedGMTDate();
long age = date.getTime() - lastModified.getTime();
if (age < 0) return false;
// TTL (Time-To-Live) is age/10 = (d2.getTime() - d1.getTime()) / 10
// the actual living-time is new Date().getTime() - d2.getTime()
// therefore the cache is stale, if Date().getTime() - d2.getTime() > age/10
if ((new Date()).getTime() - date.getTime() > age / 10) return false;
// the actual living-time is serverDate.correctedGMTDate().getTime() - d2.getTime()
// therefore the cache is stale, if serverDate.correctedGMTDate().getTime() - d2.getTime() > age/10
if (serverDate.correctedGMTDate().getTime() - date.getTime() > age / 10) return false;
}
// -cache-control in cached response
@ -723,7 +720,7 @@ public final class plasmaHTCache {
if (date == null) return false;
try {
long ttl = 1000 * Long.parseLong(cacheControl.substring(8)); // milliseconds to live
if ((new Date()).getTime() - date.getTime() > ttl) {
if (serverDate.correctedGMTDate().getTime() - date.getTime() > ttl) {
return false;
}
} catch (Exception e) {

1
source/de/anomic/plasma/plasmaSwitchboard.java
Unescape View File

@ -130,6 +130,7 @@ import de.anomic.kelondro.kelondroTables;
import de.anomic.server.serverAbstractSwitch;
import de.anomic.server.serverCodings;
import de.anomic.server.serverCore;
import de.anomic.server.serverDate;
import de.anomic.server.serverInstantThread;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSemaphore;

8
source/de/anomic/plasma/plasmaSwitchboardQueue.java
Unescape View File

@ -51,6 +51,7 @@ import de.anomic.htmlFilter.htmlFilterContentScraper;
import de.anomic.http.httpHeader;
import de.anomic.kelondro.kelondroStack;
import de.anomic.server.serverCodings;
import de.anomic.server.serverDate;
import de.anomic.yacy.yacySeedDB;
public class plasmaSwitchboardQueue {
@ -287,7 +288,7 @@ public class plasmaSwitchboardQueue {
if ((ifModifiedSince != null) && (responseHeader().containsKey(httpHeader.LAST_MODIFIED))) {
// parse date
Date d = responseHeader().lastModified();
if (d == null) d = new Date();
if (d == null) d = serverDate.correctedGMTDate();
// finally, we shall treat the cache as stale if the modification time is after the if-.. time
if (d.after(ifModifiedSince)) {
//System.out.println("***not indexed because if-modified-since");
@ -316,8 +317,7 @@ public class plasmaSwitchboardQueue {
// sometimes, the expires date is set to the past to prevent that a page is cached
// we use that information to see if we should index it
if (expires != null) {
Date yesterday = new Date((new Date()).getTime() - plasmaHTCache.oneday);
if (expires.before(yesterday)) return "Stale_(Expired)";
if (expires.before(serverDate.correctedGMTDate())) return "Stale_(Expired)";
}
// -lastModified in cached response
@ -345,7 +345,7 @@ public class plasmaSwitchboardQueue {
if (date == null) return "Stale_(no_date_given_in_response)";
try {
long ttl = 1000 * Long.parseLong(cacheControl.substring(8)); // milliseconds to live
if ((new Date()).getTime() - date.getTime() > ttl) {
if (serverDate.correctedGMTDate().getTime() - date.getTime() > ttl) {
//System.out.println("***not indexed because cache-control");
return "Stale_(expired_by_cache-control)";
}

1
source/de/anomic/yacy/yacyNewsRecord.java
Unescape View File

@ -42,6 +42,7 @@
package de.anomic.yacy;
import de.anomic.server.serverCodings;
import de.anomic.server.serverDate;
import java.util.Date;
import java.util.Map;

1
source/de/anomic/yacy/yacyPeerActions.java
Unescape View File

@ -56,6 +56,7 @@ import de.anomic.http.httpHeader;
import de.anomic.http.httpc;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverCore;
import de.anomic.server.serverDate;
import de.anomic.server.serverSystem;
import de.anomic.tools.disorderSet;

Write Preview
Loading…
Cancel
Save