From 28683530cdb24f5b93c87e50caa95e66bf8e9c1a Mon Sep 17 00:00:00 2001 From: Michael Peter Christen Date: Fri, 19 Dec 2014 17:37:58 +0100 Subject: [PATCH] fixes to usage of no-cache: use and recognize also the no-store directive --- htroot/NetworkPicture.java | 2 +- .../net/yacy/crawler/retrieval/Response.java | 22 +++++++++---------- .../yacy/http/servlets/SolrSelectServlet.java | 2 +- .../http/servlets/YaCyDefaultServlet.java | 6 +++-- source/net/yacy/peers/SeedDB.java | 2 +- source/net/yacy/search/Switchboard.java | 2 +- source/net/yacy/server/http/HTTPDemon.java | 2 +- 7 files changed, 20 insertions(+), 18 deletions(-) diff --git a/htroot/NetworkPicture.java b/htroot/NetworkPicture.java index b86ea4d22..51b8b628d 100644 --- a/htroot/NetworkPicture.java +++ b/htroot/NetworkPicture.java @@ -135,7 +135,7 @@ public class NetworkPicture { env.getConfig(SwitchboardConstants.NETWORK_NAME, "unspecified"), env.getConfig("network.unit.description", "unspecified"), Long.parseLong(bgcolor, 16), - cyc), "png"); + cyc), "png", false); lastAccessSeconds = System.currentTimeMillis() / 1000; sync.release(); diff --git a/source/net/yacy/crawler/retrieval/Response.java b/source/net/yacy/crawler/retrieval/Response.java index 0680a1fed..67039016a 100644 --- a/source/net/yacy/crawler/retrieval/Response.java +++ b/source/net/yacy/crawler/retrieval/Response.java @@ -354,7 +354,7 @@ public class Response { // if we have a pragma non-cache, we don't cache. usually if this is wanted from // the server, it makes sense String cacheControl = this.responseHeader.get(HeaderFramework.PRAGMA); - if (cacheControl != null && cacheControl.trim().toUpperCase().equals("NO-CACHE")) { return "controlled_no_cache"; } + if (cacheControl != null && cacheControl.trim().toUpperCase().contains("NO-CACHE")) { return "controlled_no_cache"; } // -expires in response // we do not care about expires, because at the time this is called the data is @@ -443,12 +443,12 @@ public class Response { // if the client requests a un-cached copy of the resource ... cacheControl = this.requestHeader.get(HeaderFramework.PRAGMA); - if (cacheControl != null && cacheControl.trim().toUpperCase().equals("NO-CACHE")) { return false; } + if (cacheControl != null && cacheControl.trim().toUpperCase().contains("NO-CACHE")) { return false; } cacheControl = this.requestHeader.get(HeaderFramework.CACHE_CONTROL); if (cacheControl != null) { cacheControl = cacheControl.trim().toUpperCase(); - if (cacheControl.startsWith("NO-CACHE") || cacheControl.startsWith("MAX-AGE=0")) { return false; } + if (cacheControl.contains("NO-CACHE") || cacheControl.startsWith("MAX-AGE=0")) { return false; } } // -if-modified-since in request @@ -487,7 +487,7 @@ public class Response { // because they cannot exist since they are not written to the cache. // So this IF should always fail.. cacheControl = this.responseHeader.get(HeaderFramework.PRAGMA); - if (cacheControl != null && cacheControl.trim().toUpperCase().equals("NO-CACHE")) { return false; } + if (cacheControl != null && cacheControl.trim().toUpperCase().contains("NO-CACHE")) { return false; } // see for documentation also: // http://www.web-caching.com/cacheability.html @@ -529,9 +529,9 @@ public class Response { // the cache-control has many value options. if (cacheControl != null) { cacheControl = cacheControl.trim().toUpperCase(); - if (cacheControl.startsWith("PRIVATE") || - cacheControl.startsWith("NO-CACHE") || - cacheControl.startsWith("NO-STORE")) { + if (cacheControl.contains("PRIVATE") || + cacheControl.contains("NO-CACHE") || + cacheControl.contains("NO-STORE")) { // easy case return false; // } else if (cacheControl.startsWith("PUBLIC")) { @@ -640,7 +640,7 @@ public class Response { // -pragma in cached response if (this.responseHeader.containsKey(HeaderFramework.PRAGMA) && - (this.responseHeader.get(HeaderFramework.PRAGMA)).toUpperCase().equals("NO-CACHE")) { + (this.responseHeader.get(HeaderFramework.PRAGMA)).toUpperCase().contains("NO-CACHE")) { return "Denied_(pragma_no_cache)"; } @@ -672,9 +672,9 @@ public class Response { "private", "no-cache", "no-store" -- cannot be indexed "max-age=" -- stale/fresh dependent on date */ - if (cacheControl.startsWith("PRIVATE") || - cacheControl.startsWith("NO-CACHE") || - cacheControl.startsWith("NO-STORE")) { + if (cacheControl.contains("PRIVATE") || + cacheControl.contains("NO-CACHE") || + cacheControl.contains("NO-STORE")) { // easy case return "Stale_(denied_by_cache-control=" + cacheControl + ")"; // } else if (cacheControl.startsWith("PUBLIC")) { diff --git a/source/net/yacy/http/servlets/SolrSelectServlet.java b/source/net/yacy/http/servlets/SolrSelectServlet.java index 0d71ec82a..03ec2f35b 100644 --- a/source/net/yacy/http/servlets/SolrSelectServlet.java +++ b/source/net/yacy/http/servlets/SolrSelectServlet.java @@ -241,7 +241,7 @@ public class SolrSelectServlet extends HttpServlet { rsp = ((EmbeddedSolrConnector) connector).query(req); // prepare response - hresponse.setHeader("Cache-Control", "no-cache"); + hresponse.setHeader("Cache-Control", "no-cache, no-store"); HttpCacheHeaderUtil.checkHttpCachingVeto(rsp, hresponse, reqMethod); // check error diff --git a/source/net/yacy/http/servlets/YaCyDefaultServlet.java b/source/net/yacy/http/servlets/YaCyDefaultServlet.java index 4cbbd0973..5ed0bdf63 100644 --- a/source/net/yacy/http/servlets/YaCyDefaultServlet.java +++ b/source/net/yacy/http/servlets/YaCyDefaultServlet.java @@ -480,7 +480,7 @@ public class YaCyDefaultServlet extends HttpServlet { byte[] data = dir.getBytes("UTF-8"); response.setContentType(TEXT_HTML_UTF_8.asString()); response.setContentLength(data.length); - response.setHeader(HeaderFramework.CACHE_CONTROL, "no-cache"); + response.setHeader(HeaderFramework.CACHE_CONTROL, "no-cache, no-store"); response.setDateHeader(HeaderFramework.EXPIRES, System.currentTimeMillis() + 10000); // consider that directories are not modified that often response.setDateHeader(HeaderFramework.LAST_MODIFIED, resource.lastModified()); response.getOutputStream().write(data); @@ -790,7 +790,6 @@ public class YaCyDefaultServlet extends HttpServlet { response.setDateHeader(HeaderFramework.EXPIRES, now + 1000); // expires in 1 seconds (reduce heavy image creation load) } else { response.setDateHeader(HeaderFramework.EXPIRES, now); // expires now - response.setHeader(HeaderFramework.CACHE_CONTROL, "no-cache"); } if ((targetClass != null)) { @@ -843,6 +842,9 @@ public class YaCyDefaultServlet extends HttpServlet { } else if (tmp instanceof EncodedImage) { final EncodedImage yp = (EncodedImage) tmp; result = yp.getImage(); + if (yp.isStatic()) { + response.setDateHeader(HeaderFramework.EXPIRES, now + 600000); // expires in ten minutes + } } else if (tmp instanceof Image) { final Image i = (Image) tmp; diff --git a/source/net/yacy/peers/SeedDB.java b/source/net/yacy/peers/SeedDB.java index e2e1bcd21..936748505 100644 --- a/source/net/yacy/peers/SeedDB.java +++ b/source/net/yacy/peers/SeedDB.java @@ -866,7 +866,7 @@ public final class SeedDB implements AlternativeDomainNames { // Configure http headers final RequestHeader reqHeader = new RequestHeader(); reqHeader.put(HeaderFramework.PRAGMA, "no-cache"); - reqHeader.put(HeaderFramework.CACHE_CONTROL, "no-cache"); // httpc uses HTTP/1.0 is this necessary? + reqHeader.put(HeaderFramework.CACHE_CONTROL, "no-cache, no-store"); // httpc uses HTTP/1.0 is this necessary? reqHeader.put(HeaderFramework.USER_AGENT, ClientIdentification.yacyInternetCrawlerAgent.userAgent); final HTTPClient client = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent); diff --git a/source/net/yacy/search/Switchboard.java b/source/net/yacy/search/Switchboard.java index 2ea224840..d6c8cd336 100644 --- a/source/net/yacy/search/Switchboard.java +++ b/source/net/yacy/search/Switchboard.java @@ -3766,7 +3766,7 @@ public final class Switchboard extends serverSwitch { //final long start = System.currentTimeMillis(); final RequestHeader reqHeader = new RequestHeader(); reqHeader.put(HeaderFramework.PRAGMA, "no-cache"); - reqHeader.put(HeaderFramework.CACHE_CONTROL, "no-cache"); + reqHeader.put(HeaderFramework.CACHE_CONTROL, "no-cache, no-store"); final HTTPClient client = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent, timeout); client.setHeader(reqHeader.entrySet()); diff --git a/source/net/yacy/server/http/HTTPDemon.java b/source/net/yacy/server/http/HTTPDemon.java index 5891bd063..6ee565ab8 100644 --- a/source/net/yacy/server/http/HTTPDemon.java +++ b/source/net/yacy/server/http/HTTPDemon.java @@ -228,7 +228,7 @@ public final class HTTPDemon { header.put(HeaderFramework.DATE, systemDate); header.put(HeaderFramework.CONTENT_TYPE, "text/html"); header.put(HeaderFramework.CONTENT_LENGTH, Integer.toString(result.length)); - header.put(HeaderFramework.PRAGMA, "no-cache"); + header.put(HeaderFramework.PRAGMA, "no-cache, no-store"); sendRespondHeader(conProp,respond,httpVersion,httpStatusCode,httpStatusText,header); if (! method.equals(HeaderFramework.METHOD_HEAD)) {