added option to make the YaCy proxy act as the cache is never stale. If

set to 'Always Fresh' the cache is always used if the entry in the cache
exist. This is a good way to archive web content and access it without
going online again in case the documents exist.
To do so, open /Settings_p.html?page=ProxyAccess and check the "Always
Fresh" checkbox.
This is set do false which behave as set before.
If you set this to true, then you have your web archive in DATA/HTCACHE.
Copy this to carry around your private copy of the internet!

defaults/yacy.init

@@ -244,6 +244,9 @@ proxyCache = DATA/HTCACHE
 # default: 4 Gigabyte
 proxyCacheSize = 4096
 
+# you can use the proxy with fresh/stale rules or in a always-fresh mode
+proxyAlwaysFresh = false
+
 # a path to the surrogate input directory
 surrogates.in = DATA/SURROGATES/in
 

htroot/SettingsAck_p.html

@@ -103,6 +103,10 @@
           <td>Transparent Proxy Support is:</td>
           <td class="settingsValue">#[isTransparentProxy]#</td>
       </tr>
+      <tr>
+          <td>Always Fresh is:</td>
+          <td class="settingsValue">#[proxyAlwaysFresh]#</td>
+      </tr>
       <tr>
           <td>Send via header is:</td>
           <td class="settingsValue">#[proxy.sendViaHeader]#</td>

htroot/SettingsAck_p.java

@@ -154,7 +154,12 @@ public class SettingsAck_p {
             boolean isTransparentProxy = post.containsKey("isTransparentProxy");
             env.setConfig("isTransparentProxy", isTransparentProxy);
             prop.put("info_isTransparentProxy", isTransparentProxy ? "on" : "off");
-
+            
+            // set proxyAlwaysFresh flag
+            boolean proxyAlwaysFresh = post.containsKey("proxyAlwaysFresh");
+            env.setConfig("proxyAlwaysFresh", proxyAlwaysFresh);
+            prop.put("info_proxyAlwaysFresh", proxyAlwaysFresh ? "on" : "off");
+            
             // setting via header property
             env.setConfig("proxy.sendViaHeader", post.containsKey("proxy.sendViaHeader"));
             prop.put("info_proxy.sendViaHeader", post.containsKey("proxy.sendViaHeader")? "on" : "off");

htroot/Settings_ProxyAccess.inc

@@ -10,6 +10,14 @@
       <small><code>iptables -t nat -A PREROUTING -p tcp -s 192.168.0.0/16 --dport 80 -j DNAT --to 192.168.0.1:#[port]#</code></small>
     </td>
   </tr>  
+  <tr valign="top">
+    <td><label for="via_header">Always Fresh</label>:</td>
+    <td><input type="checkbox" name="proxyAlwaysFresh" id="proxyAlwaysFresh" #(proxyAlwaysFresh)#::checked="checked" #(/proxyAlwaysFresh)#/></td>
+    <td>
+      If unchecked, the proxy will act using Cache Fresh / Cache Stale rules. If checked, the cache is always fresh which means
+      that a page is never loaded again if it was already stored in the cache. However, if the page does not exist in the cache, it will be loaded in any case.
+    </td>
+  </tr> 
   <tr valign="top">
     <td><label for="via_header">Send "Via" Header</label>:</td>
     <td><input type="checkbox" name="proxy.sendViaHeader" id="via_header" #(proxy.sendViaHeader)#::checked="checked" #(/proxy.sendViaHeader)#/></td>

htroot/Settings_p.java

@@ -84,6 +84,7 @@ public final class Settings_p {
         
         // http networking settings
         prop.put("isTransparentProxy", env.getConfigBool("isTransparentProxy", false) ? "1" : "0");
+        prop.put("proxyAlwaysFresh", env.getConfigBool("proxyAlwaysFresh", false) ? "1" : "0");
         prop.put("proxy.sendViaHeader", env.getConfigBool("proxy.sendViaHeader", false) ? "1" : "0");
         prop.put("proxy.sendXForwardedForHeader", env.getConfigBool("proxy.sendXForwardedForHeader", true) ? "1" : "0");
         

source/net/yacy/crawler/retrieval/Response.java

@@ -44,6 +44,7 @@ import net.yacy.crawler.data.ResultURLs.EventOrigin;
 import net.yacy.document.Document;
 import net.yacy.document.Parser;
 import net.yacy.document.TextParser;
+import net.yacy.search.Switchboard;
 
 public class Response {
 
@@ -419,6 +420,8 @@ public class Response {
      */
     public boolean isFreshForProxy() {
 
+        if (Switchboard.getSwitchboard().getConfigBool("proxyAlwaysFresh", false)) return true;
+        
         // -CGI access in request
         // CGI access makes the page very individual, and therefore not usable
         // in caches