- minor bugfixes for url-fetcher & http://www.yacy-forum.de/viewtopic.php?t=3646

- PerformanceMemory_p.html is valid XHTML again

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@3440 6c8d7289-2bf4-0310-a012-ef5d649a1542

htroot/ConfigRobotsTxt_p.java

@@ -48,8 +48,6 @@
 // javac -classpath .:../classes ConfigRobotsTxt_p.java
 // if the shell's current path is HTROOT
 
-import java.util.regex.Pattern;
-
 import de.anomic.http.httpHeader;
 import de.anomic.http.httpdRobotsTxtConfig;
 import de.anomic.plasma.plasmaSwitchboard;
@@ -60,8 +58,6 @@ import de.anomic.yacy.yacyCore;
 
 public class ConfigRobotsTxt_p {
     
-    public static final Pattern entryBeginPattern = Pattern.compile("# (\\w*) \\((\\d*) entries\\)");
-    
     public static servletProperties respond(httpHeader header, serverObjects post, serverSwitch env) {
         final servletProperties prop = new servletProperties();
         
@@ -82,7 +78,7 @@ public class ConfigRobotsTxt_p {
                 rbc.setStatusDisallowed(post.containsKey(httpdRobotsTxtConfig.STATUS));
                 rbc.setSurftipsDisallowed(post.containsKey(httpdRobotsTxtConfig.SURFTIPS));
                 rbc.setWikiDisallowed(post.containsKey(httpdRobotsTxtConfig.WIKI));
-                ((plasmaSwitchboard)env).setConfig(plasmaSwitchboard.ROBOTS_TXT, rbc.toString());
+                env.setConfig(plasmaSwitchboard.ROBOTS_TXT, rbc.toString());
             }
         }
         

htroot/CrawlURLFetch_p.html

@@ -80,7 +80,7 @@
           <dt>Last run duration:</dt><dd>#[lastRun]# ms</dd>
           <dt>Last server response:</dt><dd>#[lastServerResponse]#</dd>
           <dt>Last fetched URLs:</dt><dd>#[lastFetchedURLs]#</dd>
-          <dt>Failed URLs:</dt>
+          <dt>Last failed URLs:</dt>
           <dd>
             #[error]#
             <ul>#{error}# 

htroot/CrawlURLFetch_p.java

@@ -476,6 +476,8 @@ public class CrawlURLFetch_p {
         private int stackURLs(String[] urls) throws InterruptedException {
             this.lastFailed = 0;
             this.lastFetchedURLs = 0;
+            this.failed.clear();
+            
             if (urls == null) return 0;
             String reason;
             for (int i=0; i<urls.length && !isInterrupted(); i++) {

htroot/PerformanceMemory_p.html

@@ -58,22 +58,24 @@
       </tr>
     </table>
     
-    <p><form action="PerformanceMemory_p.html" method="post" enctype="multipart/form-data">
+    <form action="PerformanceMemory_p.html" method="post" enctype="multipart/form-data">
-      <fieldset border="0">
+      <fieldset>
         <legend>Memory Settings</legend>
         <table border="0" cellpadding="2" cellspacing="1" width="60%">
-        <colgroup><col width="20%"><col width="40%"></colgroup>
+          <colgroup><col width="20%" /><col width="40%" /></colgroup>
-        <tr>
+          <tr>
-          <td class="TableHeader" align="right"><label for="Xmx">Memory reserved for JVM</label>&nbsp;</td>
+            <td class="TableHeader" align="right"><label for="Xmx">Memory reserved for JVM</label>&nbsp;</td>
-          <td class="TableCellDark" align="right"><input name="Xmx" id="Xmx" type="text" size="4" maxlength="4" value="#[Xmx]#" /> MByte</td>
+            <td class="TableCellDark" align="right"><input name="Xmx" id="Xmx" type="text" size="4" maxlength="4" value="#[Xmx]#" /> MByte</td>
-		</tr><tr>
+		  </tr>
-          <td #(setStartupCommit)#class="TableCellLight"::class="commit"::class="error"#(/setStartupCommit)# colspan="2" align="right">
+          <tr>
-            #(setStartupCommit)#::Accepted change. This will take effect after <strong>restart</strong> of YaCy#(/setStartupCommit)#
+            <td #(setStartupCommit)#class="TableCellLight"::class="commit"::class="error"#(/setStartupCommit)# colspan="2" align="right">
-            <input type="submit" name="setStartup" value="Set" /></td>
+              #(setStartupCommit)#::Accepted change. This will take effect after <strong>restart</strong> of YaCy#(/setStartupCommit)#
-        </tr>
+              <input type="submit" name="setStartup" value="Set" />
+            </td>
+          </tr>
         </table>
       </fieldset>
-    </form></p>
+    </form>
     
     <p><strong>FlexTable RAM Index:</strong></p>
     <table border="0" cellpadding="2" cellspacing="1">
@@ -94,7 +96,7 @@
       <tr class="TableCellDark">
         <td colspan="4">Total Mem = #[TableIndexTotalMem]# MB</td>
       </tr>
-    </table></p>
+    </table>
     
     <p><strong>Node Caches:</strong></p>
     <table border="0" cellpadding="2" cellspacing="1">
@@ -127,7 +129,7 @@
       <tr class="TableCellDark">
         <td colspan="10">Total Mem = #[nodeCacheTotalMem]# MB; Stop Grow when less than #[nodeCacheStopGrow]# MB available left; Start Shrink when less than #[nodeCacheStartShrink]# MB availabe left</td>
       </tr>
-    </table></p>
+    </table>
     
     <p><strong>Object Read Caches:</strong></p>
     <table border="0" cellpadding="2" cellspacing="1">
@@ -183,7 +185,7 @@
       <tr class="TableCellDark">
         <td colspan="19">Total Mem: #[objectHitCacheTotalMem]# MB (hit), #[objectMissCacheTotalMem]# MB (miss); Stop Grow when less than #[objectCacheStopGrow]# MB available left; Start Shrink when less than #[objectCacheStartShrink]# MB availabe left</td>
       </tr>
-    </table></p>
+    </table>
     
     <!-- other cache sizes -->
     <p><strong>Other Caching Structures:</strong></p>
@@ -204,7 +206,7 @@
         <td>HashBlacklistedCache</td>
         <td>#[blacklistcache.size]#</td>
       </tr>
-    </table></p>
+    </table>
     #%env/templates/footer.template%#
   </body>
 </html>

htroot/Settings_p.java

@@ -66,12 +66,9 @@ public final class Settings_p {
         
         //if (post == null) System.out.println("POST: NULL"); else System.out.println("POST: " + post.toString());
         
-        String page = (post == null) ? "admin" : post.get("page", "admin");
+        String page = (post == null) ? "general" : post.get("page", "general");
         
-        if (page.equals("admin")) {
+        if (page.equals("general")) {
-        	prop.put("settingsTables", "Settings_Admin.inc");
-        }
-        else if (page.equals("general")) {
             prop.put("settingsTables", "Settings_General.inc");
         }
         else if (page.equals("ProxyAccess")) {
@@ -105,7 +102,7 @@ public final class Settings_p {
             prop.put("settingsTables", "Settings_Crawler.inc");
         }        
         else {
-        	prop.put("settingsTables", "Settings_Admin.inc");
+        	prop.put("settingsTables", "Settings_General.inc");
         }
 
         prop.put("port", env.getConfig("port", "8080"));               

htroot/Status_p.inc

@@ -127,7 +127,7 @@
       Disabled. To enable this you need a FTP account where you can upload files to a web space. If you do that, you become a YaCy root server.
       </td>
       <td>
-      <a href="Settings_p.html#seed">[Configure]</a>.
+      <a href="Settings_p.html?page=seed">[Configure]</a>.
       ::
       Enabled: Updating periodically to server #[seedServer]#.
       Last upload: #[lastUpload]# ago.

source/de/anomic/plasma/plasmaSwitchboard.java

@@ -646,7 +646,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
     public static final String HTTPC_NAME_CACHE_CACHING_PATTERNS_NO = "httpc.nameCacheNoCachingPatterns";
     
     public static final String ROBOTS_TXT                       = "httpd.robots.txt";
-    public static final String ROBOTS_TXT_DEFAULT               = httpdRobotsTxtConfig.LOCKED + " " + httpdRobotsTxtConfig.DIRS;
+    public static final String ROBOTS_TXT_DEFAULT               = httpdRobotsTxtConfig.LOCKED + "," + httpdRobotsTxtConfig.DIRS;
     
     //////////////////////////////////////////////////////////////////////////////////////////////
     // Lists