- continue shifting even in the case the stacked URL could not be found

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@3372 6c8d7289-2bf4-0310-a012-ef5d649a1542

htroot/CrawlURLFetchStack_p.html

@@ -38,25 +38,21 @@
           <dd>
             <input type="text" name="shiftloc" id="shiftloc" value="#[locurlsVal]#" size="5" maxlength="5" style="text-align: right;" />
             of <span class="tt">#[locurls]#</span> URLs
-            <input type="submit" name="shiftlcq" value="Shift" />
+            <input type="submit" name="shiftlcq" value="Shift" />#(shiftloc)#::
-            #(shiftloc)#::
+            <span class="success">Shifted #[value]# URLs from Local Crawler Queue to URL Fetcher Stack (not bound: #[failed]#)</span>#(/shiftloc)#
-            <span class="success">Shifted #[value]# URLs from Local Crawler Queue to URL Fetcher Stack</span>::
-            <span class="error">Shifting URLs from Local Crawler Queue to URL Fetcher Stack was unsuccessful: #[error]#</span>#(/shiftloc)#
           </dd>
           <dt><label for="shiftrem">Shift URLs from Remote Crawler</label>:</dt>
           <dd>
             <input type="text" name="shiftrem" id="shiftrem" value="#[remurlsVal]#" size="5" maxlength="5" style="text-align: right;" />
             of <span class="tt">#[remurls]#</span> URLs
-            <input type="submit" name="shiftrcq" value="Shift" />
+            <input type="submit" name="shiftrcq" value="Shift" />#(shiftrem)#::
-            #(shiftrem)#::
+            <span class="success">Shifted #[value]# URLs from Remote Crawler Queue to URL Fetcher Stack (not bound: #[failed]#)</span>#(/shiftrem)#
-            <span class="success">Shifted #[value]# URLs from Remote Crawler Queue to URL Fetcher Stack</span>::
-            <span class="error">Shifting URLs from Remote Crawler Queue to URL Fetcher Stack was unsuccessful: #[error]#</span>#(/shiftrem)#
           </dd>
           <dt><label for="upload">Upload URL-List</label>:</dt>
           <dd>
             <input type="file" name="upload" id="upload" /> #(uploadError)#::&nbsp;<span class="error">No file entered for upload</span>#(/uploadError)#<br />
             <input type="radio" name="uploadType" id="plain" value="plain" checked="checked" /> <label for="plain">Plain text, line-seperated</label><br />
-            <input type="radio" name="uploadType" id="html" value="html" disabled="disabled" /> <label for="html">HTML file, links will be added</label><br />
+            <input type="radio" name="uploadType" id="html" value="html" /> <label for="html">HTML file, links will be added</label><br />
             <input type="submit" name="subupload" value="Upload File" />
             #(upload)#::
             <span class="success">Added #[added]# and rejected #[failed]# URLs from uploaded file successfully</span>::

htroot/CrawlURLFetchStack_p.java

@@ -94,26 +94,18 @@ public class CrawlURLFetchStack_p {
                 }
             }
             else if (post.containsKey("shiftlcq")) {
-                int count = Math.min(post.getInt("shiftloc", 0), sb.noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_CORE));
+                final int count = Math.min(post.getInt("shiftloc", 0), sb.noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_CORE));
-                try {
+                final int failed = shiftFromNotice(sb.noticeURL, plasmaCrawlNURL.STACK_TYPE_CORE, getURLFetcherStack(env), count);
-                    shiftFromNotice(sb.noticeURL, plasmaCrawlNURL.STACK_TYPE_CORE, getURLFetcherStack(env), count);
+                prop.put("shiftloc", 1);
-                    prop.put("shiftloc", 1);
+                prop.put("shiftloc_value", count - failed);
-                    prop.put("shiftloc_value", count);
+                prop.put("shiftloc_failed", failed);
-                } catch (IOException e) {
-                    prop.put("shiftloc", 2);
-                    prop.put("shiftloc_error", e.getMessage());
-                }
             }
             else if (post.containsKey("shiftrcq")) {
-                int count = post.getInt("shiftrem", 0);
+                final int count = post.getInt("shiftrem", 0);
-                try {
+                final int failed = shiftFromNotice(sb.noticeURL, plasmaCrawlNURL.STACK_TYPE_LIMIT, getURLFetcherStack(env), count);
-                    shiftFromNotice(sb.noticeURL, plasmaCrawlNURL.STACK_TYPE_LIMIT, getURLFetcherStack(env), count);
+                prop.put("shiftrem", 1);
-                    prop.put("shiftrem", 1);
+                prop.put("shiftrem_value", count - failed);
-                    prop.put("shiftrem_value", count);
+                prop.put("shiftrem_failed", failed);
-                } catch (IOException e) {
-                    prop.put("shiftrem", 2);
-                    prop.put("shiftrem_error", e.getMessage());
-                }
             }
             else if (post.containsKey("subupload")) {
                 if (post.get("upload", "").length() == 0) {
@@ -190,12 +182,14 @@ public class CrawlURLFetchStack_p {
         return count;
     }
     
-    private static void shiftFromNotice(plasmaCrawlNURL nurl, int fromStackType, URLFetcherStack stack, int count) throws IOException {
+    private static int shiftFromNotice(plasmaCrawlNURL nurl, int fromStackType, URLFetcherStack stack, int count) {
         plasmaCrawlNURL.Entry entry;
-        for (int i=0; i<count; i++) {
+        int failed = 0;
+        for (int i=0; i<count; i++) try {
             entry = nurl.pop(fromStackType);
             stack.push(entry.url());
-        }
+        } catch (IOException e) { failed++; }
+        return failed;
     }
     
     private static int addURLs(serverObjects post, int amount, URLFetcherStack stack) {