- cleanup, removed unused imports

- added crawling queue sizes to /api/status_p.xml, syntax same as in queues_p.html
- fixed a bug in queue enumeration that caused a out of bounds exception

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6842 6c8d7289-2bf4-0310-a012-ef5d649a1542

htroot/Crawler_p.html

@@ -22,18 +22,6 @@
             <th>&nbsp;</th>
             <th>Max</th>
           </tr>
-          <tr class="TableCellLight"> 
-            <td align="left">Indexing</td>
-            <td align="right"><span id="indexingqueuesize">#[indexingSize]#</span></td>
-            <td>&nbsp;</td>
-            <td align="right"><span id="indexingqueuemax">#[indexingMax]#</span></td>
-          </tr>
-          <tr class="TableCellLight"> 
-            <td align="left">Loader</td>
-            <td align="right"><span id="loaderqueuesize">#[loaderSize]#</span></td>
-            <td>&nbsp;</td>
-            <td align="right"><span id="loaderqueuemax">#[loaderMax]#</span></td>
-          </tr>
           <tr class="TableCellLight"> 
             <td align="left">Local Crawler</td>
             <td align="right"><span id="localcrawlerqueuesize">#[localCrawlSize]#</span></td>
@@ -64,6 +52,12 @@
             </td>
             <td align="right">unlimited</td>
           </tr>
+          <tr class="TableCellLight"> 
+            <td align="left">Loader</td>
+            <td align="right"><span id="loaderqueuesize">#[loaderSize]#</span></td>
+            <td>&nbsp;</td>
+            <td align="right"><span id="loaderqueuemax">#[loaderMax]#</span></td>
+          </tr>
         </tbody>
       </table>
 

htroot/Crawler_p.java

@@ -74,8 +74,6 @@ public class Crawler_p {
         // inital values for AJAX Elements (without JavaScript) 
         final serverObjects prop = new serverObjects();
         prop.put("rejected", 0);
-        prop.put("indexingSize", 0);
-        prop.put("indexingMax", 0);
         prop.put("urlpublictextSize", 0);
         prop.put("rwipublictextSize", 0);
         prop.put("list", "0");

htroot/RSSLoader_p.java

@@ -27,7 +27,6 @@
 
 import java.io.ByteArrayInputStream;
 import java.net.MalformedURLException;
-import java.util.Map;
 
 import net.yacy.document.Document;
 import net.yacy.document.ParserException;
@@ -36,8 +35,6 @@ import net.yacy.kelondro.data.meta.DigestURI;
 
 import de.anomic.crawler.retrieval.Response;
 import de.anomic.http.server.RequestHeader;
-import de.anomic.search.Segment;
-import de.anomic.search.Segments;
 import de.anomic.search.Switchboard;
 import de.anomic.server.serverObjects;
 import de.anomic.server.serverSwitch;
@@ -53,18 +50,6 @@ public class RSSLoader_p {
             return prop;
         }
         
-        // get segment
-        Segment indexSegment = null;
-        if (post.containsKey("segment")) {
-            String segmentName = post.get("segment");
-            if (sb.indexSegments.segmentExist(segmentName)) {
-                indexSegment = sb.indexSegments.segment(segmentName);
-            }
-        } else {
-            // take default segment
-            indexSegment = sb.indexSegments.segment(Segments.Process.PUBLIC);
-        }
-        
         DigestURI url = null;
         
         final String urlString = post.get("url", "");
@@ -103,7 +88,7 @@ public class RSSLoader_p {
         }
         
         // get the links out of the rss
-        Map<DigestURI, String> map = doc.getAnchors();
+        //Map<DigestURI, String> map = doc.getAnchors();
         
         // put the urls into crawler using the proxy profile
         

htroot/api/queues_p.java

@@ -31,11 +31,10 @@ public class queues_p {
         //wikiCode wikiTransformer = new wikiCode(switchboard);
         final serverObjects prop = new serverObjects();
         Segment segment = null;
-        if (post == null || !post.containsKey("html")) {
+        boolean html = post != null && post.containsKey("html");
-            prop.setLocalized(false);
+        prop.setLocalized(html);
-            if (post.containsKey("segment") && sb.verifyAuthentication(header, false)) {
+        if (post != null && post.containsKey("segment") && sb.verifyAuthentication(header, false)) {
-                segment = sb.indexSegments.segment(post.get("segment"));
+            segment = sb.indexSegments.segment(post.get("segment"));
-            }
         }
         if (segment == null) segment = sb.indexSegments.segment(Segments.Process.PUBLIC);
         prop.put("rejected", "0");
@@ -48,8 +47,8 @@ public class queues_p {
         prop.putNum("rwipublictextSize", segment.termIndex().sizesMax());
 
         // loader queue
-        prop.put("loaderSize", Integer.toString(sb.crawlQueues.workerSize()));        
+        prop.putNum("loaderSize", sb.crawlQueues.workerSize());        
-        prop.put("loaderMax", sb.getConfigLong(SwitchboardConstants.CRAWLER_THREADS_ACTIVE_MAX, 10));
+        prop.putNum("loaderMax", sb.getConfigLong(SwitchboardConstants.CRAWLER_THREADS_ACTIVE_MAX, 10));
         if (sb.crawlQueues.workerSize() == 0) {
             prop.put("list-loader", "0");
         } else {
@@ -68,18 +67,18 @@ public class queues_p {
         }
         
         //local crawl queue
-        prop.putNum("localCrawlSize", Integer.toString(sb.getThread(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL).getJobCount()));
+        prop.putNum("localCrawlSize", sb.getThread(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL).getJobCount());
         prop.put("localCrawlState", sb.crawlJobIsPaused(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL) ? STATE_PAUSED : STATE_RUNNING);
         int stackSize = sb.crawlQueues.noticeURL.stackSize(NoticedURL.STACK_TYPE_CORE);
         addNTable(sb, prop, "list-local", sb.crawlQueues.noticeURL.top(NoticedURL.STACK_TYPE_CORE, Math.min(10, stackSize)));
 
         //global crawl queue
-        prop.putNum("limitCrawlSize", Integer.toString(sb.crawlQueues.limitCrawlJobSize()));
+        prop.putNum("limitCrawlSize", sb.crawlQueues.limitCrawlJobSize());
         prop.put("limitCrawlState", STATE_RUNNING);
         stackSize = sb.crawlQueues.noticeURL.stackSize(NoticedURL.STACK_TYPE_LIMIT);
 
         //global crawl queue
-        prop.putNum("remoteCrawlSize", Integer.toString(sb.getThread(SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL).getJobCount()));
+        prop.putNum("remoteCrawlSize", sb.getThread(SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL).getJobCount());
         prop.put("remoteCrawlState", sb.crawlJobIsPaused(SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL) ? STATE_PAUSED : STATE_RUNNING);
         stackSize = sb.crawlQueues.noticeURL.stackSize(NoticedURL.STACK_TYPE_LIMIT);
 

htroot/api/status_p.java

@@ -20,11 +20,10 @@ public class status_p {
         final Switchboard sb = (Switchboard) env;
         final serverObjects prop = new serverObjects();
         Segment segment = null;
-        if (post == null || !post.containsKey("html")) {
+        boolean html = post != null && post.containsKey("html");
-            prop.setLocalized(false);
+        prop.setLocalized(html);
-            if (post.containsKey("segment") && sb.verifyAuthentication(header, false)) {
+        if (post != null && post.containsKey("segment") && sb.verifyAuthentication(header, false)) {
-                segment = sb.indexSegments.segment(post.get("segment"));
+            segment = sb.indexSegments.segment(post.get("segment"));
-            }
         }
         if (segment == null) segment = sb.indexSegments.segment(Segments.Process.PUBLIC);
         
@@ -33,9 +32,16 @@ public class status_p {
         final int cacheMaxSize = (int) sb.getConfigLong(SwitchboardConstants.WORDCACHE_MAX_COUNT, 10000);
         prop.putNum("ppm", sb.currentPPM());
         prop.putNum("qpm", sb.peers.mySeed().getQPM());
-        prop.put("wordCacheSize", Integer.toString(segment.termIndex().getBufferSize()));
+        prop.putNum("wordCacheSize", segment.termIndex().getBufferSize());
-        prop.put("wordCacheMaxSize", Integer.toString(cacheMaxSize));
+        prop.putNum("wordCacheMaxSize", cacheMaxSize);
-		//
+		
+        // crawl queues
+        prop.putNum("localCrawlSize", sb.getThread(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL).getJobCount());
+        prop.putNum("limitCrawlSize", sb.crawlQueues.limitCrawlJobSize());
+        prop.putNum("remoteCrawlSize", sb.getThread(SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL).getJobCount());
+        prop.putNum("loaderSize", sb.crawlQueues.workerSize());        
+        prop.putNum("loaderMax", sb.getConfigLong(SwitchboardConstants.CRAWLER_THREADS_ACTIVE_MAX, 10));
+        
 		// memory usage and system attributes
         prop.putNum("freeMemory", MemoryControl.free());
         prop.putNum("totalMemory", MemoryControl.total());

htroot/api/status_p.xml

@@ -3,6 +3,20 @@
   <ppm>#[ppm]#</ppm>
   <wordCacheSize>#[wordCacheSize]#</wordCacheSize>
   <wordCacheMaxSize>#[wordCacheMaxSize]#</wordCacheMaxSize>
+  
+  <loaderqueue>
+    <size>#[loaderSize]#</size>
+    <max>#[loaderMax]#</max>
+  </loaderqueue>
+  <localcrawlerqueue>
+    <size>#[localCrawlSize]#</size>
+  </localcrawlerqueue>
+  <limitcrawlerqueue>
+    <size>#[limitCrawlSize]#</size>
+  </limitcrawlerqueue>
+  <remotecrawlerqueue>
+    <size>#[remoteCrawlSize]#</size>
+  </remotecrawlerqueue>
 
   <memory>
     <free>#[freeMemory]#</free>

htroot/yacy/search.java

@@ -29,7 +29,6 @@
 // if the shell's current path is htroot/yacy
 
 import java.util.ArrayList;
-import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;

source/de/anomic/crawler/AbstractImporter.java

@@ -1,12 +1,9 @@
 package de.anomic.crawler;
 
-import net.yacy.kelondro.logging.Log;
-
 public abstract class AbstractImporter extends Thread implements Importer {
 
     private int jobID = -1;
     private String jobType;
-    private Log log;
     private boolean stopped = false;
     private boolean paused = false;
     private long globalStart = System.currentTimeMillis();
@@ -19,7 +16,6 @@ public abstract class AbstractImporter extends Thread implements Importer {
     	this.jobType = theJobType;
 
         // initializing the logger and setting a more verbose thread name
-        this.log = new Log("IMPORT_" + this.jobType + "_" + this.jobID);
         this.setName("IMPORT_" + this.jobType + "_" + this.jobID);
     }
     

source/de/anomic/crawler/Balancer.java

@@ -34,7 +34,6 @@ import java.util.concurrent.ConcurrentLinkedQueue;
 
 import net.yacy.kelondro.index.BufferedObjectIndex;
 import net.yacy.kelondro.index.HandleSet;
-import net.yacy.kelondro.index.ObjectIndex;
 import net.yacy.kelondro.index.Row;
 import net.yacy.kelondro.index.RowSpaceExceededException;
 import net.yacy.kelondro.logging.Log;

source/de/anomic/crawler/CrawlQueues.java

@@ -181,7 +181,10 @@ public class CrawlQueues {
         synchronized (workers) {
             final Request[] e = new Request[workers.size()];
             int i = 0;
-            for (final crawlWorker w: workers.values()) e[i++] = w.request;
+            for (final crawlWorker w: workers.values()) {
+                if (i >= e.length) break;
+                e[i++] = w.request;
+            }
             return e;
         }
     }

source/de/anomic/data/translator.java

@@ -41,7 +41,6 @@ import java.io.OutputStreamWriter;
 import java.io.PrintWriter;
 import java.util.HashMap;
 import java.util.Hashtable;
-import java.util.Iterator;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 

source/de/anomic/data/userDB.java

@@ -49,7 +49,6 @@ import de.anomic.http.server.RequestHeader;
 
 public final class userDB {
     
-    private static final int USERNAME_MAX_LENGTH = 128;
     private static final int USERNAME_MIN_LENGTH = 4;
     
     private MapHeap userTable;

source/de/anomic/search/ResultFetcher.java

@@ -26,12 +26,9 @@
 
 package de.anomic.search;
 
-import java.io.IOException;
 import java.util.ArrayList;
-import java.util.HashMap;
 import java.util.Iterator;
 import java.util.Map;
-import java.util.Set;
 
 import net.yacy.document.Condenser;
 import net.yacy.kelondro.data.meta.DigestURI;

source/de/anomic/search/TextSnippet.java

@@ -25,7 +25,6 @@
 package de.anomic.search;
 
 import java.io.ByteArrayInputStream;
-import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Comparator;
 import java.util.Iterator;
@@ -40,7 +39,6 @@ import net.yacy.document.parser.html.CharacterCoding;
 import net.yacy.kelondro.data.meta.DigestURI;
 import net.yacy.kelondro.data.meta.URIMetadataRow;
 import net.yacy.kelondro.data.word.Word;
-import net.yacy.kelondro.data.word.WordReferenceVars;
 import net.yacy.kelondro.index.ARC;
 import net.yacy.kelondro.index.ConcurrentARC;
 import net.yacy.kelondro.index.HandleSet;
@@ -48,7 +46,6 @@ import net.yacy.kelondro.index.RowSpaceExceededException;
 import net.yacy.kelondro.logging.Log;
 import net.yacy.kelondro.order.Base64Order;
 import net.yacy.kelondro.util.ByteArray;
-import net.yacy.kelondro.util.SetTools;
 import net.yacy.repository.LoaderDispatcher;
 
 import de.anomic.crawler.retrieval.Response;

source/net/yacy/dbtest.java

@@ -5,16 +5,13 @@ package net.yacy;
 
 import java.io.File;
 import java.io.IOException;
-import java.util.ArrayList;
 import java.util.Date;
-import java.util.HashSet;
 import java.util.Iterator;
 import java.util.Random;
 
 import javax.imageio.ImageIO;
 
 import net.yacy.kelondro.index.ObjectIndex;
-import net.yacy.kelondro.index.ObjectIndexCache;
 import net.yacy.kelondro.index.Row;
 import net.yacy.kelondro.index.RowSet;
 import net.yacy.kelondro.index.RowSpaceExceededException;
@@ -26,7 +23,6 @@ import net.yacy.kelondro.table.SQLTable;
 import net.yacy.kelondro.table.SplitTable;
 import net.yacy.kelondro.table.Table;
 import net.yacy.kelondro.util.MemoryControl;
-import net.yacy.kelondro.workflow.InstantBusyThread;
 import net.yacy.visualization.ChartPlotter;
 
 

source/net/yacy/document/parser/images/genericImageParser.java

@@ -222,7 +222,7 @@ public class genericImageParser extends AbstractParser implements Idiom {
     
     public static ImageInfo parseJavaImage(
                             final DigestURI location,
-                            final BufferedImage image) throws ParserException {
+                            final BufferedImage image) {
         ImageInfo ii = new ImageInfo(location);
         ii.image = image;
         

source/net/yacy/kelondro/blob/Stacks.java

@@ -68,36 +68,22 @@ public class Stacks {
     }
     
     private class StackInstance {
-        String name;
+        public String name;
-        private final File location;
+        public Stack stack;
-        Stack stack;
         
         public StackInstance(final File location) throws IOException {
             String filename = location.getName();
             assert filename.startsWith(stacksPrefix + "_");
             assert filename.endsWith(".bstack");
             this.name = filename.substring(stacksPrefix.length() + 1, filename.length() - 7);
-            this.location = location;
             this.stack = new Stack(location);
         }
         
         public StackInstance(final String stack) throws IOException {
-            this.location = new File(stacksLocation, stacksPrefix + "_" + stack + ".bstack");
             this.name = stack;
-            this.stack = new Stack(location);
+            this.stack = new Stack(new File(stacksLocation, stacksPrefix + "_" + stack + ".bstack"));
-        }
-        
-        public String name() {
-            return this.name;
         }
         
-        public Stack stack() {
-            return this.stack;
-        }
-        
-        public File location() {
-            return this.location;
-        }
     }
     
     private Stack getStack(final String stack) {
@@ -107,13 +93,13 @@ public class Stacks {
             try {
                 si = new StackInstance(stack);
                 this.stacks.put(stack, si);
-                return si.stack();
+                return si.stack;
             } catch (IOException e) {
                 Log.logException(e);
                 return null;
             }
         }
-        return si.stack();
+        return si.stack;
     }
     
     /**

source/net/yacy/kelondro/data/meta/DigestURI.java

@@ -35,7 +35,6 @@ import java.io.Serializable;
 import java.net.MalformedURLException;
 import java.net.UnknownHostException;
 import java.text.Collator;
-import java.util.Iterator;
 import java.util.Locale;
 import java.util.TreeSet;
 import java.util.regex.Matcher;
@@ -936,7 +935,7 @@ public class DigestURI implements Serializable {
         return hosthash6("http", host, 80);
     }
     
-    private static String[] testTLDs = new String[] { "com", "net", "org", "uk", "fr", "de", "es", "it" };
+    //private static String[] testTLDs = new String[] { "com", "net", "org", "uk", "fr", "de", "es", "it" };
 
     public static final int domLengthEstimation(final byte[] urlHashBytes) {
         // generates an estimation of the original domain length

source/net/yacy/kelondro/index/RowCollection.java

@@ -33,7 +33,6 @@ import java.util.Random;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
 import java.util.concurrent.RejectedExecutionException;
 import java.util.concurrent.SynchronousQueue;

source/net/yacy/kelondro/order/StackIterator.java

@@ -22,9 +22,7 @@
 
 package net.yacy.kelondro.order;
 
-import java.util.Collection;
 import java.util.ConcurrentModificationException;
-import java.util.Iterator;
 
 
 public class StackIterator<E> implements CloneableIterator<E> {

source/net/yacy/kelondro/rwi/AbstractIndex.java

@@ -28,7 +28,6 @@
 package net.yacy.kelondro.rwi;
 
 import java.io.IOException;
-import java.util.HashMap;
 import java.util.Iterator;
 import java.util.TreeMap;
 import java.util.TreeSet;

source/net/yacy/kelondro/rwi/Index.java

@@ -29,7 +29,6 @@
 package net.yacy.kelondro.rwi;
 
 import java.io.IOException;
-import java.util.HashMap;
 import java.util.TreeMap;
 import java.util.TreeSet;
 

source/net/yacy/kelondro/rwi/IndexCell.java

@@ -28,23 +28,19 @@ package net.yacy.kelondro.rwi;
 
 import java.io.File;
 import java.io.IOException;
-import java.util.Set;
 import java.util.concurrent.Semaphore;
 
 import de.anomic.yacy.graphics.ProfilingGraph;
 
 import net.yacy.kelondro.data.meta.URIMetadataRow;
-import net.yacy.kelondro.index.ARC;
 import net.yacy.kelondro.index.HandleSet;
 import net.yacy.kelondro.index.Row;
 import net.yacy.kelondro.index.RowSpaceExceededException;
-import net.yacy.kelondro.index.SimpleARC;
 import net.yacy.kelondro.logging.Log;
 import net.yacy.kelondro.order.ByteOrder;
 import net.yacy.kelondro.order.CloneableIterator;
 import net.yacy.kelondro.order.MergeIterator;
 import net.yacy.kelondro.order.Order;
-import net.yacy.kelondro.util.ByteArray;
 import net.yacy.kelondro.util.EventTracker;
 import net.yacy.kelondro.util.MemoryControl;
 

source/net/yacy/kelondro/rwi/ReferenceContainer.java

@@ -31,7 +31,6 @@ import java.util.Collection;
 import java.util.ConcurrentModificationException;
 import java.util.Iterator;
 import java.util.Map;
-import java.util.Set;
 import java.util.TreeMap;
 
 import net.yacy.kelondro.index.HandleSet;

source/net/yacy/kelondro/rwi/ReferenceContainerCache.java

@@ -33,7 +33,6 @@ import java.util.Collections;
 import java.util.Comparator;
 import java.util.Iterator;
 import java.util.Map;
-import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 
 import net.yacy.kelondro.blob.HeapWriter;