diff --git a/htroot/Crawler_p.html b/htroot/Crawler_p.html index 543c0bfd0..03e02632d 100644 --- a/htroot/Crawler_p.html +++ b/htroot/Crawler_p.html @@ -22,18 +22,6 @@   Max - - Indexing - #[indexingSize]# -   - #[indexingMax]# - - - Loader - #[loaderSize]# -   - #[loaderMax]# - Local Crawler #[localCrawlSize]# @@ -64,6 +52,12 @@ unlimited + + Loader + #[loaderSize]# +   + #[loaderMax]# + diff --git a/htroot/Crawler_p.java b/htroot/Crawler_p.java index 1cf184b87..8c63cbe49 100644 --- a/htroot/Crawler_p.java +++ b/htroot/Crawler_p.java @@ -74,8 +74,6 @@ public class Crawler_p { // inital values for AJAX Elements (without JavaScript) final serverObjects prop = new serverObjects(); prop.put("rejected", 0); - prop.put("indexingSize", 0); - prop.put("indexingMax", 0); prop.put("urlpublictextSize", 0); prop.put("rwipublictextSize", 0); prop.put("list", "0"); diff --git a/htroot/RSSLoader_p.java b/htroot/RSSLoader_p.java index 4e8d473e7..f6c697817 100644 --- a/htroot/RSSLoader_p.java +++ b/htroot/RSSLoader_p.java @@ -27,7 +27,6 @@ import java.io.ByteArrayInputStream; import java.net.MalformedURLException; -import java.util.Map; import net.yacy.document.Document; import net.yacy.document.ParserException; @@ -36,8 +35,6 @@ import net.yacy.kelondro.data.meta.DigestURI; import de.anomic.crawler.retrieval.Response; import de.anomic.http.server.RequestHeader; -import de.anomic.search.Segment; -import de.anomic.search.Segments; import de.anomic.search.Switchboard; import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; @@ -53,18 +50,6 @@ public class RSSLoader_p { return prop; } - // get segment - Segment indexSegment = null; - if (post.containsKey("segment")) { - String segmentName = post.get("segment"); - if (sb.indexSegments.segmentExist(segmentName)) { - indexSegment = sb.indexSegments.segment(segmentName); - } - } else { - // take default segment - indexSegment = sb.indexSegments.segment(Segments.Process.PUBLIC); - } - DigestURI url = null; final String urlString = post.get("url", ""); @@ -103,7 +88,7 @@ public class RSSLoader_p { } // get the links out of the rss - Map map = doc.getAnchors(); + //Map map = doc.getAnchors(); // put the urls into crawler using the proxy profile diff --git a/htroot/api/queues_p.java b/htroot/api/queues_p.java index e7f749027..a32633ee8 100755 --- a/htroot/api/queues_p.java +++ b/htroot/api/queues_p.java @@ -31,11 +31,10 @@ public class queues_p { //wikiCode wikiTransformer = new wikiCode(switchboard); final serverObjects prop = new serverObjects(); Segment segment = null; - if (post == null || !post.containsKey("html")) { - prop.setLocalized(false); - if (post.containsKey("segment") && sb.verifyAuthentication(header, false)) { - segment = sb.indexSegments.segment(post.get("segment")); - } + boolean html = post != null && post.containsKey("html"); + prop.setLocalized(html); + if (post != null && post.containsKey("segment") && sb.verifyAuthentication(header, false)) { + segment = sb.indexSegments.segment(post.get("segment")); } if (segment == null) segment = sb.indexSegments.segment(Segments.Process.PUBLIC); prop.put("rejected", "0"); @@ -48,8 +47,8 @@ public class queues_p { prop.putNum("rwipublictextSize", segment.termIndex().sizesMax()); // loader queue - prop.put("loaderSize", Integer.toString(sb.crawlQueues.workerSize())); - prop.put("loaderMax", sb.getConfigLong(SwitchboardConstants.CRAWLER_THREADS_ACTIVE_MAX, 10)); + prop.putNum("loaderSize", sb.crawlQueues.workerSize()); + prop.putNum("loaderMax", sb.getConfigLong(SwitchboardConstants.CRAWLER_THREADS_ACTIVE_MAX, 10)); if (sb.crawlQueues.workerSize() == 0) { prop.put("list-loader", "0"); } else { @@ -68,18 +67,18 @@ public class queues_p { } //local crawl queue - prop.putNum("localCrawlSize", Integer.toString(sb.getThread(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL).getJobCount())); + prop.putNum("localCrawlSize", sb.getThread(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL).getJobCount()); prop.put("localCrawlState", sb.crawlJobIsPaused(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL) ? STATE_PAUSED : STATE_RUNNING); int stackSize = sb.crawlQueues.noticeURL.stackSize(NoticedURL.STACK_TYPE_CORE); addNTable(sb, prop, "list-local", sb.crawlQueues.noticeURL.top(NoticedURL.STACK_TYPE_CORE, Math.min(10, stackSize))); //global crawl queue - prop.putNum("limitCrawlSize", Integer.toString(sb.crawlQueues.limitCrawlJobSize())); + prop.putNum("limitCrawlSize", sb.crawlQueues.limitCrawlJobSize()); prop.put("limitCrawlState", STATE_RUNNING); stackSize = sb.crawlQueues.noticeURL.stackSize(NoticedURL.STACK_TYPE_LIMIT); //global crawl queue - prop.putNum("remoteCrawlSize", Integer.toString(sb.getThread(SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL).getJobCount())); + prop.putNum("remoteCrawlSize", sb.getThread(SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL).getJobCount()); prop.put("remoteCrawlState", sb.crawlJobIsPaused(SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL) ? STATE_PAUSED : STATE_RUNNING); stackSize = sb.crawlQueues.noticeURL.stackSize(NoticedURL.STACK_TYPE_LIMIT); diff --git a/htroot/api/status_p.java b/htroot/api/status_p.java index f9e8c944f..3bc018901 100644 --- a/htroot/api/status_p.java +++ b/htroot/api/status_p.java @@ -20,11 +20,10 @@ public class status_p { final Switchboard sb = (Switchboard) env; final serverObjects prop = new serverObjects(); Segment segment = null; - if (post == null || !post.containsKey("html")) { - prop.setLocalized(false); - if (post.containsKey("segment") && sb.verifyAuthentication(header, false)) { - segment = sb.indexSegments.segment(post.get("segment")); - } + boolean html = post != null && post.containsKey("html"); + prop.setLocalized(html); + if (post != null && post.containsKey("segment") && sb.verifyAuthentication(header, false)) { + segment = sb.indexSegments.segment(post.get("segment")); } if (segment == null) segment = sb.indexSegments.segment(Segments.Process.PUBLIC); @@ -33,9 +32,16 @@ public class status_p { final int cacheMaxSize = (int) sb.getConfigLong(SwitchboardConstants.WORDCACHE_MAX_COUNT, 10000); prop.putNum("ppm", sb.currentPPM()); prop.putNum("qpm", sb.peers.mySeed().getQPM()); - prop.put("wordCacheSize", Integer.toString(segment.termIndex().getBufferSize())); - prop.put("wordCacheMaxSize", Integer.toString(cacheMaxSize)); - // + prop.putNum("wordCacheSize", segment.termIndex().getBufferSize()); + prop.putNum("wordCacheMaxSize", cacheMaxSize); + + // crawl queues + prop.putNum("localCrawlSize", sb.getThread(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL).getJobCount()); + prop.putNum("limitCrawlSize", sb.crawlQueues.limitCrawlJobSize()); + prop.putNum("remoteCrawlSize", sb.getThread(SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL).getJobCount()); + prop.putNum("loaderSize", sb.crawlQueues.workerSize()); + prop.putNum("loaderMax", sb.getConfigLong(SwitchboardConstants.CRAWLER_THREADS_ACTIVE_MAX, 10)); + // memory usage and system attributes prop.putNum("freeMemory", MemoryControl.free()); prop.putNum("totalMemory", MemoryControl.total()); diff --git a/htroot/api/status_p.xml b/htroot/api/status_p.xml index 1142d73d8..0ab1ec071 100644 --- a/htroot/api/status_p.xml +++ b/htroot/api/status_p.xml @@ -3,6 +3,20 @@ #[ppm]# #[wordCacheSize]# #[wordCacheMaxSize]# + + + #[loaderSize]# + #[loaderMax]# + + + #[localCrawlSize]# + + + #[limitCrawlSize]# + + + #[remoteCrawlSize]# + #[freeMemory]# diff --git a/htroot/yacy/search.java b/htroot/yacy/search.java index c7ff42ffd..841b2a95b 100644 --- a/htroot/yacy/search.java +++ b/htroot/yacy/search.java @@ -29,7 +29,6 @@ // if the shell's current path is htroot/yacy import java.util.ArrayList; -import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; diff --git a/source/de/anomic/crawler/AbstractImporter.java b/source/de/anomic/crawler/AbstractImporter.java index 16f7f2a53..57e9fccdd 100644 --- a/source/de/anomic/crawler/AbstractImporter.java +++ b/source/de/anomic/crawler/AbstractImporter.java @@ -1,12 +1,9 @@ package de.anomic.crawler; -import net.yacy.kelondro.logging.Log; - public abstract class AbstractImporter extends Thread implements Importer { private int jobID = -1; private String jobType; - private Log log; private boolean stopped = false; private boolean paused = false; private long globalStart = System.currentTimeMillis(); @@ -19,7 +16,6 @@ public abstract class AbstractImporter extends Thread implements Importer { this.jobType = theJobType; // initializing the logger and setting a more verbose thread name - this.log = new Log("IMPORT_" + this.jobType + "_" + this.jobID); this.setName("IMPORT_" + this.jobType + "_" + this.jobID); } diff --git a/source/de/anomic/crawler/Balancer.java b/source/de/anomic/crawler/Balancer.java index 1ae48eef4..c01c18ba8 100644 --- a/source/de/anomic/crawler/Balancer.java +++ b/source/de/anomic/crawler/Balancer.java @@ -34,7 +34,6 @@ import java.util.concurrent.ConcurrentLinkedQueue; import net.yacy.kelondro.index.BufferedObjectIndex; import net.yacy.kelondro.index.HandleSet; -import net.yacy.kelondro.index.ObjectIndex; import net.yacy.kelondro.index.Row; import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.logging.Log; diff --git a/source/de/anomic/crawler/CrawlQueues.java b/source/de/anomic/crawler/CrawlQueues.java index bb40d04a8..5a2111837 100644 --- a/source/de/anomic/crawler/CrawlQueues.java +++ b/source/de/anomic/crawler/CrawlQueues.java @@ -181,7 +181,10 @@ public class CrawlQueues { synchronized (workers) { final Request[] e = new Request[workers.size()]; int i = 0; - for (final crawlWorker w: workers.values()) e[i++] = w.request; + for (final crawlWorker w: workers.values()) { + if (i >= e.length) break; + e[i++] = w.request; + } return e; } } diff --git a/source/de/anomic/data/translator.java b/source/de/anomic/data/translator.java index 74bfeee1c..cf2c10ba5 100644 --- a/source/de/anomic/data/translator.java +++ b/source/de/anomic/data/translator.java @@ -41,7 +41,6 @@ import java.io.OutputStreamWriter; import java.io.PrintWriter; import java.util.HashMap; import java.util.Hashtable; -import java.util.Iterator; import java.util.regex.Matcher; import java.util.regex.Pattern; diff --git a/source/de/anomic/data/userDB.java b/source/de/anomic/data/userDB.java index 48c6cff1e..97490c829 100644 --- a/source/de/anomic/data/userDB.java +++ b/source/de/anomic/data/userDB.java @@ -49,7 +49,6 @@ import de.anomic.http.server.RequestHeader; public final class userDB { - private static final int USERNAME_MAX_LENGTH = 128; private static final int USERNAME_MIN_LENGTH = 4; private MapHeap userTable; diff --git a/source/de/anomic/search/ResultFetcher.java b/source/de/anomic/search/ResultFetcher.java index 9302ff26a..4d014971a 100644 --- a/source/de/anomic/search/ResultFetcher.java +++ b/source/de/anomic/search/ResultFetcher.java @@ -26,12 +26,9 @@ package de.anomic.search; -import java.io.IOException; import java.util.ArrayList; -import java.util.HashMap; import java.util.Iterator; import java.util.Map; -import java.util.Set; import net.yacy.document.Condenser; import net.yacy.kelondro.data.meta.DigestURI; diff --git a/source/de/anomic/search/TextSnippet.java b/source/de/anomic/search/TextSnippet.java index bef83f455..38875fd51 100644 --- a/source/de/anomic/search/TextSnippet.java +++ b/source/de/anomic/search/TextSnippet.java @@ -25,7 +25,6 @@ package de.anomic.search; import java.io.ByteArrayInputStream; -import java.io.IOException; import java.util.ArrayList; import java.util.Comparator; import java.util.Iterator; @@ -40,7 +39,6 @@ import net.yacy.document.parser.html.CharacterCoding; import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.data.meta.URIMetadataRow; import net.yacy.kelondro.data.word.Word; -import net.yacy.kelondro.data.word.WordReferenceVars; import net.yacy.kelondro.index.ARC; import net.yacy.kelondro.index.ConcurrentARC; import net.yacy.kelondro.index.HandleSet; @@ -48,7 +46,6 @@ import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.order.Base64Order; import net.yacy.kelondro.util.ByteArray; -import net.yacy.kelondro.util.SetTools; import net.yacy.repository.LoaderDispatcher; import de.anomic.crawler.retrieval.Response; diff --git a/source/net/yacy/dbtest.java b/source/net/yacy/dbtest.java index 2f503c50f..25b47b2ae 100644 --- a/source/net/yacy/dbtest.java +++ b/source/net/yacy/dbtest.java @@ -5,16 +5,13 @@ package net.yacy; import java.io.File; import java.io.IOException; -import java.util.ArrayList; import java.util.Date; -import java.util.HashSet; import java.util.Iterator; import java.util.Random; import javax.imageio.ImageIO; import net.yacy.kelondro.index.ObjectIndex; -import net.yacy.kelondro.index.ObjectIndexCache; import net.yacy.kelondro.index.Row; import net.yacy.kelondro.index.RowSet; import net.yacy.kelondro.index.RowSpaceExceededException; @@ -26,7 +23,6 @@ import net.yacy.kelondro.table.SQLTable; import net.yacy.kelondro.table.SplitTable; import net.yacy.kelondro.table.Table; import net.yacy.kelondro.util.MemoryControl; -import net.yacy.kelondro.workflow.InstantBusyThread; import net.yacy.visualization.ChartPlotter; diff --git a/source/net/yacy/document/parser/images/genericImageParser.java b/source/net/yacy/document/parser/images/genericImageParser.java index 12e38df3a..da63dfb0a 100644 --- a/source/net/yacy/document/parser/images/genericImageParser.java +++ b/source/net/yacy/document/parser/images/genericImageParser.java @@ -222,7 +222,7 @@ public class genericImageParser extends AbstractParser implements Idiom { public static ImageInfo parseJavaImage( final DigestURI location, - final BufferedImage image) throws ParserException { + final BufferedImage image) { ImageInfo ii = new ImageInfo(location); ii.image = image; diff --git a/source/net/yacy/kelondro/blob/Stacks.java b/source/net/yacy/kelondro/blob/Stacks.java index 9e7277e75..7d5b7f6a2 100644 --- a/source/net/yacy/kelondro/blob/Stacks.java +++ b/source/net/yacy/kelondro/blob/Stacks.java @@ -68,36 +68,22 @@ public class Stacks { } private class StackInstance { - String name; - private final File location; - Stack stack; + public String name; + public Stack stack; public StackInstance(final File location) throws IOException { String filename = location.getName(); assert filename.startsWith(stacksPrefix + "_"); assert filename.endsWith(".bstack"); this.name = filename.substring(stacksPrefix.length() + 1, filename.length() - 7); - this.location = location; this.stack = new Stack(location); } public StackInstance(final String stack) throws IOException { - this.location = new File(stacksLocation, stacksPrefix + "_" + stack + ".bstack"); this.name = stack; - this.stack = new Stack(location); - } - - public String name() { - return this.name; + this.stack = new Stack(new File(stacksLocation, stacksPrefix + "_" + stack + ".bstack")); } - public Stack stack() { - return this.stack; - } - - public File location() { - return this.location; - } } private Stack getStack(final String stack) { @@ -107,13 +93,13 @@ public class Stacks { try { si = new StackInstance(stack); this.stacks.put(stack, si); - return si.stack(); + return si.stack; } catch (IOException e) { Log.logException(e); return null; } } - return si.stack(); + return si.stack; } /** diff --git a/source/net/yacy/kelondro/data/meta/DigestURI.java b/source/net/yacy/kelondro/data/meta/DigestURI.java index 864328126..e20844b33 100644 --- a/source/net/yacy/kelondro/data/meta/DigestURI.java +++ b/source/net/yacy/kelondro/data/meta/DigestURI.java @@ -35,7 +35,6 @@ import java.io.Serializable; import java.net.MalformedURLException; import java.net.UnknownHostException; import java.text.Collator; -import java.util.Iterator; import java.util.Locale; import java.util.TreeSet; import java.util.regex.Matcher; @@ -936,7 +935,7 @@ public class DigestURI implements Serializable { return hosthash6("http", host, 80); } - private static String[] testTLDs = new String[] { "com", "net", "org", "uk", "fr", "de", "es", "it" }; + //private static String[] testTLDs = new String[] { "com", "net", "org", "uk", "fr", "de", "es", "it" }; public static final int domLengthEstimation(final byte[] urlHashBytes) { // generates an estimation of the original domain length diff --git a/source/net/yacy/kelondro/index/RowCollection.java b/source/net/yacy/kelondro/index/RowCollection.java index 9af107477..f533d5b05 100644 --- a/source/net/yacy/kelondro/index/RowCollection.java +++ b/source/net/yacy/kelondro/index/RowCollection.java @@ -33,7 +33,6 @@ import java.util.Random; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.SynchronousQueue; diff --git a/source/net/yacy/kelondro/order/StackIterator.java b/source/net/yacy/kelondro/order/StackIterator.java index bb528bbf8..260cfa652 100644 --- a/source/net/yacy/kelondro/order/StackIterator.java +++ b/source/net/yacy/kelondro/order/StackIterator.java @@ -22,9 +22,7 @@ package net.yacy.kelondro.order; -import java.util.Collection; import java.util.ConcurrentModificationException; -import java.util.Iterator; public class StackIterator implements CloneableIterator { diff --git a/source/net/yacy/kelondro/rwi/AbstractIndex.java b/source/net/yacy/kelondro/rwi/AbstractIndex.java index 9e262b58d..a0b1a5609 100644 --- a/source/net/yacy/kelondro/rwi/AbstractIndex.java +++ b/source/net/yacy/kelondro/rwi/AbstractIndex.java @@ -28,7 +28,6 @@ package net.yacy.kelondro.rwi; import java.io.IOException; -import java.util.HashMap; import java.util.Iterator; import java.util.TreeMap; import java.util.TreeSet; diff --git a/source/net/yacy/kelondro/rwi/Index.java b/source/net/yacy/kelondro/rwi/Index.java index aa2a9ef85..3a90b89b4 100644 --- a/source/net/yacy/kelondro/rwi/Index.java +++ b/source/net/yacy/kelondro/rwi/Index.java @@ -29,7 +29,6 @@ package net.yacy.kelondro.rwi; import java.io.IOException; -import java.util.HashMap; import java.util.TreeMap; import java.util.TreeSet; diff --git a/source/net/yacy/kelondro/rwi/IndexCell.java b/source/net/yacy/kelondro/rwi/IndexCell.java index b98c320ba..24be692e9 100644 --- a/source/net/yacy/kelondro/rwi/IndexCell.java +++ b/source/net/yacy/kelondro/rwi/IndexCell.java @@ -28,23 +28,19 @@ package net.yacy.kelondro.rwi; import java.io.File; import java.io.IOException; -import java.util.Set; import java.util.concurrent.Semaphore; import de.anomic.yacy.graphics.ProfilingGraph; import net.yacy.kelondro.data.meta.URIMetadataRow; -import net.yacy.kelondro.index.ARC; import net.yacy.kelondro.index.HandleSet; import net.yacy.kelondro.index.Row; import net.yacy.kelondro.index.RowSpaceExceededException; -import net.yacy.kelondro.index.SimpleARC; import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.order.ByteOrder; import net.yacy.kelondro.order.CloneableIterator; import net.yacy.kelondro.order.MergeIterator; import net.yacy.kelondro.order.Order; -import net.yacy.kelondro.util.ByteArray; import net.yacy.kelondro.util.EventTracker; import net.yacy.kelondro.util.MemoryControl; diff --git a/source/net/yacy/kelondro/rwi/ReferenceContainer.java b/source/net/yacy/kelondro/rwi/ReferenceContainer.java index 3dca96c56..5d5c81479 100644 --- a/source/net/yacy/kelondro/rwi/ReferenceContainer.java +++ b/source/net/yacy/kelondro/rwi/ReferenceContainer.java @@ -31,7 +31,6 @@ import java.util.Collection; import java.util.ConcurrentModificationException; import java.util.Iterator; import java.util.Map; -import java.util.Set; import java.util.TreeMap; import net.yacy.kelondro.index.HandleSet; diff --git a/source/net/yacy/kelondro/rwi/ReferenceContainerCache.java b/source/net/yacy/kelondro/rwi/ReferenceContainerCache.java index 99bb1b968..71221086b 100644 --- a/source/net/yacy/kelondro/rwi/ReferenceContainerCache.java +++ b/source/net/yacy/kelondro/rwi/ReferenceContainerCache.java @@ -33,7 +33,6 @@ import java.util.Collections; import java.util.Comparator; import java.util.Iterator; import java.util.Map; -import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import net.yacy.kelondro.blob.HeapWriter;