diff --git a/source/de/anomic/crawler/CrawlStacker.java b/source/de/anomic/crawler/CrawlStacker.java index ab53a7d6c..d0d29a34f 100644 --- a/source/de/anomic/crawler/CrawlStacker.java +++ b/source/de/anomic/crawler/CrawlStacker.java @@ -355,7 +355,7 @@ public final class CrawlStacker { (entry.url().getFileExtension().length() > 0 && TextParser.supports(entry.url(), null) != null) */) { warning = this.nextQueue.noticeURL.push(NoticedURL.StackType.NOLOAD, entry); - if (warning != null) this.log.logWarning("CrawlStacker.stackCrawl of URL " + entry.url().toNormalform(true, false) + " - not pushed: " + warning); + //if (warning != null) this.log.logWarning("CrawlStacker.stackCrawl of URL " + entry.url().toNormalform(true, false) + " - not pushed: " + warning); return null; } diff --git a/source/de/anomic/http/client/Cache.java b/source/de/anomic/http/client/Cache.java index d78bc6c83..ca84c9aba 100644 --- a/source/de/anomic/http/client/Cache.java +++ b/source/de/anomic/http/client/Cache.java @@ -52,21 +52,21 @@ import net.yacy.kelondro.order.Base64Order; public final class Cache { - + private static final String RESPONSE_HEADER_DB_NAME = "responseHeader.heap"; private static final String FILE_DB_NAME = "file.array"; private static Map> responseHeaderDB = null; private static Compressor fileDB = null; private static ArrayStack fileDBunbuffered = null; - + private static long maxCacheSize = Long.MAX_VALUE; private static File cachePath = null; private static String prefix; public static final Log log = new Log("HTCACHE"); - + public static void init(final File htCachePath, final String peerSalt, final long CacheSizeMax) { - + cachePath = htCachePath; maxCacheSize = CacheSizeMax; prefix = peerSalt; @@ -87,11 +87,11 @@ public final class Cache { fileDBunbuffered = new ArrayStack(new File(cachePath, FILE_DB_NAME), prefix, Base64Order.enhancedCoder, 12, 1024 * 1024 * 2, false); fileDBunbuffered.setMaxSize(maxCacheSize); fileDB = new Compressor(fileDBunbuffered, 2 * 1024 * 1024); - } catch (IOException e) { + } catch (final IOException e) { Log.logException(e); } } - + /** * clear the cache */ @@ -99,12 +99,12 @@ public final class Cache { responseHeaderDB.clear(); try { fileDB.clear(); - } catch (IOException e) { + } catch (final IOException e) { Log.logException(e); } try { fileDBunbuffered.clear(); - } catch (IOException e) { + } catch (final IOException e) { Log.logException(e); } } @@ -117,7 +117,7 @@ public final class Cache { maxCacheSize = newCacheSize; fileDBunbuffered.setMaxSize(maxCacheSize); } - + /** * get the current actual cache size * @return @@ -125,7 +125,7 @@ public final class Cache { public static long getActualCacheSize() { return fileDBunbuffered.length(); } - + /** * close the databases */ @@ -135,7 +135,7 @@ public final class Cache { } fileDB.close(true); } - + public static void store(final DigestURI url, final ResponseHeader responseHeader, final byte[] file) throws IOException { if (responseHeader == null) throw new IOException("Cache.store of url " + url.toString() + " not possible: responseHeader == null"); if (file == null) throw new IOException("Cache.store of url " + url.toString() + " not possible: file == null"); @@ -144,12 +144,12 @@ public final class Cache { // store the file try { fileDB.insert(url.hash(), file); - } catch (UnsupportedEncodingException e) { + } catch (final UnsupportedEncodingException e) { throw new IOException("Cache.store: cannot write to fileDB (1): " + e.getMessage()); - } catch (IOException e) { + } catch (final IOException e) { throw new IOException("Cache.store: cannot write to fileDB (2): " + e.getMessage()); } - + // store the response header into the header database final HashMap hm = new HashMap(); hm.putAll(responseHeader); @@ -160,12 +160,12 @@ public final class Cache { } else { responseHeaderDB.put(url.hash(), hm); } - } catch (Exception e) { + } catch (final Exception e) { throw new IOException("Cache.store: cannot write to headerDB: " + e.getMessage()); } if (log.isFine()) log.logFine("stored in cache: " + url.toNormalform(true, false)); } - + /** * check if the responseHeaderDB and the fileDB has an entry for the given url * @param url the url of the resource @@ -188,14 +188,14 @@ public final class Cache { } else { responseHeaderDB.remove(url.hash()); } - } catch (IOException e) {} + } catch (final IOException e) {} if (fileExists) try { - log.logWarning("content but not header of url " + url.toString() + " in cache; cleaned up"); + //log.logWarning("content but not header of url " + url.toString() + " in cache; cleaned up"); fileDB.delete(url.hash()); - } catch (IOException e) {} + } catch (final IOException e) {} return false; } - + /** * Returns an object containing metadata about a cached resource * @param url the {@link URL} of the resource @@ -205,46 +205,46 @@ public final class Cache { * @throws UnsupportedProtocolException if the protocol is not supported and therefore the * info object couldn't be created */ - public static ResponseHeader getResponseHeader(final byte[] hash) { - + public static ResponseHeader getResponseHeader(final byte[] hash) { + // loading data from database Map hdb; hdb = responseHeaderDB.get(hash); if (hdb == null) return null; - + return new ResponseHeader(null, hdb); } - - + + /** * Returns the content of a cached resource as byte[] * @param url the requested resource * @return the resource content as byte[]. If no data * is available or the cached file is not readable, null * is returned. - * @throws IOException + * @throws IOException */ public static byte[] getContent(final byte[] hash) { // load the url as resource from the cache try { - byte[] b = fileDB.get(hash); + final byte[] b = fileDB.get(hash); if (b == null) return null; return b; - } catch (UnsupportedEncodingException e) { + } catch (final UnsupportedEncodingException e) { Log.logException(e); return null; - } catch (IOException e) { + } catch (final IOException e) { Log.logException(e); return null; - } catch (RowSpaceExceededException e) { + } catch (final RowSpaceExceededException e) { Log.logException(e); return null; - } catch (OutOfMemoryError e) { + } catch (final OutOfMemoryError e) { Log.logException(e); return null; } } - + /** * removed response header and cached content from the database * @param url diff --git a/source/net/yacy/kelondro/workflow/WorkflowProcessor.java b/source/net/yacy/kelondro/workflow/WorkflowProcessor.java index 1e8692c52..ba4805dbe 100644 --- a/source/net/yacy/kelondro/workflow/WorkflowProcessor.java +++ b/source/net/yacy/kelondro/workflow/WorkflowProcessor.java @@ -7,7 +7,7 @@ // $LastChangedBy$ // // LICENSE -// +// // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by // the Free Software Foundation; either version 2 of the License, or @@ -33,7 +33,6 @@ import java.util.concurrent.Executors; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; - import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.util.NamePrefixThreadFactory; @@ -50,9 +49,9 @@ public class WorkflowProcessor { private final Object environment; private final String processName, methodName, description; private final String[] childs; - private long blockTime, execTime, passOnTime; + private long blockTime, execTime, passOnTime; private long execCount; - + public WorkflowProcessor( final String name, final String description, final String[] childnames, final Object env, final String jobExecMethod, @@ -71,57 +70,57 @@ public class WorkflowProcessor { this.executor.submit(new InstantBlockingThread(env, jobExecMethod, this)); } // init statistics - blockTime = 0; - execTime = 0; - passOnTime = 0; - execCount = 0; - + this.blockTime = 0; + this.execTime = 0; + this.passOnTime = 0; + this.execCount = 0; + // store this object for easy monitoring processMonitor.add(this); } - + public int queueSize() { return this.input.size(); } - + public boolean queueIsEmpty() { return this.input.isEmpty(); } - + public int queueSizeMax() { return this.input.size() + this.input.remainingCapacity(); } - + public int concurrency() { return this.poolsize; } - + public J take() throws InterruptedException { // read from the input queue if (this.input == null) return null; - long t = System.currentTimeMillis(); - J j = this.input.take(); + final long t = System.currentTimeMillis(); + final J j = this.input.take(); this.blockTime += System.currentTimeMillis() - t; return j; } - + public void passOn(final J next) throws InterruptedException { // don't mix this method up with enQueue()! // this method enqueues into the _next_ queue, not this queue! if (this.output == null) return; - long t = System.currentTimeMillis(); + final long t = System.currentTimeMillis(); this.output.enQueue(next); this.passOnTime += System.currentTimeMillis() - t; } - + public void clear() { if (this.input != null) this.input.clear(); } - + public synchronized void relaxCapacity() { if (this.input.isEmpty()) return; if (this.input.remainingCapacity() > 1000) return; - BlockingQueue i = new LinkedBlockingQueue(); + final BlockingQueue i = new LinkedBlockingQueue(); J e; while (!this.input.isEmpty()) { e = this.input.poll(); @@ -130,15 +129,15 @@ public class WorkflowProcessor { } this.input = i; } - + @SuppressWarnings("unchecked") public void enQueue(final J in) throws InterruptedException { // ensure that enough job executors are running - if ((this.input == null) || (executor == null) || (executor.isShutdown()) || (executor.isTerminated())) { + if ((this.input == null) || (this.executor == null) || (this.executor.isShutdown()) || (this.executor.isTerminated())) { // execute serialized without extra thread - Log.logWarning("PROCESSOR", "executing job " + environment.getClass().getName() + "." + methodName + " serialized"); + //Log.logWarning("PROCESSOR", "executing job " + environment.getClass().getName() + "." + methodName + " serialized"); try { - final J out = (J) InstantBlockingThread.execMethod(this.environment, this.methodName).invoke(environment, new Object[]{in}); + final J out = (J) InstantBlockingThread.execMethod(this.environment, this.methodName).invoke(this.environment, new Object[]{in}); if (out != null && this.output != null) this.output.enQueue(out); } catch (final IllegalArgumentException e) { Log.logException(e); @@ -154,41 +153,41 @@ public class WorkflowProcessor { try { this.input.put(in); break; - } catch (InterruptedException e) { - try {Thread.sleep(10);} catch (InterruptedException ee) {} + } catch (final InterruptedException e) { + try {Thread.sleep(10);} catch (final InterruptedException ee) {} } } } - + @SuppressWarnings("unchecked") public void announceShutdown() { - if (executor == null) return; - if (executor.isShutdown()) return; + if (this.executor == null) return; + if (this.executor.isShutdown()) return; // before we put pills into the queue, make sure that they will take them relaxCapacity(); // put poison pills into the queue - for (int i = 0; i < poolsize; i++) { + for (int i = 0; i < this.poolsize; i++) { try { Log.logInfo("serverProcessor", "putting poison pill in queue " + this.processName + ", thread " + i); - input.put((J) WorkflowJob.poisonPill); // put a poison pill into the queue which will kill the job + this.input.put((J) WorkflowJob.poisonPill); // put a poison pill into the queue which will kill the job Log.logInfo("serverProcessor", ".. poison pill is in queue " + this.processName + ", thread " + i + ". awaiting termination"); } catch (final InterruptedException e) { } } } - + public void awaitShutdown(final long millisTimeout) { - if (executor != null & !executor.isShutdown()) { + if (this.executor != null & !this.executor.isShutdown()) { // wait for shutdown try { - executor.shutdown(); - executor.awaitTermination(millisTimeout, TimeUnit.MILLISECONDS); + this.executor.shutdown(); + this.executor.awaitTermination(millisTimeout, TimeUnit.MILLISECONDS); } catch (final InterruptedException e) {} } Log.logInfo("serverProcessor", "queue " + this.processName + ": shutdown."); this.executor = null; this.input = null; // remove entry from monitor - Iterator> i = processes(); + final Iterator> i = processes(); WorkflowProcessor p; while (i.hasNext()) { p = i.next(); @@ -198,59 +197,59 @@ public class WorkflowProcessor { } } } - + public static Iterator> processes() { return processMonitor.iterator(); } - + protected void increaseJobTime(final long time) { this.execTime += time; this.execCount++; } - + public String getName() { return this.processName; } - + public String getDescription() { return this.description; } - + public String getChilds() { - StringBuilder s = new StringBuilder(this.childs.length * 40 + 1); - for (int i = 0; i < this.childs.length; i++) { - s.append(this.childs[i]); + final StringBuilder s = new StringBuilder(this.childs.length * 40 + 1); + for (final String child : this.childs) { + s.append(child); s.append(' '); } return s.toString(); } - + /** * the block time is the time that a take() blocks until it gets a value * @return */ public long getBlockTime() { - return blockTime; + return this.blockTime; } - + /** * the exec time is the complete time of the execution and processing of the value from take() * @return */ public long getExecTime() { - return execTime; + return this.execTime; } public long getExecCount() { - return execCount; + return this.execCount; } - + /** * the passOn time is the time that a put() takes to enqueue a result value to the next queue * in case that the target queue is limited and may be full, this value may increase * @return */ public long getPassOnTime() { - return passOnTime; + return this.passOnTime; } - + } diff --git a/source/net/yacy/yacy.java b/source/net/yacy/yacy.java index f4236cae5..bc875f7c7 100644 --- a/source/net/yacy/yacy.java +++ b/source/net/yacy/yacy.java @@ -315,7 +315,7 @@ public final class yacy { //boolean properPW = (sb.getConfig("adminAccount", "").length() == 0) && (sb.getConfig(httpd.ADMIN_ACCOUNT_B64MD5, "").length() > 0); //if (!properPW) browserPopUpPage = "ConfigBasic.html"; Browser.openBrowser((server.withSSL()?"https":"http") + "://localhost:" + serverCore.getPortNr(port) + "/" + browserPopUpPage); - } catch (final RuntimeException e) { + } catch (final Throwable e) { // cannot open browser. This may be normal in headless environments //Log.logException(e); }