From f78ce93a8022d04da0bd573b75df0e3607b86519 Mon Sep 17 00:00:00 2001 From: Michael Peter Christen Date: Fri, 13 Jul 2012 21:15:38 +0200 Subject: [PATCH] collection of speed and memory saving hacks --- htroot/api/yacydoc.java | 2 +- source/de/anomic/crawler/ResultImages.java | 11 ++-- source/de/anomic/crawler/RobotsTxt.java | 9 ++- source/de/anomic/crawler/ZURL.java | 4 +- .../yacy/cora/document/MultiProtocolURI.java | 16 ++--- source/net/yacy/cora/lod/JenaTripleStore.java | 33 +++++----- .../yacy/cora/protocol/http/HTTPClient.java | 1 + source/net/yacy/kelondro/blob/HeapReader.java | 5 +- source/net/yacy/kelondro/blob/HeapWriter.java | 61 ++++++++++--------- source/net/yacy/kelondro/data/word/Word.java | 4 +- .../kelondro/logging/ConsoleOutHandler.java | 3 +- source/net/yacy/kelondro/logging/Log.java | 22 +++---- .../net/yacy/kelondro/rwi/IODispatcher.java | 48 ++++++++------- .../kelondro/rwi/ReferenceContainerCache.java | 8 ++- .../net/yacy/visualization/RasterPlotter.java | 4 +- 15 files changed, 123 insertions(+), 108 deletions(-) diff --git a/htroot/api/yacydoc.java b/htroot/api/yacydoc.java index 2cd5aace2..4a0ce0a30 100644 --- a/htroot/api/yacydoc.java +++ b/htroot/api/yacydoc.java @@ -143,7 +143,7 @@ public class yacydoc { references += r.toString()+","; } - Log.logInfo ("TRIPLESTORE", references); + JenaTripleStore.log.logInfo(references); prop.put("taglinks", references); diff --git a/source/de/anomic/crawler/ResultImages.java b/source/de/anomic/crawler/ResultImages.java index 2e1c681dd..28c16ce8a 100644 --- a/source/de/anomic/crawler/ResultImages.java +++ b/source/de/anomic/crawler/ResultImages.java @@ -28,11 +28,11 @@ package de.anomic.crawler; import java.util.Map; import java.util.Queue; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; +import java.util.Set; import java.util.concurrent.LinkedBlockingQueue; import net.yacy.cora.document.MultiProtocolURI; +import net.yacy.cora.storage.SizeLimitedSet; import net.yacy.document.Document; import net.yacy.document.parser.html.ImageEntry; import net.yacy.kelondro.data.meta.DigestURI; @@ -52,7 +52,7 @@ public class ResultImages { // we also check all links for a double-check so we don't get the same image more than once in any queue // image links may appear double here even if the pages where the image links are embedded already are checked for double-occurrence: // the same images may be linked from different pages - private static final ConcurrentMap doubleCheck = new ConcurrentHashMap(); // (url, time) when the url appeared first + private static final Set doubleCheck = new SizeLimitedSet(10000); public static void registerImages(final DigestURI source, final Document document, final boolean privateEntry) { if (document == null) return; @@ -65,8 +65,9 @@ public class ResultImages { for (final ImageEntry image: images.values()) { // do a double-check; attention: this can be time-consuming since this possibly needs a DNS-lookup if (image == null || image.url() == null) continue; - if (doubleCheck.containsKey(image.url())) continue; - doubleCheck.put(image.url(), System.currentTimeMillis()); + String url = image.url().toNormalform(true, false); + if (doubleCheck.contains(url)) continue; + doubleCheck.add(url); final String name = image.url().getFile(); boolean good = false; diff --git a/source/de/anomic/crawler/RobotsTxt.java b/source/de/anomic/crawler/RobotsTxt.java index 3d4c55beb..b20c6bb39 100644 --- a/source/de/anomic/crawler/RobotsTxt.java +++ b/source/de/anomic/crawler/RobotsTxt.java @@ -267,12 +267,11 @@ public class RobotsTxt { private static final int DOWNLOAD_MODDATE = 3; static final String getHostPort(final MultiProtocolURI theURL) { - String urlHostPort = null; final int port = getPort(theURL); - urlHostPort = theURL.getHost() + ":" + port; - urlHostPort = urlHostPort.toLowerCase().intern(); - - return urlHostPort; + String host = theURL.getHost(); + StringBuilder sb = new StringBuilder(host.length() + 6); + sb.append(host).append(':').append(Integer.toString(port)); + return sb.toString(); } private static final int getPort(final MultiProtocolURI theURL) { diff --git a/source/de/anomic/crawler/ZURL.java b/source/de/anomic/crawler/ZURL.java index 1ebd1f36c..3a3d2f9a3 100644 --- a/source/de/anomic/crawler/ZURL.java +++ b/source/de/anomic/crawler/ZURL.java @@ -55,6 +55,8 @@ import de.anomic.crawler.retrieval.Request; public class ZURL implements Iterable { + public static Log log = new Log("REJECTED"); + private static final int EcoFSBufferSize = 2000; private static final int maxStackSize = 1000; @@ -167,7 +169,7 @@ public class ZURL implements Iterable { final Entry entry = new Entry(bentry, executor, workdate, workcount, reason); put(entry); this.stack.add(entry.hash()); - if (!reason.startsWith("double")) Log.logInfo("Rejected URL", bentry.url().toNormalform(false, false) + " - " + reason); + if (!reason.startsWith("double")) log.logInfo(bentry.url().toNormalform(false, false) + " - " + reason); if (this.solrConnector != null && failCategory.store) { // send the error to solr try { diff --git a/source/net/yacy/cora/document/MultiProtocolURI.java b/source/net/yacy/cora/document/MultiProtocolURI.java index 822d06667..32a0c8538 100644 --- a/source/net/yacy/cora/document/MultiProtocolURI.java +++ b/source/net/yacy/cora/document/MultiProtocolURI.java @@ -158,7 +158,7 @@ public class MultiProtocolURI implements Serializable, Comparable ni = JenaTripleStore.getObjects(subject, predicate); String object = ""; if (ni.hasNext()) object = ni.next().toString(); - Log.logInfo("TRIPLESTORE", "GET " + subject + " - " + predicate + " - " + object); + log.logInfo("GET " + subject + " - " + predicate + " - " + object); return object; } @@ -195,7 +196,7 @@ public class JenaTripleStore { Iterator ni = JenaTripleStore.getPrivateObjects(subject, predicate, username); String object = ""; if (ni.hasNext()) object = ni.next().toString(); - Log.logInfo("TRIPLESTORE", "GET (" + username + ") " + subject + " - " + predicate + " - " + object); + log.logInfo("GET (" + username + ") " + subject + " - " + predicate + " - " + object); return object; } @@ -270,7 +271,7 @@ public class JenaTripleStore { public static void initPrivateStores() { Switchboard switchboard = Switchboard.getSwitchboard(); - Log.logInfo("TRIPLESTORE", "Init private stores"); + log.logInfo("Init private stores"); if (privatestorage == null) privatestorage = new ConcurrentHashMap(); if (privatestorage != null) privatestorage.clear(); @@ -281,17 +282,17 @@ public class JenaTripleStore { String username = e.getUserName(); File triplestore = new File(switchboard.getConfig("triplestore", new File(switchboard.getDataPath(), "DATA/TRIPLESTORE").getAbsolutePath())); File currentuserfile = new File(triplestore, "private_store_"+username+".rdf"); - Log.logInfo("TRIPLESTORE", "Init " + username + " from "+currentuserfile.getAbsolutePath()); + log.logInfo("Init " + username + " from "+currentuserfile.getAbsolutePath()); Model tmp = ModelFactory.createDefaultModel(); init (tmp); if (currentuserfile.exists()) { - Log.logInfo("TRIPLESTORE", "Loading from " + currentuserfile.getAbsolutePath()); + log.logInfo("Loading from " + currentuserfile.getAbsolutePath()); InputStream is = FileManager.get().open(currentuserfile.getAbsolutePath()); if (is != null) { // read the RDF/XML file tmp.read(is, null); - Log.logInfo("TRIPLESTORE", "loaded " + tmp.size() + " triples from " + currentuserfile.getAbsolutePath()); + log.logInfo("loaded " + tmp.size() + " triples from " + currentuserfile.getAbsolutePath()); } else { throw new IOException("cannot read " + currentuserfile.getAbsolutePath()); } @@ -308,7 +309,7 @@ public class JenaTripleStore { public static void savePrivateStores() { Switchboard switchboard = Switchboard.getSwitchboard(); - Log.logInfo("TRIPLESTORE", "Saving user triplestores"); + log.logInfo("Saving user triplestores"); if (privatestorage == null) return; for (Entry s : privatestorage.entrySet()) { File triplestore = new File(switchboard.getConfig("triplestore", new File(switchboard.getDataPath(), "DATA/TRIPLESTORE").getAbsolutePath())); diff --git a/source/net/yacy/cora/protocol/http/HTTPClient.java b/source/net/yacy/cora/protocol/http/HTTPClient.java index ec9901094..503eb1be6 100644 --- a/source/net/yacy/cora/protocol/http/HTTPClient.java +++ b/source/net/yacy/cora/protocol/http/HTTPClient.java @@ -598,6 +598,7 @@ public class HTTPClient { assert !hrequest.expectContinue(); } + Thread.currentThread().setName("HTTPClient-" + httpUriRequest.getURI().getHost()); try { final long time = System.currentTimeMillis(); this.httpResponse = httpClient.execute(httpUriRequest, httpContext); diff --git a/source/net/yacy/kelondro/blob/HeapReader.java b/source/net/yacy/kelondro/blob/HeapReader.java index 3699ce87a..2c0ec21bd 100644 --- a/source/net/yacy/kelondro/blob/HeapReader.java +++ b/source/net/yacy/kelondro/blob/HeapReader.java @@ -58,7 +58,7 @@ public class HeapReader { //public final static long keepFreeMem = 20 * 1024 * 1024; private final static Log log = new Log("HeapReader"); - + // input values protected int keylength; // the length of the primary key protected File heapFile; // the file of the heap @@ -731,7 +731,7 @@ public class HeapReader { public entries(final File blobFile, final int keylen) throws IOException { if (!(blobFile.exists())) throw new IOException("file " + blobFile + " does not exist"); try { - this.is = new DataInputStream(new BufferedInputStream(new FileInputStream(blobFile), 8*1024*1024)); + this.is = new DataInputStream(new BufferedInputStream(new FileInputStream(blobFile), 256 * 1024)); } catch (OutOfMemoryError e) { this.is = new DataInputStream(new FileInputStream(blobFile)); } @@ -810,6 +810,7 @@ public class HeapReader { } } + @Override public synchronized void close() { if (this.is != null) try { this.is.close(); } catch (final IOException e) {Log.logException(e);} this.is = null; diff --git a/source/net/yacy/kelondro/blob/HeapWriter.java b/source/net/yacy/kelondro/blob/HeapWriter.java index 185302d6b..cfc8b48df 100644 --- a/source/net/yacy/kelondro/blob/HeapWriter.java +++ b/source/net/yacy/kelondro/blob/HeapWriter.java @@ -7,7 +7,7 @@ // $LastChangedBy$ // // LICENSE -// +// // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by // the Free Software Foundation; either version 2 of the License, or @@ -40,8 +40,9 @@ import net.yacy.kelondro.util.FileUtils; public final class HeapWriter { + public final static Log log = new Log("HeapWriter"); public final static byte[] ZERO = new byte[]{0}; - + private final int keylength; // the length of the primary key private HandleMap index; // key/seek relation for used records private final File heapFileTMP; // the temporary file of the heap during writing @@ -49,7 +50,7 @@ public final class HeapWriter { private DataOutputStream os; // the output stream where the BLOB is written private long seek; // the current write position //private HashSet doublecheck;// only for testing - + /* * This class implements a BLOB management based on a sequence of records * The data structure is: @@ -59,9 +60,9 @@ public final class HeapWriter { * key :== * blob :== * that means that each record has the size reclen+4 - * + * * Because the blob sizes are stored with integers, one entry may not exceed 2GB - * + * * With this class a BLOB file can only be written. * To read them, use a kelondroBLOBHeapReader. * A BLOBHeap can be also read and write in random access mode with kelondroBLOBHeap. @@ -96,61 +97,61 @@ public final class HeapWriter { * @param key * @param blob * @throws IOException - * @throws RowSpaceExceededException - * @throws RowSpaceExceededException + * @throws RowSpaceExceededException + * @throws RowSpaceExceededException */ public synchronized void add(byte[] key, final byte[] blob) throws IOException, RowSpaceExceededException { assert blob.length > 0; key = HeapReader.normalizeKey(key, this.keylength); - assert index.row().primaryKeyLength == this.keylength : index.row().primaryKeyLength + "!=" + key.length; + assert this.index.row().primaryKeyLength == this.keylength : this.index.row().primaryKeyLength + "!=" + key.length; assert key.length == this.keylength : "key.length == " + key.length + ", this.keylength = " + this.keylength; // after normalizing they should be equal in length - assert index.get(key) < 0 : "index.get(key) = " + index.get(key) + ", index.size() = " + index.size() + ", file.length() = " + this.heapFileTMP.length() + ", key = " + UTF8.String(key); // must not occur before + assert this.index.get(key) < 0 : "index.get(key) = " + this.index.get(key) + ", index.size() = " + this.index.size() + ", file.length() = " + this.heapFileTMP.length() + ", key = " + UTF8.String(key); // must not occur before if ((blob == null) || (blob.length == 0)) return; - index.putUnique(key, this.seek); + this.index.putUnique(key, this.seek); int chunkl = this.keylength + blob.length; - os.writeInt(chunkl); - os.write(key); - os.write(blob); + this.os.writeInt(chunkl); + this.os.write(key); + this.os.write(blob); this.seek += chunkl + 4; //os.flush(); // necessary? may cause bad IO performance :-( } - + /** * close the BLOB table - * @throws + * @throws */ public synchronized void close(boolean writeIDX) throws IOException { // close the file - os.flush(); - os.close(); - os = null; - + this.os.flush(); + this.os.close(); + this.os = null; + // rename the file into final name if (this.heapFileREADY.exists()) FileUtils.deletedelete(this.heapFileREADY); boolean renameok = this.heapFileTMP.renameTo(this.heapFileREADY); if (!renameok) throw new IOException("cannot rename " + this.heapFileTMP + " to " + this.heapFileREADY); if (!this.heapFileREADY.exists()) throw new IOException("renaming of " + this.heapFileREADY.toString() + " failed: files still exists"); if (this.heapFileTMP.exists()) throw new IOException("renaming to " + this.heapFileTMP.toString() + " failed: file does not exist"); - + // generate index and gap files - if (writeIDX && index.size() > 3) { + if (writeIDX && this.index.size() > 3) { // now we can create a dump of the index and the gap information // to speed up the next start long start = System.currentTimeMillis(); String fingerprint = HeapReader.fingerprintFileHash(this.heapFileREADY); if (fingerprint == null) { - Log.logSevere("kelondroBLOBHeapWriter", "cannot write a dump for " + heapFileREADY.getName()+ ": fingerprint is null"); + log.logSevere("cannot write a dump for " + this.heapFileREADY.getName()+ ": fingerprint is null"); } else { new Gap().dump(fingerprintGapFile(this.heapFileREADY, fingerprint)); - index.dump(fingerprintIndexFile(this.heapFileREADY, fingerprint)); - Log.logInfo("kelondroBLOBHeapWriter", "wrote a dump for the " + this.index.size() + " index entries of " + heapFileREADY.getName()+ " in " + (System.currentTimeMillis() - start) + " milliseconds."); + this.index.dump(fingerprintIndexFile(this.heapFileREADY, fingerprint)); + log.logInfo("wrote a dump for the " + this.index.size() + " index entries of " + this.heapFileREADY.getName()+ " in " + (System.currentTimeMillis() - start) + " milliseconds."); } - index.close(); - index = null; + this.index.close(); + this.index = null; } else { // this is small.. just free resources, do not write index - index.close(); - index = null; + this.index.close(); + this.index = null; } } @@ -165,12 +166,12 @@ public final class HeapWriter { FileUtils.deletedelete(new File(p, s)); } } - + protected static File fingerprintIndexFile(File f, String fingerprint) { assert f != null; return new File(f.getParentFile(), f.getName() + "." + fingerprint + ".idx"); } - + protected static File fingerprintGapFile(File f, String fingerprint) { assert f != null; return new File(f.getParentFile(), f.getName() + "." + fingerprint + ".gap"); diff --git a/source/net/yacy/kelondro/data/word/Word.java b/source/net/yacy/kelondro/data/word/Word.java index 6a48594e5..0a74d36c5 100644 --- a/source/net/yacy/kelondro/data/word/Word.java +++ b/source/net/yacy/kelondro/data/word/Word.java @@ -56,13 +56,15 @@ public class Word { */ public static final int commonHashLength = 12; - private static final int hashCacheSize = Math.max(100000, Math.min(1000000, (int) (MemoryControl.available() / 40000L))); + private static final int hashCacheSize = Math.max(20000, Math.min(200000, (int) (MemoryControl.available() / 40000L))); private static ARC hashCache = null; static { try { hashCache = new ConcurrentARC(hashCacheSize, Math.max(32, 4 * Runtime.getRuntime().availableProcessors())); + Log.logInfo("Word", "hashCache.size = " + hashCacheSize); } catch (final OutOfMemoryError e) { hashCache = new ConcurrentARC(1000, Math.max(8, 2 * Runtime.getRuntime().availableProcessors())); + Log.logInfo("Word", "hashCache.size = " + 1000); } } /* diff --git a/source/net/yacy/kelondro/logging/ConsoleOutHandler.java b/source/net/yacy/kelondro/logging/ConsoleOutHandler.java index e94cca485..4b456bc13 100644 --- a/source/net/yacy/kelondro/logging/ConsoleOutHandler.java +++ b/source/net/yacy/kelondro/logging/ConsoleOutHandler.java @@ -27,6 +27,7 @@ package net.yacy.kelondro.logging; +import java.io.BufferedOutputStream; import java.util.logging.Level; import java.util.logging.LogRecord; import java.util.logging.SimpleFormatter; @@ -39,7 +40,7 @@ public final class ConsoleOutHandler extends StreamHandler { public ConsoleOutHandler() { setLevel(Level.FINEST); setFormatter(new SimpleFormatter()); - setOutputStream(System.out); + setOutputStream(new BufferedOutputStream(System.out)); } @Override diff --git a/source/net/yacy/kelondro/logging/Log.java b/source/net/yacy/kelondro/logging/Log.java index 81e2c5ae8..76bb1ddc5 100644 --- a/source/net/yacy/kelondro/logging/Log.java +++ b/source/net/yacy/kelondro/logging/Log.java @@ -288,37 +288,37 @@ public final class Log { } protected final static class logEntry { - public final Logger logger; - public final String loggername; public final Level level; public final String message; - public final Throwable thrown; + public Logger logger; + public String loggername; + public Throwable thrown; + private logEntry(final Level level, final String message) { + this.level = level; + this.message = message == null || message.length() <= 120 ? message : message.substring(0, 120); + } public logEntry(final Logger logger, final Level level, final String message, final Throwable thrown) { + this(level, message); this.logger = logger; this.loggername = null; - this.level = level; - this.message = message; this.thrown = thrown; } public logEntry(final Logger logger, final Level level, final String message) { + this(level, message); this.logger = logger; this.loggername = null; - this.level = level; - this.message = message; this.thrown = null; } public logEntry(final String loggername, final Level level, final String message, final Throwable thrown) { + this(level, message); this.logger = null; this.loggername = loggername; - this.level = level; - this.message = message; this.thrown = thrown; } public logEntry(final String loggername, final Level level, final String message) { + this(level, message); this.logger = null; this.loggername = loggername; - this.level = level; - this.message = message; this.thrown = null; } public logEntry() { diff --git a/source/net/yacy/kelondro/rwi/IODispatcher.java b/source/net/yacy/kelondro/rwi/IODispatcher.java index 04512256b..b46b1a022 100644 --- a/source/net/yacy/kelondro/rwi/IODispatcher.java +++ b/source/net/yacy/kelondro/rwi/IODispatcher.java @@ -48,6 +48,8 @@ import net.yacy.kelondro.util.MemoryControl; */ public class IODispatcher extends Thread { + public static final Log log = new Log("IODispatcher"); + private Semaphore controlQueue; private final Semaphore termination; private ArrayBlockingQueue mergeQueue; @@ -81,7 +83,7 @@ public class IODispatcher extends Thread { @SuppressWarnings("unchecked") protected synchronized void dump(final ReferenceContainerCache cache, final File file, final ReferenceContainerArray array) { if (this.dumpQueue == null || this.controlQueue == null || !isAlive()) { - Log.logWarning("IODispatcher", "emergency dump of file " + file.getName()); + log.logWarning("emergency dump of file " + file.getName()); if (!cache.isEmpty()) cache.dump(file, (int) Math.min(MemoryControl.available() / 3, this.writeBufferSize), true); } else { @SuppressWarnings("rawtypes") @@ -91,7 +93,7 @@ public class IODispatcher extends Thread { if (isAlive()) { try { this.dumpQueue.put(job); - Log.logInfo("IODispatcher", "appended dump job for file " + file.getName()); + log.logInfo("appended dump job for file " + file.getName()); } catch (final InterruptedException e) { Log.logException(e); cache.dump(file, (int) Math.min(MemoryControl.available() / 3, this.writeBufferSize), true); @@ -100,7 +102,7 @@ public class IODispatcher extends Thread { } } else { job.dump(); - Log.logWarning("IODispatcher", "dispatcher is not alive, just dumped file " + file.getName()); + log.logWarning("dispatcher is not alive, just dumped file " + file.getName()); } } } @@ -112,9 +114,9 @@ public class IODispatcher extends Thread { protected synchronized void merge(final File f1, final File f2, final ReferenceFactory factory, final ArrayStack array, final File newFile) { if (this.mergeQueue == null || this.controlQueue == null || !isAlive()) { if (f2 == null) { - Log.logWarning("IODispatcher", "emergency rewrite of file " + f1.getName() + " to " + newFile.getName()); + log.logWarning("emergency rewrite of file " + f1.getName() + " to " + newFile.getName()); } else { - Log.logWarning("IODispatcher", "emergency merge of files " + f1.getName() + ", " + f2.getName() + " to " + newFile.getName()); + log.logWarning("emergency merge of files " + f1.getName() + ", " + f2.getName() + " to " + newFile.getName()); } array.mergeMount(f1, f2, factory, newFile, (int) Math.min(MemoryControl.available() / 3, this.writeBufferSize)); } else { @@ -123,12 +125,12 @@ public class IODispatcher extends Thread { try { this.mergeQueue.put(job); if (f2 == null) { - Log.logInfo("IODispatcher", "appended rewrite job of file " + f1.getName() + " to " + newFile.getName()); + log.logInfo("appended rewrite job of file " + f1.getName() + " to " + newFile.getName()); } else { - Log.logInfo("IODispatcher", "appended merge job of files " + f1.getName() + ", " + f2.getName() + " to " + newFile.getName()); + log.logInfo("appended merge job of files " + f1.getName() + ", " + f2.getName() + " to " + newFile.getName()); } } catch (final InterruptedException e) { - Log.logWarning("IODispatcher", "interrupted: " + e.getMessage(), e); + log.logWarning("interrupted: " + e.getMessage(), e); array.mergeMount(f1, f2, factory, newFile, (int) Math.min(MemoryControl.available() / 3, this.writeBufferSize)); } finally { this.controlQueue.release(); @@ -136,9 +138,9 @@ public class IODispatcher extends Thread { } else { job.merge(); if (f2 == null) { - Log.logWarning("IODispatcher", "dispatcher not running, merged files " + f1.getName() + " to " + newFile.getName()); + log.logWarning("dispatcher not running, merged files " + f1.getName() + " to " + newFile.getName()); } else { - Log.logWarning("IODispatcher", "dispatcher not running, rewrote file " + f1.getName() + ", " + f2.getName() + " to " + newFile.getName()); + log.logWarning("dispatcher not running, rewrote file " + f1.getName() + ", " + f2.getName() + " to " + newFile.getName()); } } } @@ -160,10 +162,10 @@ public class IODispatcher extends Thread { f = dumpJob.file; dumpJob.dump(); } catch (final InterruptedException e) { - Log.logSevere("IODispatcher", "main run job was interrupted (1)", e); + log.logSevere("main run job was interrupted (1)", e); Log.logException(e); } catch (final Throwable e) { - Log.logSevere("IODispatcher", "main run job had errors (1), dump to " + f + " failed.", e); + log.logSevere("main run job had errors (1), dump to " + f + " failed.", e); Log.logException(e); } continue loop; @@ -179,13 +181,13 @@ public class IODispatcher extends Thread { f2 = mergeJob.f2; mergeJob.merge(); } catch (final InterruptedException e) { - Log.logSevere("IODispatcher", "main run job was interrupted (2)", e); + log.logSevere("main run job was interrupted (2)", e); Log.logException(e); } catch (final Throwable e) { if (f2 == null) { - Log.logSevere("IODispatcher", "main run job had errors (2), dump to " + f + " failed. Input file is " + f1, e); + log.logSevere("main run job had errors (2), dump to " + f + " failed. Input file is " + f1, e); } else { - Log.logSevere("IODispatcher", "main run job had errors (2), dump to " + f + " failed. Input files are " + f1 + " and " + f2, e); + log.logSevere("main run job had errors (2), dump to " + f + " failed. Input files are " + f1 + " and " + f2, e); } Log.logException(e); } @@ -194,22 +196,22 @@ public class IODispatcher extends Thread { // check termination if (this.terminate) { - Log.logInfo("IODispatcher", "caught termination signal"); + log.logInfo("caught termination signal"); break; } - Log.logSevere("IODispatcher", "main loop in bad state, dumpQueue.size() = " + this.dumpQueue.size() + ", mergeQueue.size() = " + this.mergeQueue.size() + ", controlQueue.availablePermits() = " + this.controlQueue.availablePermits()); + log.logSevere("main loop in bad state, dumpQueue.size() = " + this.dumpQueue.size() + ", mergeQueue.size() = " + this.mergeQueue.size() + ", controlQueue.availablePermits() = " + this.controlQueue.availablePermits()); assert false : "this process statt should not be reached"; // this should never happen } catch (final Throwable e) { - Log.logSevere("IODispatcher", "main run job failed (X)", e); + log.logSevere("main run job failed (X)", e); Log.logException(e); } - Log.logInfo("IODispatcher", "loop terminated"); + log.logInfo("loop terminated"); } catch (final Throwable e) { - Log.logSevere("IODispatcher", "main run job failed (4)", e); + log.logSevere("main run job failed (4)", e); Log.logException(e); } finally { - Log.logInfo("IODispatcher", "terminating run job"); + log.logInfo("terminating run job"); this.controlQueue = null; this.dumpQueue = null; this.mergeQueue = null; @@ -257,11 +259,11 @@ public class IODispatcher extends Thread { private File merge() { if (!this.f1.exists()) { - Log.logWarning("IODispatcher", "merge of file (1) " + this.f1.getName() + " failed: file does not exists"); + log.logWarning("merge of file (1) " + this.f1.getName() + " failed: file does not exists"); return null; } if (this.f2 != null && !this.f2.exists()) { - Log.logWarning("IODispatcher", "merge of file (2) " + this.f2.getName() + " failed: file does not exists"); + log.logWarning("merge of file (2) " + this.f2.getName() + " failed: file does not exists"); return null; } return this.array.mergeMount(this.f1, this.f2, this.factory, this.newFile, (int) Math.min(MemoryControl.available() / 3, IODispatcher.this.writeBufferSize)); diff --git a/source/net/yacy/kelondro/rwi/ReferenceContainerCache.java b/source/net/yacy/kelondro/rwi/ReferenceContainerCache.java index f904e4dee..c923e1769 100644 --- a/source/net/yacy/kelondro/rwi/ReferenceContainerCache.java +++ b/source/net/yacy/kelondro/rwi/ReferenceContainerCache.java @@ -59,6 +59,8 @@ import net.yacy.kelondro.util.FileUtils; */ public final class ReferenceContainerCache extends AbstractIndex implements Index, IndexReader, Iterable> { + public static final Log log = new Log("ReferenceContainerCache"); + private final int termSize; private final ByteOrder termOrder; private final ContainerOrder containerOrder; @@ -117,7 +119,7 @@ public final class ReferenceContainerCache exte public void dump(final File heapFile, final int writeBuffer, final boolean destructive) { assert this.cache != null; if (this.cache == null) return; - Log.logInfo("indexContainerRAMHeap", "creating rwi heap dump '" + heapFile.getName() + "', " + this.cache.size() + " rwi's"); + log.logInfo("creating rwi heap dump '" + heapFile.getName() + "', " + this.cache.size() + " rwi's"); if (heapFile.exists()) FileUtils.deletedelete(heapFile); final File tmpFile = new File(heapFile.getParentFile(), heapFile.getName() + ".prt"); HeapWriter dump; @@ -162,9 +164,9 @@ public final class ReferenceContainerCache exte } try { dump.close(true); - Log.logInfo("indexContainerRAMHeap", "finished rwi heap dump: " + wordcount + " words, " + urlcount + " word/URL relations in " + (System.currentTimeMillis() - startTime) + " milliseconds"); + log.logInfo("finished rwi heap dump: " + wordcount + " words, " + urlcount + " word/URL relations in " + (System.currentTimeMillis() - startTime) + " milliseconds"); } catch (final IOException e) { - Log.logSevere("indexContainerRAMHeap", "failed rwi heap dump: " + e.getMessage(), e); + log.logSevere("failed rwi heap dump: " + e.getMessage(), e); } finally { dump = null; } diff --git a/source/net/yacy/visualization/RasterPlotter.java b/source/net/yacy/visualization/RasterPlotter.java index d6eb08da8..728f0f594 100644 --- a/source/net/yacy/visualization/RasterPlotter.java +++ b/source/net/yacy/visualization/RasterPlotter.java @@ -49,6 +49,8 @@ import net.yacy.kelondro.util.ByteBuffer; public class RasterPlotter { + public final Log log = new Log("RasterPlotter"); + public static final double PI180 = Math.PI / 180.0d; // colors regarding RGB Color Model @@ -217,7 +219,7 @@ public class RasterPlotter { this.grid.setPixel(x, y, c); } } catch (ArrayIndexOutOfBoundsException e) { - Log.logWarning("RasterPlotter", e.getMessage() + ": x = " + x + ", y = " + y); + this.log.logWarning(e.getMessage() + ": x = " + x + ", y = " + y); } // may appear when pixel coordinate is out of bounds }