boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

collection of speed and memory saving hacks

Browse Source
pull/1/head
Michael Peter Christen 13 years ago
parent c00a3cf74d
commit f78ce93a80
15 changed files with 123 additions and 108 deletions
Show Stats Download Patch File Download Diff File

2
htroot/api/yacydoc.java
Unescape View File

@ -143,7 +143,7 @@ public class yacydoc {
references += r.toString()+",";
}
Log.logInfo ("TRIPLESTORE", references);
JenaTripleStore.log.logInfo(references);
prop.put("taglinks", references);

11
source/de/anomic/crawler/ResultImages.java
Unescape View File

@ -28,11 +28,11 @@ package de.anomic.crawler;
import java.util.Map;
import java.util.Queue;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.Set;
import java.util.concurrent.LinkedBlockingQueue;
import net.yacy.cora.document.MultiProtocolURI;
import net.yacy.cora.storage.SizeLimitedSet;
import net.yacy.document.Document;
import net.yacy.document.parser.html.ImageEntry;
import net.yacy.kelondro.data.meta.DigestURI;
@ -52,7 +52,7 @@ public class ResultImages {
// we also check all links for a double-check so we don't get the same image more than once in any queue
// image links may appear double here even if the pages where the image links are embedded already are checked for double-occurrence:
// the same images may be linked from different pages
private static final ConcurrentMap<MultiProtocolURI, Long> doubleCheck = new ConcurrentHashMap<MultiProtocolURI, Long>(); // (url, time) when the url appeared first
private static final Set<String> doubleCheck = new SizeLimitedSet<String>(10000);
public static void registerImages(final DigestURI source, final Document document, final boolean privateEntry) {
if (document == null) return;
@ -65,8 +65,9 @@ public class ResultImages {
for (final ImageEntry image: images.values()) {
// do a double-check; attention: this can be time-consuming since this possibly needs a DNS-lookup
if (image == null || image.url() == null) continue;
if (doubleCheck.containsKey(image.url())) continue;
doubleCheck.put(image.url(), System.currentTimeMillis());
String url = image.url().toNormalform(true, false);
if (doubleCheck.contains(url)) continue;
doubleCheck.add(url);
final String name = image.url().getFile();
boolean good = false;

9
source/de/anomic/crawler/RobotsTxt.java
Unescape View File

@ -267,12 +267,11 @@ public class RobotsTxt {
private static final int DOWNLOAD_MODDATE = 3;
static final String getHostPort(final MultiProtocolURI theURL) {
String urlHostPort = null;
final int port = getPort(theURL);
urlHostPort = theURL.getHost() + ":" + port;
urlHostPort = urlHostPort.toLowerCase().intern();
return urlHostPort;
String host = theURL.getHost();
StringBuilder sb = new StringBuilder(host.length() + 6);
sb.append(host).append(':').append(Integer.toString(port));
return sb.toString();
}
private static final int getPort(final MultiProtocolURI theURL) {

4
source/de/anomic/crawler/ZURL.java
Unescape View File

@ -55,6 +55,8 @@ import de.anomic.crawler.retrieval.Request;
public class ZURL implements Iterable<ZURL.Entry> {
public static Log log = new Log("REJECTED");
private static final int EcoFSBufferSize = 2000;
private static final int maxStackSize = 1000;
@ -167,7 +169,7 @@ public class ZURL implements Iterable<ZURL.Entry> {
final Entry entry = new Entry(bentry, executor, workdate, workcount, reason);
put(entry);
this.stack.add(entry.hash());
if (!reason.startsWith("double")) Log.logInfo("Rejected URL", bentry.url().toNormalform(false, false) + " - " + reason);
if (!reason.startsWith("double")) log.logInfo(bentry.url().toNormalform(false, false) + " - " + reason);
if (this.solrConnector != null && failCategory.store) {
// send the error to solr
try {

16
source/net/yacy/cora/document/MultiProtocolURI.java
Unescape View File

@ -158,7 +158,7 @@ public class MultiProtocolURI implements Serializable, Comparable<MultiProtocolU
url = "http://" + url;
p = 4;
}
this.protocol = url.substring(0, p).toLowerCase().trim();
this.protocol = url.substring(0, p).toLowerCase().trim().intern();
if (url.length() < p + 4) throw new MalformedURLException("URL not parseable: '" + url + "'");
if (!this.protocol.equals("file") && url.substring(p + 1, p + 3).equals("//")) {
// identify host, userInfo and file for http and ftp protocol
@ -166,20 +166,20 @@ public class MultiProtocolURI implements Serializable, Comparable<MultiProtocolU
int r;
if (q < 0) {
if ((r = url.indexOf('@', p + 3)) < 0) {
this.host = url.substring(p + 3);
this.host = url.substring(p + 3).intern();
this.userInfo = null;
} else {
this.host = url.substring(r + 1);
this.host = url.substring(r + 1).intern();
this.userInfo = url.substring(p + 3, r);
}
this.path = "/";
} else {
this.host = url.substring(p + 3, q).trim();
this.host = url.substring(p + 3, q).trim().intern();
if ((r = this.host.indexOf('@')) < 0) {
this.userInfo = null;
} else {
this.userInfo = this.host.substring(0, r);
this.host = this.host.substring(r + 1);
this.host = this.host.substring(r + 1).intern();
}
this.path = url.substring(q);
}
@ -2129,9 +2129,9 @@ public class MultiProtocolURI implements Serializable, Comparable<MultiProtocolU
String environment, url;
MultiProtocolURI aURL, aURL1;
java.net.URL jURL;
for (int i = 0; i < test.length; i++) {
environment = test[i][0];
url = test[i][1];
for (String[] element : test) {
environment = element[0];
url = element[1];
try {aURL = MultiProtocolURI.newURL(environment, url);} catch (final MalformedURLException e) {e.printStackTrace(); aURL = null;}
if (environment == null) {
try {jURL = new java.net.URL(url);} catch (final MalformedURLException e) {jURL = null;}

33
source/net/yacy/cora/lod/JenaTripleStore.java
Unescape View File

@ -37,6 +37,7 @@ import com.hp.hpl.jena.util.FileManager;
public class JenaTripleStore {
public static Log log = new Log("TRIPLESTORE");
public static Model model = ModelFactory.createDefaultModel();
static {
init(model);
@ -66,12 +67,12 @@ public class JenaTripleStore {
public static void loadRDF(String fileNameOrUri) throws IOException {
Model tmp = ModelFactory.createDefaultModel();
Log.logInfo("TRIPLESTORE", "Loading from " + fileNameOrUri);
log.logInfo("Loading from " + fileNameOrUri);
InputStream is = FileManager.get().open(fileNameOrUri);
if (is != null) {
// read the RDF/XML file
tmp.read(is, null);
Log.logInfo("TRIPLESTORE", "loaded " + tmp.size() + " triples from " + fileNameOrUri);
log.logInfo("loaded " + tmp.size() + " triples from " + fileNameOrUri);
model = model.union(tmp);
} else {
throw new IOException("cannot read " + fileNameOrUri);
@ -79,7 +80,7 @@ public class JenaTripleStore {
}
public static void LoadNTriples(String fileNameOrUri) throws IOException {
Log.logInfo("TRIPLESTORE", "Loading N-Triples from " + fileNameOrUri);
log.logInfo("Loading N-Triples from " + fileNameOrUri);
InputStream is = FileManager.get().open(fileNameOrUri);
LoadNTriples(is);
}
@ -88,7 +89,7 @@ public class JenaTripleStore {
Model tmp = ModelFactory.createDefaultModel();
if (is != null) {
tmp.read(is, null, "N-TRIPLE");
Log.logInfo("TRIPLESTORE", "loaded " + tmp.size() + " triples");
log.logInfo("loaded " + tmp.size() + " triples");
model = model.union(tmp);
//model.write(System.out, "TURTLE");
} else {
@ -117,10 +118,10 @@ public class JenaTripleStore {
File ftmp = new File(filename + "." + System.currentTimeMillis());
if (model.isEmpty() && !f.exists()) {
// we don't store zero-size models if they did not exist before
Log.logInfo("TRIPLESTORE", "NOT saving triplestore with " + model.size() + " triples to " + filename);
log.logInfo("NOT saving triplestore with " + model.size() + " triples to " + filename);
return;
}
Log.logInfo("TRIPLESTORE", "Saving triplestore with " + model.size() + " triples to " + filename);
log.logInfo("Saving triplestore with " + model.size() + " triples to " + filename);
OutputStream fout;
try {
fout = new BufferedOutputStream(new FileOutputStream(ftmp));
@ -132,9 +133,9 @@ public class JenaTripleStore {
if (!f.exists()) {
ftmp.renameTo(f);
}
Log.logInfo("TRIPLESTORE", "Saved triplestore with " + model.size() + " triples to " + filename);
log.logInfo("Saved triplestore with " + model.size() + " triples to " + filename);
} catch (Exception e) {
Log.logWarning("TRIPLESTORE", "Saving to " + filename+" failed");
log.logWarning("Saving to " + filename+" failed");
}
}
@ -175,14 +176,14 @@ public class JenaTripleStore {
Resource r = model.getResource(subject);
Property pr = model.getProperty(predicate);
r.addProperty(pr, object);
Log.logInfo("TRIPLESTORE", "ADD " + subject + " - " + predicate + " - " + object);
log.logInfo("ADD " + subject + " - " + predicate + " - " + object);
}
public static String getObject(final String subject, final String predicate) {
Iterator<RDFNode> ni = JenaTripleStore.getObjects(subject, predicate);
String object = "";
if (ni.hasNext()) object = ni.next().toString();
Log.logInfo("TRIPLESTORE", "GET " + subject + " - " + predicate + " - " + object);
log.logInfo("GET " + subject + " - " + predicate + " - " + object);
return object;
}
@ -195,7 +196,7 @@ public class JenaTripleStore {
Iterator<RDFNode> ni = JenaTripleStore.getPrivateObjects(subject, predicate, username);
String object = "";
if (ni.hasNext()) object = ni.next().toString();
Log.logInfo("TRIPLESTORE", "GET (" + username + ") " + subject + " - " + predicate + " - " + object);
log.logInfo("GET (" + username + ") " + subject + " - " + predicate + " - " + object);
return object;
}
@ -270,7 +271,7 @@ public class JenaTripleStore {
public static void initPrivateStores() {
Switchboard switchboard = Switchboard.getSwitchboard();
Log.logInfo("TRIPLESTORE", "Init private stores");
log.logInfo("Init private stores");
if (privatestorage == null) privatestorage = new ConcurrentHashMap<String, Model>();
if (privatestorage != null) privatestorage.clear();
@ -281,17 +282,17 @@ public class JenaTripleStore {
String username = e.getUserName();
File triplestore = new File(switchboard.getConfig("triplestore", new File(switchboard.getDataPath(), "DATA/TRIPLESTORE").getAbsolutePath()));
File currentuserfile = new File(triplestore, "private_store_"+username+".rdf");
Log.logInfo("TRIPLESTORE", "Init " + username + " from "+currentuserfile.getAbsolutePath());
log.logInfo("Init " + username + " from "+currentuserfile.getAbsolutePath());
Model tmp = ModelFactory.createDefaultModel();
init (tmp);
if (currentuserfile.exists()) {
Log.logInfo("TRIPLESTORE", "Loading from " + currentuserfile.getAbsolutePath());
log.logInfo("Loading from " + currentuserfile.getAbsolutePath());
InputStream is = FileManager.get().open(currentuserfile.getAbsolutePath());
if (is != null) {
// read the RDF/XML file
tmp.read(is, null);
Log.logInfo("TRIPLESTORE", "loaded " + tmp.size() + " triples from " + currentuserfile.getAbsolutePath());
log.logInfo("loaded " + tmp.size() + " triples from " + currentuserfile.getAbsolutePath());
} else {
throw new IOException("cannot read " + currentuserfile.getAbsolutePath());
}
@ -308,7 +309,7 @@ public class JenaTripleStore {
public static void savePrivateStores() {
Switchboard switchboard = Switchboard.getSwitchboard();
Log.logInfo("TRIPLESTORE", "Saving user triplestores");
log.logInfo("Saving user triplestores");
if (privatestorage == null) return;
for (Entry<String, Model> s : privatestorage.entrySet()) {
File triplestore = new File(switchboard.getConfig("triplestore", new File(switchboard.getDataPath(), "DATA/TRIPLESTORE").getAbsolutePath()));

1
source/net/yacy/cora/protocol/http/HTTPClient.java
Unescape View File

@ -598,6 +598,7 @@ public class HTTPClient {
assert !hrequest.expectContinue();
}
Thread.currentThread().setName("HTTPClient-" + httpUriRequest.getURI().getHost());
try {
final long time = System.currentTimeMillis();
this.httpResponse = httpClient.execute(httpUriRequest, httpContext);

5
source/net/yacy/kelondro/blob/HeapReader.java
Unescape View File

@ -58,7 +58,7 @@ public class HeapReader {
//public final static long keepFreeMem = 20 * 1024 * 1024;
private final static Log log = new Log("HeapReader");
// input values
protected int keylength; // the length of the primary key
protected File heapFile; // the file of the heap
@ -731,7 +731,7 @@ public class HeapReader {
public entries(final File blobFile, final int keylen) throws IOException {
if (!(blobFile.exists())) throw new IOException("file " + blobFile + " does not exist");
try {
this.is = new DataInputStream(new BufferedInputStream(new FileInputStream(blobFile), 8*1024*1024));
this.is = new DataInputStream(new BufferedInputStream(new FileInputStream(blobFile), 256 * 1024));
} catch (OutOfMemoryError e) {
this.is = new DataInputStream(new FileInputStream(blobFile));
}
@ -810,6 +810,7 @@ public class HeapReader {
}
}
@Override
public synchronized void close() {
if (this.is != null) try { this.is.close(); } catch (final IOException e) {Log.logException(e);}
this.is = null;

61
source/net/yacy/kelondro/blob/HeapWriter.java
Unescape View File

@ -7,7 +7,7 @@
// $LastChangedBy$
//
// LICENSE
//
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
@ -40,8 +40,9 @@ import net.yacy.kelondro.util.FileUtils;
public final class HeapWriter {
public final static Log log = new Log("HeapWriter");
public final static byte[] ZERO = new byte[]{0};
private final int keylength; // the length of the primary key
private HandleMap index; // key/seek relation for used records
private final File heapFileTMP; // the temporary file of the heap during writing
@ -49,7 +50,7 @@ public final class HeapWriter {
private DataOutputStream os; // the output stream where the BLOB is written
private long seek; // the current write position
//private HashSet<String> doublecheck;// only for testing
/*
* This class implements a BLOB management based on a sequence of records
* The data structure is:
@ -59,9 +60,9 @@ public final class HeapWriter {
* key :== <bytes as defined with keylen, if first byte is zero then record is empty>
* blob :== <bytes of length reclen - keylen>
* that means that each record has the size reclen+4
*
*
* Because the blob sizes are stored with integers, one entry may not exceed 2GB
*
*
* With this class a BLOB file can only be written.
* To read them, use a kelondroBLOBHeapReader.
* A BLOBHeap can be also read and write in random access mode with kelondroBLOBHeap.
@ -96,61 +97,61 @@ public final class HeapWriter {
* @param key
* @param blob
* @throws IOException
* @throws RowSpaceExceededException
* @throws RowSpaceExceededException
* @throws RowSpaceExceededException
* @throws RowSpaceExceededException
*/
public synchronized void add(byte[] key, final byte[] blob) throws IOException, RowSpaceExceededException {
assert blob.length > 0;
key = HeapReader.normalizeKey(key, this.keylength);
assert index.row().primaryKeyLength == this.keylength : index.row().primaryKeyLength + "!=" + key.length;
assert this.index.row().primaryKeyLength == this.keylength : this.index.row().primaryKeyLength + "!=" + key.length;
assert key.length == this.keylength : "key.length == " + key.length + ", this.keylength = " + this.keylength; // after normalizing they should be equal in length
assert index.get(key) < 0 : "index.get(key) = " + index.get(key) + ", index.size() = " + index.size() + ", file.length() = " + this.heapFileTMP.length() + ", key = " + UTF8.String(key); // must not occur before
assert this.index.get(key) < 0 : "index.get(key) = " + this.index.get(key) + ", index.size() = " + this.index.size() + ", file.length() = " + this.heapFileTMP.length() + ", key = " + UTF8.String(key); // must not occur before
if ((blob == null) || (blob.length == 0)) return;
index.putUnique(key, this.seek);
this.index.putUnique(key, this.seek);
int chunkl = this.keylength + blob.length;
os.writeInt(chunkl);
os.write(key);
os.write(blob);
this.os.writeInt(chunkl);
this.os.write(key);
this.os.write(blob);
this.seek += chunkl + 4;
//os.flush(); // necessary? may cause bad IO performance :-(
}
/**
* close the BLOB table
* @throws
* @throws
*/
public synchronized void close(boolean writeIDX) throws IOException {
// close the file
os.flush();
os.close();
os = null;
this.os.flush();
this.os.close();
this.os = null;
// rename the file into final name
if (this.heapFileREADY.exists()) FileUtils.deletedelete(this.heapFileREADY);
boolean renameok = this.heapFileTMP.renameTo(this.heapFileREADY);
if (!renameok) throw new IOException("cannot rename " + this.heapFileTMP + " to " + this.heapFileREADY);
if (!this.heapFileREADY.exists()) throw new IOException("renaming of " + this.heapFileREADY.toString() + " failed: files still exists");
if (this.heapFileTMP.exists()) throw new IOException("renaming to " + this.heapFileTMP.toString() + " failed: file does not exist");
// generate index and gap files
if (writeIDX && index.size() > 3) {
if (writeIDX && this.index.size() > 3) {
// now we can create a dump of the index and the gap information
// to speed up the next start
long start = System.currentTimeMillis();
String fingerprint = HeapReader.fingerprintFileHash(this.heapFileREADY);
if (fingerprint == null) {
Log.logSevere("kelondroBLOBHeapWriter", "cannot write a dump for " + heapFileREADY.getName()+ ": fingerprint is null");
log.logSevere("cannot write a dump for " + this.heapFileREADY.getName()+ ": fingerprint is null");
} else {
new Gap().dump(fingerprintGapFile(this.heapFileREADY, fingerprint));
index.dump(fingerprintIndexFile(this.heapFileREADY, fingerprint));
Log.logInfo("kelondroBLOBHeapWriter", "wrote a dump for the " + this.index.size() + " index entries of " + heapFileREADY.getName()+ " in " + (System.currentTimeMillis() - start) + " milliseconds.");
this.index.dump(fingerprintIndexFile(this.heapFileREADY, fingerprint));
log.logInfo("wrote a dump for the " + this.index.size() + " index entries of " + this.heapFileREADY.getName()+ " in " + (System.currentTimeMillis() - start) + " milliseconds.");
}
index.close();
index = null;
this.index.close();
this.index = null;
} else {
// this is small.. just free resources, do not write index
index.close();
index = null;
this.index.close();
this.index = null;
}
}
@ -165,12 +166,12 @@ public final class HeapWriter {
FileUtils.deletedelete(new File(p, s));
}
}
protected static File fingerprintIndexFile(File f, String fingerprint) {
assert f != null;
return new File(f.getParentFile(), f.getName() + "." + fingerprint + ".idx");
}
protected static File fingerprintGapFile(File f, String fingerprint) {
assert f != null;
return new File(f.getParentFile(), f.getName() + "." + fingerprint + ".gap");

4
source/net/yacy/kelondro/data/word/Word.java
Unescape View File

@ -56,13 +56,15 @@ public class Word {
*/
public static final int commonHashLength = 12;
private static final int hashCacheSize = Math.max(100000, Math.min(1000000, (int) (MemoryControl.available() / 40000L)));
private static final int hashCacheSize = Math.max(20000, Math.min(200000, (int) (MemoryControl.available() / 40000L)));
private static ARC<String, byte[]> hashCache = null;
static {
try {
hashCache = new ConcurrentARC<String, byte[]>(hashCacheSize, Math.max(32, 4 * Runtime.getRuntime().availableProcessors()));
Log.logInfo("Word", "hashCache.size = " + hashCacheSize);
} catch (final OutOfMemoryError e) {
hashCache = new ConcurrentARC<String, byte[]>(1000, Math.max(8, 2 * Runtime.getRuntime().availableProcessors()));
Log.logInfo("Word", "hashCache.size = " + 1000);
}
}
/*

3
source/net/yacy/kelondro/logging/ConsoleOutHandler.java
Unescape View File

@ -27,6 +27,7 @@
package net.yacy.kelondro.logging;
import java.io.BufferedOutputStream;
import java.util.logging.Level;
import java.util.logging.LogRecord;
import java.util.logging.SimpleFormatter;
@ -39,7 +40,7 @@ public final class ConsoleOutHandler extends StreamHandler {
public ConsoleOutHandler() {
setLevel(Level.FINEST);
setFormatter(new SimpleFormatter());
setOutputStream(System.out);
setOutputStream(new BufferedOutputStream(System.out));
}
@Override

22
source/net/yacy/kelondro/logging/Log.java
Unescape View File

@ -288,37 +288,37 @@ public final class Log {
}
protected final static class logEntry {
public final Logger logger;
public final String loggername;
public final Level level;
public final String message;
public final Throwable thrown;
public Logger logger;
public String loggername;
public Throwable thrown;
private logEntry(final Level level, final String message) {
this.level = level;
this.message = message == null || message.length() <= 120 ? message : message.substring(0, 120);
}
public logEntry(final Logger logger, final Level level, final String message, final Throwable thrown) {
this(level, message);
this.logger = logger;
this.loggername = null;
this.level = level;
this.message = message;
this.thrown = thrown;
}
public logEntry(final Logger logger, final Level level, final String message) {
this(level, message);
this.logger = logger;
this.loggername = null;
this.level = level;
this.message = message;
this.thrown = null;
}
public logEntry(final String loggername, final Level level, final String message, final Throwable thrown) {
this(level, message);
this.logger = null;
this.loggername = loggername;
this.level = level;
this.message = message;
this.thrown = thrown;
}
public logEntry(final String loggername, final Level level, final String message) {
this(level, message);
this.logger = null;
this.loggername = loggername;
this.level = level;
this.message = message;
this.thrown = null;
}
public logEntry() {

48
source/net/yacy/kelondro/rwi/IODispatcher.java
Unescape View File

@ -48,6 +48,8 @@ import net.yacy.kelondro.util.MemoryControl;
*/
public class IODispatcher extends Thread {
public static final Log log = new Log("IODispatcher");
private Semaphore controlQueue;
private final Semaphore termination;
private ArrayBlockingQueue<MergeJob> mergeQueue;
@ -81,7 +83,7 @@ public class IODispatcher extends Thread {
@SuppressWarnings("unchecked")
protected synchronized void dump(final ReferenceContainerCache<? extends Reference> cache, final File file, final ReferenceContainerArray<? extends Reference> array) {
if (this.dumpQueue == null || this.controlQueue == null || !isAlive()) {
Log.logWarning("IODispatcher", "emergency dump of file " + file.getName());
log.logWarning("emergency dump of file " + file.getName());
if (!cache.isEmpty()) cache.dump(file, (int) Math.min(MemoryControl.available() / 3, this.writeBufferSize), true);
} else {
@SuppressWarnings("rawtypes")
@ -91,7 +93,7 @@ public class IODispatcher extends Thread {
if (isAlive()) {
try {
this.dumpQueue.put(job);
Log.logInfo("IODispatcher", "appended dump job for file " + file.getName());
log.logInfo("appended dump job for file " + file.getName());
} catch (final InterruptedException e) {
Log.logException(e);
cache.dump(file, (int) Math.min(MemoryControl.available() / 3, this.writeBufferSize), true);
@ -100,7 +102,7 @@ public class IODispatcher extends Thread {
}
} else {
job.dump();
Log.logWarning("IODispatcher", "dispatcher is not alive, just dumped file " + file.getName());
log.logWarning("dispatcher is not alive, just dumped file " + file.getName());
}
}
}
@ -112,9 +114,9 @@ public class IODispatcher extends Thread {
protected synchronized void merge(final File f1, final File f2, final ReferenceFactory<? extends Reference> factory, final ArrayStack array, final File newFile) {
if (this.mergeQueue == null || this.controlQueue == null || !isAlive()) {
if (f2 == null) {
Log.logWarning("IODispatcher", "emergency rewrite of file " + f1.getName() + " to " + newFile.getName());
log.logWarning("emergency rewrite of file " + f1.getName() + " to " + newFile.getName());
} else {
Log.logWarning("IODispatcher", "emergency merge of files " + f1.getName() + ", " + f2.getName() + " to " + newFile.getName());
log.logWarning("emergency merge of files " + f1.getName() + ", " + f2.getName() + " to " + newFile.getName());
}
array.mergeMount(f1, f2, factory, newFile, (int) Math.min(MemoryControl.available() / 3, this.writeBufferSize));
} else {
@ -123,12 +125,12 @@ public class IODispatcher extends Thread {
try {
this.mergeQueue.put(job);
if (f2 == null) {
Log.logInfo("IODispatcher", "appended rewrite job of file " + f1.getName() + " to " + newFile.getName());
log.logInfo("appended rewrite job of file " + f1.getName() + " to " + newFile.getName());
} else {
Log.logInfo("IODispatcher", "appended merge job of files " + f1.getName() + ", " + f2.getName() + " to " + newFile.getName());
log.logInfo("appended merge job of files " + f1.getName() + ", " + f2.getName() + " to " + newFile.getName());
}
} catch (final InterruptedException e) {
Log.logWarning("IODispatcher", "interrupted: " + e.getMessage(), e);
log.logWarning("interrupted: " + e.getMessage(), e);
array.mergeMount(f1, f2, factory, newFile, (int) Math.min(MemoryControl.available() / 3, this.writeBufferSize));
} finally {
this.controlQueue.release();
@ -136,9 +138,9 @@ public class IODispatcher extends Thread {
} else {
job.merge();
if (f2 == null) {
Log.logWarning("IODispatcher", "dispatcher not running, merged files " + f1.getName() + " to " + newFile.getName());
log.logWarning("dispatcher not running, merged files " + f1.getName() + " to " + newFile.getName());
} else {
Log.logWarning("IODispatcher", "dispatcher not running, rewrote file " + f1.getName() + ", " + f2.getName() + " to " + newFile.getName());
log.logWarning("dispatcher not running, rewrote file " + f1.getName() + ", " + f2.getName() + " to " + newFile.getName());
}
}
}
@ -160,10 +162,10 @@ public class IODispatcher extends Thread {
f = dumpJob.file;
dumpJob.dump();
} catch (final InterruptedException e) {
Log.logSevere("IODispatcher", "main run job was interrupted (1)", e);
log.logSevere("main run job was interrupted (1)", e);
Log.logException(e);
} catch (final Throwable e) {
Log.logSevere("IODispatcher", "main run job had errors (1), dump to " + f + " failed.", e);
log.logSevere("main run job had errors (1), dump to " + f + " failed.", e);
Log.logException(e);
}
continue loop;
@ -179,13 +181,13 @@ public class IODispatcher extends Thread {
f2 = mergeJob.f2;
mergeJob.merge();
} catch (final InterruptedException e) {
Log.logSevere("IODispatcher", "main run job was interrupted (2)", e);
log.logSevere("main run job was interrupted (2)", e);
Log.logException(e);
} catch (final Throwable e) {
if (f2 == null) {
Log.logSevere("IODispatcher", "main run job had errors (2), dump to " + f + " failed. Input file is " + f1, e);
log.logSevere("main run job had errors (2), dump to " + f + " failed. Input file is " + f1, e);
} else {
Log.logSevere("IODispatcher", "main run job had errors (2), dump to " + f + " failed. Input files are " + f1 + " and " + f2, e);
log.logSevere("main run job had errors (2), dump to " + f + " failed. Input files are " + f1 + " and " + f2, e);
}
Log.logException(e);
}
@ -194,22 +196,22 @@ public class IODispatcher extends Thread {
// check termination
if (this.terminate) {
Log.logInfo("IODispatcher", "caught termination signal");
log.logInfo("caught termination signal");
break;
}
Log.logSevere("IODispatcher", "main loop in bad state, dumpQueue.size() = " + this.dumpQueue.size() + ", mergeQueue.size() = " + this.mergeQueue.size() + ", controlQueue.availablePermits() = " + this.controlQueue.availablePermits());
log.logSevere("main loop in bad state, dumpQueue.size() = " + this.dumpQueue.size() + ", mergeQueue.size() = " + this.mergeQueue.size() + ", controlQueue.availablePermits() = " + this.controlQueue.availablePermits());
assert false : "this process statt should not be reached"; // this should never happen
} catch (final Throwable e) {
Log.logSevere("IODispatcher", "main run job failed (X)", e);
log.logSevere("main run job failed (X)", e);
Log.logException(e);
}
Log.logInfo("IODispatcher", "loop terminated");
log.logInfo("loop terminated");
} catch (final Throwable e) {
Log.logSevere("IODispatcher", "main run job failed (4)", e);
log.logSevere("main run job failed (4)", e);
Log.logException(e);
} finally {
Log.logInfo("IODispatcher", "terminating run job");
log.logInfo("terminating run job");
this.controlQueue = null;
this.dumpQueue = null;
this.mergeQueue = null;
@ -257,11 +259,11 @@ public class IODispatcher extends Thread {
private File merge() {
if (!this.f1.exists()) {
Log.logWarning("IODispatcher", "merge of file (1) " + this.f1.getName() + " failed: file does not exists");
log.logWarning("merge of file (1) " + this.f1.getName() + " failed: file does not exists");
return null;
}
if (this.f2 != null && !this.f2.exists()) {
Log.logWarning("IODispatcher", "merge of file (2) " + this.f2.getName() + " failed: file does not exists");
log.logWarning("merge of file (2) " + this.f2.getName() + " failed: file does not exists");
return null;
}
return this.array.mergeMount(this.f1, this.f2, this.factory, this.newFile, (int) Math.min(MemoryControl.available() / 3, IODispatcher.this.writeBufferSize));

8
source/net/yacy/kelondro/rwi/ReferenceContainerCache.java
Unescape View File

@ -59,6 +59,8 @@ import net.yacy.kelondro.util.FileUtils;
*/
public final class ReferenceContainerCache<ReferenceType extends Reference> extends AbstractIndex<ReferenceType> implements Index<ReferenceType>, IndexReader<ReferenceType>, Iterable<ReferenceContainer<ReferenceType>> {
public static final Log log = new Log("ReferenceContainerCache");
private final int termSize;
private final ByteOrder termOrder;
private final ContainerOrder<ReferenceType> containerOrder;
@ -117,7 +119,7 @@ public final class ReferenceContainerCache<ReferenceType extends Reference> exte
public void dump(final File heapFile, final int writeBuffer, final boolean destructive) {
assert this.cache != null;
if (this.cache == null) return;
Log.logInfo("indexContainerRAMHeap", "creating rwi heap dump '" + heapFile.getName() + "', " + this.cache.size() + " rwi's");
log.logInfo("creating rwi heap dump '" + heapFile.getName() + "', " + this.cache.size() + " rwi's");
if (heapFile.exists()) FileUtils.deletedelete(heapFile);
final File tmpFile = new File(heapFile.getParentFile(), heapFile.getName() + ".prt");
HeapWriter dump;
@ -162,9 +164,9 @@ public final class ReferenceContainerCache<ReferenceType extends Reference> exte
}
try {
dump.close(true);
Log.logInfo("indexContainerRAMHeap", "finished rwi heap dump: " + wordcount + " words, " + urlcount + " word/URL relations in " + (System.currentTimeMillis() - startTime) + " milliseconds");
log.logInfo("finished rwi heap dump: " + wordcount + " words, " + urlcount + " word/URL relations in " + (System.currentTimeMillis() - startTime) + " milliseconds");
} catch (final IOException e) {
Log.logSevere("indexContainerRAMHeap", "failed rwi heap dump: " + e.getMessage(), e);
log.logSevere("failed rwi heap dump: " + e.getMessage(), e);
} finally {
dump = null;
}

4
source/net/yacy/visualization/RasterPlotter.java
Unescape View File

@ -49,6 +49,8 @@ import net.yacy.kelondro.util.ByteBuffer;
public class RasterPlotter {
public final Log log = new Log("RasterPlotter");
public static final double PI180 = Math.PI / 180.0d;
// colors regarding RGB Color Model
@ -217,7 +219,7 @@ public class RasterPlotter {
this.grid.setPixel(x, y, c);
}
} catch (ArrayIndexOutOfBoundsException e) {
Log.logWarning("RasterPlotter", e.getMessage() + ": x = " + x + ", y = " + y);
this.log.logWarning(e.getMessage() + ": x = " + x + ", y = " + y);
} // may appear when pixel coordinate is out of bounds
}

Write Preview
Loading…
Cancel
Save