reduction of logging to prevent too much IO caused be logging

pull/1/head
Michael Peter Christen 13 years ago
parent 83da68c4c1
commit 58e7d1952f

@ -443,7 +443,6 @@ public final class CrawlStacker {
if (oldEntry == null) { if (oldEntry == null) {
if (dbocc != null) { if (dbocc != null) {
// do double-check // do double-check
if (this.log.isFine()) this.log.logFine("URL '" + urlstring + "' is double registered in '" + dbocc + "'.");
if (dbocc.equals("errors")) { if (dbocc.equals("errors")) {
final ZURL.Entry errorEntry = this.nextQueue.errorURL.get(url.hash()); final ZURL.Entry errorEntry = this.nextQueue.errorURL.get(url.hash());
return "double in: errors (" + errorEntry.anycause() + ")"; return "double in: errors (" + errorEntry.anycause() + ")";

@ -167,7 +167,7 @@ public class ZURL implements Iterable<ZURL.Entry> {
final Entry entry = new Entry(bentry, executor, workdate, workcount, reason); final Entry entry = new Entry(bentry, executor, workdate, workcount, reason);
put(entry); put(entry);
this.stack.add(entry.hash()); this.stack.add(entry.hash());
Log.logInfo("Rejected URL", bentry.url().toNormalform(false, false) + " - " + reason); if (!reason.startsWith("double")) Log.logInfo("Rejected URL", bentry.url().toNormalform(false, false) + " - " + reason);
if (this.solrConnector != null && failCategory.store) { if (this.solrConnector != null && failCategory.store) {
// send the error to solr // send the error to solr
try { try {

@ -292,7 +292,7 @@ public final class Switchboard extends serverSwitch
sb = this; sb = this;
// set loglevel and log // set loglevel and log
setLog(new Log("YACY_SEARCH")); setLog(new Log("SWITCHBOARD"));
// set default peer name // set default peer name
Seed.ANON_PREFIX = getConfig("peernameprefix", "_anon"); Seed.ANON_PREFIX = getConfig("peernameprefix", "_anon");
@ -1661,10 +1661,6 @@ public final class Switchboard extends serverSwitch
} }
// put document into the concurrent processing queue // put document into the concurrent processing queue
if ( this.log.isFinest() ) {
this.log.logFinest("deQueue: passing to indexing queue: "
+ response.url().toNormalform(true, false));
}
try { try {
this.indexingDocumentProcessor.enQueue(new indexingQueueEntry( this.indexingDocumentProcessor.enQueue(new indexingQueueEntry(
response, response,
@ -2249,11 +2245,6 @@ public final class Switchboard extends serverSwitch
public indexingQueueEntry parseDocument(final indexingQueueEntry in) { public indexingQueueEntry parseDocument(final indexingQueueEntry in) {
in.queueEntry.updateStatus(Response.QUEUE_STATE_PARSING); in.queueEntry.updateStatus(Response.QUEUE_STATE_PARSING);
// debug
if ( this.log.isFinest() ) {
this.log.logFinest("PARSE " + in.queueEntry);
}
Document[] documents = null; Document[] documents = null;
try { try {
documents = parseDocument(in.queueEntry); documents = parseDocument(in.queueEntry);
@ -2487,9 +2478,6 @@ public final class Switchboard extends serverSwitch
} }
in.documents = doclist.toArray(new Document[doclist.size()]); in.documents = doclist.toArray(new Document[doclist.size()]);
final Condenser[] condenser = new Condenser[in.documents.length]; final Condenser[] condenser = new Condenser[in.documents.length];
if ( this.log.isFine() ) {
this.log.logFine("Condensing for '" + in.queueEntry.url().toNormalform(false, true) + "'");
}
for ( int i = 0; i < in.documents.length; i++ ) { for ( int i = 0; i < in.documents.length; i++ ) {
condenser[i] = condenser[i] =
new Condenser(in.documents[i], in.queueEntry.profile().indexText(), in.queueEntry new Condenser(in.documents[i], in.queueEntry.profile().indexText(), in.queueEntry

Loading…
Cancel
Save