From 228b04b499137a37ae6f1a259180e5a96e1f031e Mon Sep 17 00:00:00 2001 From: theli Date: Tue, 2 Aug 2005 15:36:10 +0000 Subject: [PATCH] *) Bugfix for "wrong seed-upload timestamp" problem http://www.yacy-forum.de/viewtopic.php?t=817 git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@480 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- htroot/Performance_p.java | 88 ++++----- htroot/SettingsAck_p.java | 6 +- htroot/Status.java | 2 +- .../de/anomic/plasma/plasmaSwitchboard.java | 5 +- source/de/anomic/yacy/yacyCore.java | 179 +++++++++--------- 5 files changed, 141 insertions(+), 139 deletions(-) diff --git a/htroot/Performance_p.java b/htroot/Performance_p.java index f5d8a5f26..77c0032cb 100644 --- a/htroot/Performance_p.java +++ b/htroot/Performance_p.java @@ -1,47 +1,47 @@ -// Performace_p.java -// ----------------------- -// part of YaCy -// (C) by Michael Peter Christen; mc@anomic.de -// first published on http://www.anomic.de -// Frankfurt, Germany, 2004, 2005 -// last major change: 16.02.2005 +//Performace_p.java +//----------------------- +//part of YaCy +//(C) by Michael Peter Christen; mc@anomic.de +//first published on http://www.anomic.de +//Frankfurt, Germany, 2004, 2005 +//last major change: 16.02.2005 // -// This program is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. +//This program is free software; you can redistribute it and/or modify +//it under the terms of the GNU General Public License as published by +//the Free Software Foundation; either version 2 of the License, or +//(at your option) any later version. // -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. +//This program is distributed in the hope that it will be useful, +//but WITHOUT ANY WARRANTY; without even the implied warranty of +//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +//GNU General Public License for more details. // -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +//You should have received a copy of the GNU General Public License +//along with this program; if not, write to the Free Software +//Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA // -// Using this software in any meaning (reading, learning, copying, compiling, -// running) means that you agree that the Author(s) is (are) not responsible -// for cost, loss of data or any harm that may be caused directly or indirectly -// by usage of this softare or this documentation. The usage of this software -// is on your own risk. The installation and usage (starting/running) of this -// software may allow other people or application to access your computer and -// any attached devices and is highly dependent on the configuration of the -// software which must be done by the user of the software; the author(s) is -// (are) also not responsible for proper configuration and usage of the -// software, even if provoked by documentation provided together with -// the software. +//Using this software in any meaning (reading, learning, copying, compiling, +//running) means that you agree that the Author(s) is (are) not responsible +//for cost, loss of data or any harm that may be caused directly or indirectly +//by usage of this softare or this documentation. The usage of this software +//is on your own risk. The installation and usage (starting/running) of this +//software may allow other people or application to access your computer and +//any attached devices and is highly dependent on the configuration of the +//software which must be done by the user of the software; the author(s) is +//(are) also not responsible for proper configuration and usage of the +//software, even if provoked by documentation provided together with +//the software. // -// Any changes to this file according to the GPL as documented in the file -// gpl.txt aside this file in the shipment you received can be done to the -// lines that follows this copyright notice here, but changes must not be -// done inside the copyright notive above. A re-distribution must contain -// the intact and unchanged copyright notice. -// Contributions and changes to the program code must be marked as such. +//Any changes to this file according to the GPL as documented in the file +//gpl.txt aside this file in the shipment you received can be done to the +//lines that follows this copyright notice here, but changes must not be +//done inside the copyright notive above. A re-distribution must contain +//the intact and unchanged copyright notice. +//Contributions and changes to the program code must be marked as such. -// You must compile this file with -// javac -classpath .:../classes Network.java -// if the shell's current path is HTROOT +//You must compile this file with +//javac -classpath .:../classes Network.java +//if the shell's current path is HTROOT import java.util.Iterator; import java.util.Map; @@ -70,7 +70,7 @@ public class Performance_p { Iterator threads = switchboard.threadNames(); String threadName; serverThread thread; - + // calculate totals long blocktime_total = 0, sleeptime_total = 0, exectime_total = 0; while (threads.hasNext()) { @@ -127,10 +127,10 @@ public class Performance_p { busysleep = Long.parseLong((String) post.get(threadName + "_busysleep", "100")); memprereq = Long.parseLong((String) post.get(threadName + "_memprereq", "0")); - // check values to prevent short-cut loops - if (idlesleep < 1000) idlesleep = 1000; + // check values to prevent short-cut loops + if (idlesleep < 1000) idlesleep = 1000; if (threadName.equals("10_httpd")) { idlesleep = 0; busysleep = 0; memprereq = 0; } - + // on-the-fly re-configuration switchboard.setThreadPerformance(threadName, idlesleep, busysleep, memprereq); switchboard.setConfig(threadName + "_idlesleep", idlesleep); @@ -142,8 +142,8 @@ public class Performance_p { busysleep = Long.parseLong(d((String) defaultSettings.get(threadName + "_busysleep"), "100")); memprereq = Long.parseLong(d((String) defaultSettings.get(threadName + "_memprereq"), "0")); - // check values to prevent short-cut loops - if (idlesleep < 1000) idlesleep = 1000; + // check values to prevent short-cut loops + if (idlesleep < 1000) idlesleep = 1000; if (threadName.equals("10_httpd")) { idlesleep = 0; busysleep = 0; memprereq = 0; } if ((threadName.equals("50_localcrawl")) && (busysleep < 100)) busysleep = 100; if ((threadName.equals("61_globalcrawltrigger")) && (busysleep < 100)) busysleep = 100; diff --git a/htroot/SettingsAck_p.java b/htroot/SettingsAck_p.java index 638f7e5ae..53373ec8d 100644 --- a/htroot/SettingsAck_p.java +++ b/htroot/SettingsAck_p.java @@ -346,7 +346,7 @@ public class SettingsAck_p { if (post.containsKey("seedUploadRetry")) { String error; - if ((error = yacyCore.saveSeedList(env)) == null) { + if ((error = ((plasmaSwitchboard)env).yc.saveSeedList(env)) == null) { // trying to upload the seed-list file prop.put("info", 13); prop.put("info_success",1); @@ -377,7 +377,7 @@ public class SettingsAck_p { // try an upload String error; - if ((error = yacyCore.saveSeedList(env)) == null) { + if ((error = ((plasmaSwitchboard)env).yc.saveSeedList(env)) == null) { // we have successfully uploaded the seed-list file prop.put("info_seedUploadMethod",newSeedUploadMethod); prop.put("info_seedURL",newSeedURLStr); @@ -423,7 +423,7 @@ public class SettingsAck_p { // were changed, we now try to upload the seed list with the new settings if (env.getConfig("seedUploadMethod","none").equalsIgnoreCase(uploaderName)) { String error; - if ((error = yacyCore.saveSeedList(env)) == null) {; + if ((error = ((plasmaSwitchboard)env).yc.saveSeedList(env)) == null) {; // we have successfully uploaded the seed file prop.put("info", 13); diff --git a/htroot/Status.java b/htroot/Status.java index 9a350ee0a..e429628ba 100644 --- a/htroot/Status.java +++ b/htroot/Status.java @@ -179,7 +179,7 @@ public class Status { prop.put("seedServer_seedFile", env.getConfig("seedFilePath","")); } prop.put("seedServer_lastUpload", - serverDate.intervalToString(System.currentTimeMillis()-yacyCore.lastSeedUpload_timeStamp)); + serverDate.intervalToString(System.currentTimeMillis()-((plasmaSwitchboard)env).yc.lastSeedUpload_timeStamp)); } else { prop.put("seedServer", 0);//disabled } diff --git a/source/de/anomic/plasma/plasmaSwitchboard.java b/source/de/anomic/plasma/plasmaSwitchboard.java index 967b893d9..391c3e3e8 100644 --- a/source/de/anomic/plasma/plasmaSwitchboard.java +++ b/source/de/anomic/plasma/plasmaSwitchboard.java @@ -156,7 +156,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser public static TreeSet stopwords = null; public static plasmaURLPattern urlBlacklist; - // storage management + // storage management private File cachePath; private File plasmaPath; public File listsPath; @@ -181,6 +181,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser public plasmaParser parser; public plasmaWordIndexClassicCacheMigration classicCache; public long proxyLastAccess; + public yacyCore yc; private serverSemaphore shutdownSync = new serverSemaphore(0); private boolean terminate = false; @@ -342,7 +343,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser // start yacy core log.logSystem("Starting YaCy Protocol Core"); //try{Thread.currentThread().sleep(5000);} catch (InterruptedException e) {} // for profiler - yacyCore yc = new yacyCore(this); + this.yc = new yacyCore(this); //log.logSystem("Started YaCy Protocol Core"); //System.gc(); try{Thread.currentThread().sleep(5000);} catch (InterruptedException e) {} // for profiler serverInstantThread.oneTimeJob(yc, "loadSeeds", yc.log, 3000); diff --git a/source/de/anomic/yacy/yacyCore.java b/source/de/anomic/yacy/yacyCore.java index 0f059b1ae..db4d51028 100644 --- a/source/de/anomic/yacy/yacyCore.java +++ b/source/de/anomic/yacy/yacyCore.java @@ -98,7 +98,7 @@ public class yacyCore { // class variables private int lastSeedUpload_seedDBSize = 0; - public static long lastSeedUpload_timeStamp = System.currentTimeMillis(); + public long lastSeedUpload_timeStamp = System.currentTimeMillis(); private String lastSeedUpload_myPeerType = ""; private String lastSeedUpload_myIP = ""; @@ -238,7 +238,7 @@ public class yacyCore { (this.lastSeedUpload_myIP.equals(seedDB.mySeed.get("IP", "127.0.0.1"))) && (this.lastSeedUpload_seedDBSize == seedDB.sizeConnected()) && (canReachMyself()) && - (System.currentTimeMillis() - yacyCore.lastSeedUpload_timeStamp < 1000*60*60*24) && + (System.currentTimeMillis() - this.lastSeedUpload_timeStamp < 1000*60*60*24) && (seedDB.mySeed.isPrincipal()) ) { log.logDebug("yacyCore.publishSeedList: not necessary to publish: oldIP is equal, sizeConnected is equal and I can reach myself under the old IP."); @@ -260,14 +260,7 @@ public class yacyCore { this.switchboard.setConfig("seedUploadMethod","File"); } // we want to be a principal... - saveSeedList(); - - this.lastSeedUpload_seedDBSize = seedDB.sizeConnected(); - this.lastSeedUpload_timeStamp = System.currentTimeMillis(); - - this.lastSeedUpload_myIP = seedDB.mySeed.get("IP", "127.0.0.1"); - this.lastSeedUpload_myPeerType = seedDB.mySeed.get("PeerType", yacySeed.PEERTYPE_JUNIOR); - + saveSeedList(); } else { if (seedUploadMethod.equals("")) this.switchboard.setConfig("seedUploadMethod","none"); log.logDebug("yacyCore.publishSeedList: No uploading method configured"); @@ -629,91 +622,99 @@ public class yacyCore { return saveSeedList(this.switchboard); } - public static String saveSeedList(serverSwitch sb) { - // return an error if this is not successful, and NULL if everything is fine - String logt; - - // be shure that we have something to say - if (seedDB.mySeed.getAddress() == null) { - String errorMsg = "We have no valid IP address until now"; - log.logWarning("SaveSeedList: " + errorMsg); - return errorMsg; - } - - // getting the configured seed uploader - String seedUploadMethod = sb.getConfig("seedUploadMethod",""); - - // for backward compatiblity .... - if ( - (seedUploadMethod.equalsIgnoreCase("Ftp")) || - ((seedUploadMethod.equals("")) && - (sb.getConfig("seedFTPPassword","").length() > 0)) - ) { - seedUploadMethod = "Ftp"; - sb.setConfig("seedUploadMethod",seedUploadMethod); - } else if ( - (seedUploadMethod.equalsIgnoreCase("File")) || - ((seedUploadMethod.equals("")) && - (sb.getConfig("seedFilePath", "").length() > 0)) - ) { - seedUploadMethod = "File"; - sb.setConfig("seedUploadMethod",seedUploadMethod); - } - - // determine the seed uploader that should be used ... - if (seedUploadMethod.equalsIgnoreCase("none")) return "no uploader specified"; - - yacySeedUploader uploader = getSeedUploader(seedUploadMethod); - if (uploader == null) { - String errorMsg = "Unable to get the proper uploader-class for seed uploading method '" + seedUploadMethod + "'."; - log.logWarning("SaveSeedList: " + errorMsg); - return errorMsg; - } - - // ensure that the seed file url is configured properly - URL seedURL; - try{ - String seedURLStr = sb.getConfig("seedURL",""); - if (seedURLStr.length() == 0) throw new MalformedURLException("The seed-file url must not be empty."); - if (!seedURLStr.toLowerCase().startsWith("http://")) throw new MalformedURLException("Unsupported protocol."); - seedURL = new URL(seedURLStr); - }catch(MalformedURLException e){ - String errorMsg = "Malformed seed file URL '" + sb.getConfig("seedURL","") + "'. " + e.getMessage(); - log.logWarning("SaveSeedList: " + errorMsg); - return errorMsg; - } - - // upload the seed-list using the configured uploader class - String prevStatus = seedDB.mySeed.get("PeerType", "junior"); - if (prevStatus.equals("principal")) prevStatus = "senior"; - + public String saveSeedList(serverSwitch sb) { try { - seedDB.mySeed.put("PeerType", "principal"); // this information shall also be uploaded + // return an error if this is not successful, and NULL if everything is fine + String logt; - log.logDebug("SaveSeedList: Using seed uploading method '" + seedUploadMethod + "' for seed-list uploading." + - "\n\tPrevious peerType is '" + seedDB.mySeed.get("PeerType", "junior") + "'."); + // be shure that we have something to say + if (seedDB.mySeed.getAddress() == null) { + String errorMsg = "We have no valid IP address until now"; + log.logWarning("SaveSeedList: " + errorMsg); + return errorMsg; + } + + // getting the configured seed uploader + String seedUploadMethod = sb.getConfig("seedUploadMethod",""); + + // for backward compatiblity .... + if ( + (seedUploadMethod.equalsIgnoreCase("Ftp")) || + ((seedUploadMethod.equals("")) && + (sb.getConfig("seedFTPPassword","").length() > 0)) + ) { + seedUploadMethod = "Ftp"; + sb.setConfig("seedUploadMethod",seedUploadMethod); + } else if ( + (seedUploadMethod.equalsIgnoreCase("File")) || + ((seedUploadMethod.equals("")) && + (sb.getConfig("seedFilePath", "").length() > 0)) + ) { + seedUploadMethod = "File"; + sb.setConfig("seedUploadMethod",seedUploadMethod); + } + + // determine the seed uploader that should be used ... + if (seedUploadMethod.equalsIgnoreCase("none")) return "no uploader specified"; - //logt = seedDB.uploadCache(seedFTPServer, seedFTPAccount, seedFTPPassword, seedFTPPath, seedURL); - logt = seedDB.uploadCache(uploader,sb, seedDB, seedURL); - if (logt != null) { - if (logt.indexOf("Error") >= 0) { - seedDB.mySeed.put("PeerType", prevStatus); - String errorMsg = "SaveSeedList: seed upload failed using " + uploader.getClass().getName() + " (error): " + logt.substring(logt.indexOf("Error") + 6); - log.logError(errorMsg); - return errorMsg; + yacySeedUploader uploader = getSeedUploader(seedUploadMethod); + if (uploader == null) { + String errorMsg = "Unable to get the proper uploader-class for seed uploading method '" + seedUploadMethod + "'."; + log.logWarning("SaveSeedList: " + errorMsg); + return errorMsg; + } + + // ensure that the seed file url is configured properly + URL seedURL; + try{ + String seedURLStr = sb.getConfig("seedURL",""); + if (seedURLStr.length() == 0) throw new MalformedURLException("The seed-file url must not be empty."); + if (!seedURLStr.toLowerCase().startsWith("http://")) throw new MalformedURLException("Unsupported protocol."); + seedURL = new URL(seedURLStr); + }catch(MalformedURLException e){ + String errorMsg = "Malformed seed file URL '" + sb.getConfig("seedURL","") + "'. " + e.getMessage(); + log.logWarning("SaveSeedList: " + errorMsg); + return errorMsg; + } + + // upload the seed-list using the configured uploader class + String prevStatus = seedDB.mySeed.get("PeerType", "junior"); + if (prevStatus.equals("principal")) prevStatus = "senior"; + + try { + seedDB.mySeed.put("PeerType", "principal"); // this information shall also be uploaded + + log.logDebug("SaveSeedList: Using seed uploading method '" + seedUploadMethod + "' for seed-list uploading." + + "\n\tPrevious peerType is '" + seedDB.mySeed.get("PeerType", "junior") + "'."); + + //logt = seedDB.uploadCache(seedFTPServer, seedFTPAccount, seedFTPPassword, seedFTPPath, seedURL); + logt = seedDB.uploadCache(uploader,sb, seedDB, seedURL); + if (logt != null) { + if (logt.indexOf("Error") >= 0) { + seedDB.mySeed.put("PeerType", prevStatus); + String errorMsg = "SaveSeedList: seed upload failed using " + uploader.getClass().getName() + " (error): " + logt.substring(logt.indexOf("Error") + 6); + log.logError(errorMsg); + return errorMsg; + } + log.logInfo(logt); } - log.logInfo(logt); + + // finally, set the principal status + sb.setConfig("yacyStatus","principal"); + return null; + } catch (Exception e) { + seedDB.mySeed.put("PeerType", prevStatus); + sb.setConfig("yacyStatus", prevStatus); + String errorMsg = "SaveSeedList: Seed upload failed (IO error): " + e.getMessage(); + log.logInfo(errorMsg); + return errorMsg; } + } finally { + this.lastSeedUpload_seedDBSize = seedDB.sizeConnected(); + this.lastSeedUpload_timeStamp = System.currentTimeMillis(); - // finally, set the principal status - sb.setConfig("yacyStatus","principal"); - return null; - } catch (Exception e) { - seedDB.mySeed.put("PeerType", prevStatus); - sb.setConfig("yacyStatus", prevStatus); - String errorMsg = "SaveSeedList: Seed upload failed (IO error): " + e.getMessage(); - log.logInfo(errorMsg); - return errorMsg; + this.lastSeedUpload_myIP = seedDB.mySeed.get("IP", "127.0.0.1"); + this.lastSeedUpload_myPeerType = seedDB.mySeed.get("PeerType", yacySeed.PEERTYPE_JUNIOR); } }