From ea473e32b8dcf13fce152f2518e48a8a8359ca3a Mon Sep 17 00:00:00 2001 From: orbiter Date: Fri, 9 Oct 2009 22:27:50 +0000 Subject: [PATCH] refactoring git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6390 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- debian/changelog | 2 +- source/de/anomic/kelondro/text/Segments.java | 4 +- source/de/anomic/search/Switchboard.java | 97 ++- .../anomic/server/serverAbstractSwitch.java | 547 ---------------- .../de/anomic/server/serverPlainSwitch.java | 42 -- source/de/anomic/server/serverSwitch.java | 585 +++++++++++++++--- 6 files changed, 553 insertions(+), 724 deletions(-) delete mode 100644 source/de/anomic/server/serverAbstractSwitch.java delete mode 100644 source/de/anomic/server/serverPlainSwitch.java diff --git a/debian/changelog b/debian/changelog index 6adb92661..b4b4e4e57 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,4 @@ -yacy (0.91svn6326) unstable; urgency=low +yacy (0.91svn6388) unstable; urgency=low * SVN Update diff --git a/source/de/anomic/kelondro/text/Segments.java b/source/de/anomic/kelondro/text/Segments.java index fda6f4432..ec8932305 100644 --- a/source/de/anomic/kelondro/text/Segments.java +++ b/source/de/anomic/kelondro/text/Segments.java @@ -76,7 +76,7 @@ public final class Segments implements Iterable { final int entityCacheMaxSize, final long maxFileSize, final boolean useTailCache, - final boolean exceed134217727) throws IOException { + final boolean exceed134217727) { this.log = log; this.segmentsPath = segmentsPath; this.entityCacheMaxSize = entityCacheMaxSize; @@ -215,7 +215,7 @@ public final class Segments implements Iterable { ); } - public synchronized Segment.ReferenceCleaner getReferenceCleaner(final String segmentName, final byte[] startHash) throws IOException { + public synchronized Segment.ReferenceCleaner getReferenceCleaner(final String segmentName, final byte[] startHash) { return segment(segmentName).getReferenceCleaner(startHash); } diff --git a/source/de/anomic/search/Switchboard.java b/source/de/anomic/search/Switchboard.java index d46ab5f6a..44ee4584c 100644 --- a/source/de/anomic/search/Switchboard.java +++ b/source/de/anomic/search/Switchboard.java @@ -169,17 +169,15 @@ import de.anomic.kelondro.util.MemoryControl; import de.anomic.kelondro.util.SetTools; import de.anomic.net.UPnP; import de.anomic.search.blockrank.CRDistribution; -import de.anomic.server.serverAbstractSwitch; +import de.anomic.server.serverSwitch; import de.anomic.server.serverBusyThread; import de.anomic.server.serverCore; import de.anomic.server.serverDomains; import de.anomic.server.serverInstantBusyThread; -import de.anomic.server.serverObjects; import de.anomic.server.serverProcessor; import de.anomic.server.serverProcessorJob; import de.anomic.server.serverProfiling; import de.anomic.server.serverSemaphore; -import de.anomic.server.serverSwitch; import de.anomic.server.serverSystem; import de.anomic.server.serverThread; import de.anomic.tools.crypt; @@ -200,7 +198,7 @@ import de.anomic.yacy.dht.PeerSelection; import de.anomic.yacy.logging.Log; import de.anomic.ymage.WebStructureGraph; -public final class Switchboard extends serverAbstractSwitch implements serverSwitch { +public final class Switchboard extends serverSwitch { // load slots public static int xstackCrawlSlots = 2000; @@ -244,18 +242,18 @@ public final class Switchboard extends serverAbstractSwitch implements serverSwi public blogBoardComments blogCommentDB; public RobotsTxt robots; public boolean rankingOn; - public CRDistribution rankingOwnDistribution; - public CRDistribution rankingOtherDistribution; + public CRDistribution rankingOwnDistribution; + public CRDistribution rankingOtherDistribution; public HashMap outgoingCookies, incomingCookies; public volatile long proxyLastAccess, localSearchLastAccess, remoteSearchLastAccess; public yacyCore yc; public ResourceObserver observer; public userDB userDB; public bookmarksDB bookmarksDB; - public WebStructureGraph webStructure; + public WebStructureGraph webStructure; public ImporterManager dbImportManager; - public ArrayList localSearches; // array of search result properties as HashMaps - public ArrayList remoteSearches; // array of search result properties as HashMaps + public ArrayList localSearches; // array of search result properties as HashMaps + public ArrayList remoteSearches; // array of search result properties as HashMaps public ConcurrentHashMap> localSearchTracker, remoteSearchTracker; // mappings from requesting host to a TreeSet of Long(access time) public long indexedPages = 0; public double requestedQueries = 0d; @@ -352,37 +350,32 @@ public final class Switchboard extends serverAbstractSwitch implements serverSwi this.queuesRoot = new File(new File(indexPath, networkName), "QUEUES"); this.networkRoot.mkdirs(); this.queuesRoot.mkdirs(); - try { - final File mySeedFile = new File(networkRoot, yacySeedDB.DBFILE_OWN_SEED); - peers = new yacySeedDB( - networkRoot, - "seed.new.heap", - "seed.old.heap", - "seed.pot.heap", - mySeedFile, - redundancy, - partitionExponent, - this.useTailCache, - this.exceed134217727); - File oldSingleSegment = new File(new File(indexPath, networkName), "TEXT"); - File newSegmentsPath = new File(new File(indexPath, networkName), "SEGMENTS"); - Segments.migrateOld(oldSingleSegment, newSegmentsPath, getConfig(SwitchboardConstants.SEGMENT_PUBLIC, "default")); - indexSegments = new Segments( - log, - newSegmentsPath, - wordCacheMaxCount, - fileSizeMax, - this.useTailCache, - this.exceed134217727); - crawler = new CrawlSwitchboard( - peers, - networkName, - log, - this.queuesRoot); - } catch (IOException e1) { - e1.printStackTrace(); - indexSegments = null; - } + final File mySeedFile = new File(networkRoot, yacySeedDB.DBFILE_OWN_SEED); + peers = new yacySeedDB( + networkRoot, + "seed.new.heap", + "seed.old.heap", + "seed.pot.heap", + mySeedFile, + redundancy, + partitionExponent, + this.useTailCache, + this.exceed134217727); + File oldSingleSegment = new File(new File(indexPath, networkName), "TEXT"); + File newSegmentsPath = new File(new File(indexPath, networkName), "SEGMENTS"); + Segments.migrateOld(oldSingleSegment, newSegmentsPath, getConfig(SwitchboardConstants.SEGMENT_PUBLIC, "default")); + indexSegments = new Segments( + log, + newSegmentsPath, + wordCacheMaxCount, + fileSizeMax, + this.useTailCache, + this.exceed134217727); + crawler = new CrawlSwitchboard( + peers, + networkName, + log, + this.queuesRoot); // set the default segment names indexSegments.setSegment(Segments.Process.RECEIPTS, getConfig(SwitchboardConstants.SEGMENT_RECEIPTS, "default")); @@ -846,17 +839,13 @@ public final class Switchboard extends serverAbstractSwitch implements serverSwi partitionExponent, this.useTailCache, this.exceed134217727); - try { - indexSegments = new Segments( - log, - new File(new File(indexPrimaryPath, networkName), "SEGMENTS"), - wordCacheMaxCount, - fileSizeMax, - this.useTailCache, - this.exceed134217727); - } catch (IOException e) { - e.printStackTrace(); - } + indexSegments = new Segments( + log, + new File(new File(indexPrimaryPath, networkName), "SEGMENTS"), + wordCacheMaxCount, + fileSizeMax, + this.useTailCache, + this.exceed134217727); // startup crawler = new CrawlSwitchboard( @@ -1817,12 +1806,6 @@ public final class Switchboard extends serverAbstractSwitch implements serverSwi } } - - public serverObjects action(final String actionName, final serverObjects actionInput) { - // perform an action. (not used) - return null; - } - // method for index deletion public int removeAllUrlReferences(Segment indexSegment, final yacyURL url, final boolean fetchOnline) { return removeAllUrlReferences(indexSegment, url.hash(), fetchOnline); diff --git a/source/de/anomic/server/serverAbstractSwitch.java b/source/de/anomic/server/serverAbstractSwitch.java deleted file mode 100644 index 71170f41b..000000000 --- a/source/de/anomic/server/serverAbstractSwitch.java +++ /dev/null @@ -1,547 +0,0 @@ -// serverAbstractSwitch.java -// ------------------------------------- -// (C) by Michael Peter Christen; mc@yacy.net -// first published on http://www.anomic.de -// Frankfurt, Germany, 2004, 2005 -// last major change: 24.03.2005 -// -// This program is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - -package de.anomic.server; - -import java.io.File; -import java.io.IOException; -import java.net.InetAddress; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Iterator; -import java.util.Map; -import java.util.SortedMap; -import java.util.TreeMap; - -import de.anomic.kelondro.util.FileUtils; -import de.anomic.server.serverCore.Session; -import de.anomic.yacy.logging.Log; - -public abstract class serverAbstractSwitch implements serverSwitch { - - // configuration management - private final File configFile; - private final String configComment; - private final File rootPath; - protected boolean firstInit; - protected Log log; - protected int serverJobs; - private Map configProps; - private final Map configRemoved; - private final HashMap authorization; - private final TreeMap workerThreads; - private final TreeMap switchActions; - private final serverAccessTracker accessTracker; - - public serverAbstractSwitch(final File rootPath, final String initPath, final String configPath, final boolean applyPro) { - // we initialize the switchboard with a property file, - // but maintain these properties then later in a new 'config' file - // to reset all changed configs, the config file must - // be deleted, but not the init file - // the only attribute that will always be read from the init is the - // file name of the config file - this.rootPath = rootPath; - this.configComment = "This is an automatically generated file, updated by serverAbstractSwitch and initialized by " + initPath; - final File initFile = new File(rootPath, initPath); - this.configFile = new File(rootPath, configPath); // propertiesFile(config); - firstInit = !configFile.exists(); // this is true if the application was started for the first time - new File(configFile.getParent()).mkdir(); - - // predefine init's - Map initProps; - if (initFile.exists()) - initProps = FileUtils.loadMap(initFile); - else - initProps = new HashMap(); - - // if 'pro'-version is selected, overload standard settings with 'pro'-settings - Iterator i; - String prop; - if (applyPro) { - i = new HashMap(initProps).keySet().iterator(); // clone the map to avoid concurrent modification exceptions - while (i.hasNext()) { - prop = i.next(); - if (prop.endsWith("__pro")) { - initProps.put(prop.substring(0, prop.length() - 5), initProps.get(prop)); - } - } - } - // delete the 'pro' init settings - i = initProps.keySet().iterator(); - while (i.hasNext()) { - prop = i.next(); - if (prop.endsWith("__pro")) { - i.remove(); - } - } - - // load config's from last save - if (configFile.exists()) - configProps = FileUtils.loadMap(configFile); - else - configProps = new HashMap(); - - // remove all values from config that do not appear in init - configRemoved = new HashMap(); - synchronized (configProps) { - i = configProps.keySet().iterator(); - String key; - while (i.hasNext()) { - key = i.next(); - if (!(initProps.containsKey(key))) { - configRemoved.put(key, this.configProps.get(key)); - i.remove(); - } - } - - // doing a config settings migration - //HashMap migratedSettings = migrateSwitchConfigSettings((HashMap) removedProps); - //if (migratedSettings != null) configProps.putAll(migratedSettings); - - // merge new props from init to config - // this is necessary for migration, when new properties are attached - initProps.putAll(configProps); - configProps = initProps; - - // save result; this may initially create a config file after - // initialization - saveConfig(); - } - - // other settings - authorization = new HashMap(); - - // init thread control - workerThreads = new TreeMap(); - - // init switch actions - switchActions = new TreeMap(); - - // init busy state control - serverJobs = 0; - - // init server tracking - this.accessTracker = new serverAccessTracker( - getConfigLong("server.maxTrackingTime", 60 * 60 * 1000), - (int) getConfigLong("server.maxTrackingCount", 1000), - (int) getConfigLong("server.maxTrackingHostCount", 100) - ); - } - - // a logger for this switchboard - public void setLog(final Log log) { - this.log = log; - } - - public Log getLog() { - return log; - } - - public void setConfig(final Map otherConfigs) { - final Iterator> i = otherConfigs.entrySet().iterator(); - Map.Entry entry; - while (i.hasNext()) { - entry = i.next(); - setConfig(entry.getKey(), entry.getValue()); - } - } - - public void setConfig(final String key, final boolean value) { - setConfig(key, (value) ? "true" : "false"); - } - - public void setConfig(final String key, final long value) { - setConfig(key, Long.toString(value)); - } - - public void setConfig(final String key, final double value) { - setConfig(key, Double.toString(value)); - } - - public void setConfig(final String key, final String value) { - // perform action before setting new value - final Iterator bevore = switchActions.values().iterator(); - final Iterator after = switchActions.values().iterator(); - synchronized (configProps) { - serverSwitchAction action; - - while (bevore.hasNext()) { - action = bevore.next(); - try { - action.doBevoreSetConfig(key, value); - } catch (final Exception e) { - log.logSevere("serverAction bevoreSetConfig '" + action.getShortDescription() + "' failed with exception: " + e.getMessage()); - } - } - - // set the value - final Object oldValue = configProps.put(key, value); - saveConfig(); - - // perform actions afterwards - while (after.hasNext()) { - action = after.next(); - try { - action.doAfterSetConfig(key, value, (oldValue == null) ? null : (String) oldValue); - } catch (final Exception e) { - log.logSevere("serverAction afterSetConfig '" + action.getShortDescription() + "' failed with exception: " + e.getMessage()); - } - } - } - } - - public void removeConfig(final String key) { - configProps.remove(key); - } - - /* (non-Javadoc) - * @see de.anomic.server.serverSwitch#getConfig(java.lang.String, java.lang.String) - */ - public String getConfig(final String key, final String dflt) { - final Iterator i = switchActions.values().iterator(); - synchronized (configProps) { - // get the value - final Object s = configProps.get(key); - - // do action - serverSwitchAction action; - while (i.hasNext()) { - action = i.next(); - try { - action.doWhenGetConfig(key, (s == null) ? null : (String) s, dflt); - } catch (final Exception e) { - log.logSevere("serverAction whenGetConfig '" + action.getShortDescription() + "' failed with exception: " + e.getMessage()); - } - } - - // return value - if (s == null) return dflt; - return (String) s; - } - } - - public long getConfigLong(final String key, final long dflt) { - try { - return Long.parseLong(getConfig(key, Long.toString(dflt))); - } catch (final NumberFormatException e) { - return dflt; - } - } - - public double getConfigDouble(final String key, final double dflt) { - try { - return Double.parseDouble(getConfig(key, Double.toString(dflt))); - } catch (final NumberFormatException e) { - return dflt; - } - } - - public boolean getConfigBool(final String key, final boolean dflt) { - return Boolean.valueOf(getConfig(key, Boolean.toString(dflt))).booleanValue(); - } - - /** - * Create a File instance for a configuration setting specifying a path. - * @param key config key - * @param dflt default path value, that is used when there is no value - * key in the configuration. - * @return if the value of the setting is an absolute path String, then the - * returned File is derived from this setting only. Otherwise the path's file - * is constructed from the applications root path + the relative path setting. - */ - public File getConfigPath(final String key, final String dflt) { - File ret; - final String path = getConfig(key, dflt).replace('\\', '/'); - final File f = new File(path); - if (f == null) { - ret = null; - } else { - ret = (f.isAbsolute() ? f : new File(this.rootPath, path)); - } - - return ret; - } - - public Iterator configKeys() { - return configProps.keySet().iterator(); - } - - private void saveConfig() { - try { - synchronized (configProps) { - FileUtils.saveMap(configFile, configProps, configComment); - } - } catch (final IOException e) { - System.out.println("ERROR: cannot write config file " + configFile.toString() + ": " + e.getMessage()); - } - } - - public Map getRemoved() { - // returns configuration that had been removed during initialization - return configRemoved; - } - - // add/remove action listener - public void deployAction(final String actionName, final String actionShortDescription, final String actionLongDescription, - final serverSwitchAction newAction) { - newAction.setLog(log); - newAction.setDescription(actionShortDescription, actionLongDescription); - switchActions.put(actionName, newAction); - log.logInfo("Deployed Action '" + actionShortDescription + "', (" + switchActions.size() + " actions registered)"); - } - - public void undeployAction(final String actionName) { - final serverSwitchAction action = switchActions.get(actionName); - action.close(); - switchActions.remove(actionName); - log.logInfo("Undeployed Action '" + action.getShortDescription() + "', (" + switchActions.size() + " actions registered)"); - } - - public void deployThread( - final String threadName, - final String threadShortDescription, - final String threadLongDescription, - final String threadMonitorURL, - final serverBusyThread newThread, - final long startupDelay) { - deployThread(threadName, threadShortDescription, threadLongDescription, threadMonitorURL, - newThread, startupDelay, - Long.parseLong(getConfig(threadName + "_idlesleep" , "100")), - Long.parseLong(getConfig(threadName + "_busysleep" , "1000")), - Long.parseLong(getConfig(threadName + "_memprereq" , "1000000"))); - } - - public void deployThread( - final String threadName, - final String threadShortDescription, - final String threadLongDescription, - final String threadMonitorURL, - final serverBusyThread newThread, - final long startupDelay, - final long initialIdleSleep, - final long initialBusySleep, - final long initialMemoryPreRequisite) { - if (newThread.isAlive()) throw new RuntimeException("undeployed threads must not live; they are started as part of the deployment"); - newThread.setStartupSleep(startupDelay); - long x; - try { - x = Long.parseLong(getConfig(threadName + "_idlesleep" , "novalue")); - newThread.setIdleSleep(x); - } catch (final NumberFormatException e) { - newThread.setIdleSleep(initialIdleSleep); - setConfig(threadName + "_idlesleep", initialIdleSleep); - } - try { - x = Long.parseLong(getConfig(threadName + "_busysleep" , "novalue")); - newThread.setBusySleep(x); - } catch (final NumberFormatException e) { - newThread.setBusySleep(initialBusySleep); - setConfig(threadName + "_busysleep", initialBusySleep); - } - try { - x = Long.parseLong(getConfig(threadName + "_memprereq" , "novalue")); - newThread.setMemPreReqisite(x); - } catch (final NumberFormatException e) { - newThread.setMemPreReqisite(initialMemoryPreRequisite); - setConfig(threadName + "_memprereq", initialMemoryPreRequisite); - } - newThread.setLog(log); - newThread.setDescription(threadShortDescription, threadLongDescription, threadMonitorURL); - workerThreads.put(threadName, newThread); - // start the thread - if (workerThreads.containsKey(threadName)) newThread.start(); - } - - public serverBusyThread getThread(final String threadName) { - return workerThreads.get(threadName); - } - - public void setThreadPerformance(final String threadName, final long idleMillis, final long busyMillis, final long memprereqBytes) { - final serverBusyThread thread = workerThreads.get(threadName); - if (thread != null) { - thread.setIdleSleep(idleMillis); - thread.setBusySleep(busyMillis); - thread.setMemPreReqisite(memprereqBytes); - } - } - - public synchronized void terminateThread(final String threadName, final boolean waitFor) { - if (workerThreads.containsKey(threadName)) { - ((serverThread) workerThreads.get(threadName)).terminate(waitFor); - workerThreads.remove(threadName); - } - } - - public void intermissionAllThreads(final long pause) { - final Iterator e = workerThreads.keySet().iterator(); - while (e.hasNext()) { - workerThreads.get(e.next()).intermission(pause); - } - } - - public synchronized void terminateAllThreads(final boolean waitFor) { - Iterator e = workerThreads.keySet().iterator(); - while (e.hasNext()) { - ((serverThread) workerThreads.get(e.next())).terminate(false); - } - if (waitFor) { - e = workerThreads.keySet().iterator(); - while (e.hasNext()) { - ((serverThread) workerThreads.get(e.next())).terminate(true); - e.remove(); - } - } - } - - public String[] sessionsOlderThan(String threadName, long timeout) { - ArrayList list = new ArrayList(); - final serverThread st = getThread(threadName); - final Thread[] threadList = new Thread[((serverCore) st).getJobCount()]; - serverCore.sessionThreadGroup.enumerate(threadList); - - for (Thread t: threadList) { - if (t == null) continue; - if (!(t instanceof serverCore.Session)) continue; - if (!t.isAlive()) continue; - if (t == null) continue; - final Session s = (Session) t; - if (s.getTime() > timeout) { - list.add(s.getName()); - } - } - return (String[]) list.toArray(); - } - - public void closeSessions(String threadName, String sessionName) { - if (sessionName == null) return; - final serverThread st = getThread(threadName); - final Thread[] threadList = new Thread[((serverCore) st).getJobCount()]; - serverCore.sessionThreadGroup.enumerate(threadList); - - for (Thread t: threadList) { - if ( - (t != null) && - (t instanceof serverCore.Session) && - (t.isAlive()) && - (t.getName().equals(sessionName)) - ) { - // try to stop session - ((Session)t).setStopped(true); - try { Thread.sleep(100); } catch (final InterruptedException ex) {} - - // try to interrupt session - if (t.isAlive()) { - t.interrupt(); - try { Thread.sleep(100); } catch (final InterruptedException ex) {} - } - - // try to close socket - if (t.isAlive()) { - ((Session)t).close(); - } - - // wait for session to finish - if (t.isAlive()) { - try { t.join(500); } catch (final InterruptedException ex) {} - } - } - } - } - - public Iterator /*of serverThread-Names (String)*/ threadNames() { - return workerThreads.keySet().iterator(); - } - - // authentification routines: - - public void setAuthentify(final InetAddress host, final String user, final String rights) { - // sets access attributes according to host addresses - authorization.put(host, user + "@" + rights); - } - - public void removeAuthentify(final InetAddress host) { - // remove access attributes according to host addresses - authorization.remove(host); - } - - public String getAuthentifyUser(final InetAddress host) { - // read user name according to host addresses - final String a = authorization.get(host); - if (a == null) return null; - final int p = a.indexOf("@"); - if (p < 0) return null; - return a.substring(0, p); - } - - public String getAuthentifyRights(final InetAddress host) { - // read access rigths according to host addresses - final String a = authorization.get(host); - if (a == null) return null; - final int p = a.indexOf("@"); - if (p < 0) return null; - return a.substring(p + 1); - } - - public void addAuthentifyRight(final InetAddress host, final String right) { - final String rights = getAuthentifyRights(host); - if (rights == null) { - // create new authentification - setAuthentify(host, "unknown", right); - } else { - // add more authentification - final String user = getAuthentifyUser(host); - setAuthentify(host, user, rights + right); - } - } - - public boolean hasAuthentifyRight(final InetAddress host, final String right) { - final String rights = getAuthentifyRights(host); - if (rights == null) return false; - return rights.indexOf(right) >= 0; - } - - public abstract serverObjects action(String actionName, serverObjects actionInput); - - public File getRootPath() { - return rootPath; - } - - public String toString() { - return configProps.toString(); - } - - public void handleBusyState(final int jobs) { - serverJobs = jobs; - } - - public void track(String host, String accessPath) { - this.accessTracker.track(host, accessPath); - } - - public SortedMap accessTrack(String host) { - return this.accessTracker.accessTrack(host); - } - - public Iterator accessHosts() { - return this.accessTracker.accessHosts(); - } - -} diff --git a/source/de/anomic/server/serverPlainSwitch.java b/source/de/anomic/server/serverPlainSwitch.java deleted file mode 100644 index 0ad22ad22..000000000 --- a/source/de/anomic/server/serverPlainSwitch.java +++ /dev/null @@ -1,42 +0,0 @@ -// serverPlainSwitch.java -// (C) 2006 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany -// first published 10.08.2006 on http://www.anomic.de -// -// This is a part of YaCy, a peer-to-peer based web search engine -// -// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $ -// $LastChangedRevision: 1986 $ -// $LastChangedBy: orbiter $ -// -// LICENSE -// -// This program is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - -package de.anomic.server; - -import java.io.File; - -public class serverPlainSwitch extends serverAbstractSwitch implements serverSwitch { - - public serverPlainSwitch(final File rootPath, final String initPath, final String configPath, final boolean applyPro) { - super(rootPath, initPath, configPath, applyPro); - } - - public serverObjects action(final String actionName, final serverObjects actionInput) { - // no actions - return null; - } - -} diff --git a/source/de/anomic/server/serverSwitch.java b/source/de/anomic/server/serverSwitch.java index 288cef3bc..2036e6bff 100644 --- a/source/de/anomic/server/serverSwitch.java +++ b/source/de/anomic/server/serverSwitch.java @@ -1,12 +1,9 @@ // serverSwitch.java -// ------------------------------------------- +// ------------------------------------- // (C) by Michael Peter Christen; mc@yacy.net // first published on http://www.anomic.de -// Frankfurt, Germany, 2004 -// -// $LastChangedDate$ -// $LastChangedRevision$ -// $LastChangedBy$ +// Frankfurt, Germany, 2004, 2005 +// last major change: 24.03.2005 // // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by @@ -22,89 +19,527 @@ // along with this program; if not, write to the Free Software // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -/* - this is an interface for possible switchboard implementations - Its purpose is to provide a mechanism which cgi pages can use - to influence the behavior of a concurrntly running application -*/ - package de.anomic.server; import java.io.File; +import java.io.IOException; import java.net.InetAddress; +import java.util.ArrayList; +import java.util.HashMap; import java.util.Iterator; import java.util.Map; import java.util.SortedMap; +import java.util.TreeMap; +import de.anomic.kelondro.util.FileUtils; +import de.anomic.server.serverCore.Session; import de.anomic.yacy.logging.Log; -public interface serverSwitch { +public class serverSwitch { + + // configuration management + private final File configFile; + private final String configComment; + private final File rootPath; + protected boolean firstInit; + protected Log log; + protected int serverJobs; + private Map configProps; + private final Map configRemoved; + private final HashMap authorization; + private final TreeMap workerThreads; + private final TreeMap switchActions; + private final serverAccessTracker accessTracker; + + public serverSwitch(final File rootPath, final String initPath, final String configPath, final boolean applyPro) { + // we initialize the switchboard with a property file, + // but maintain these properties then later in a new 'config' file + // to reset all changed configs, the config file must + // be deleted, but not the init file + // the only attribute that will always be read from the init is the + // file name of the config file + this.rootPath = rootPath; + this.configComment = "This is an automatically generated file, updated by serverAbstractSwitch and initialized by " + initPath; + final File initFile = new File(rootPath, initPath); + this.configFile = new File(rootPath, configPath); // propertiesFile(config); + firstInit = !configFile.exists(); // this is true if the application was started for the first time + new File(configFile.getParent()).mkdir(); + + // predefine init's + Map initProps; + if (initFile.exists()) + initProps = FileUtils.loadMap(initFile); + else + initProps = new HashMap(); + + // if 'pro'-version is selected, overload standard settings with 'pro'-settings + Iterator i; + String prop; + if (applyPro) { + i = new HashMap(initProps).keySet().iterator(); // clone the map to avoid concurrent modification exceptions + while (i.hasNext()) { + prop = i.next(); + if (prop.endsWith("__pro")) { + initProps.put(prop.substring(0, prop.length() - 5), initProps.get(prop)); + } + } + } + // delete the 'pro' init settings + i = initProps.keySet().iterator(); + while (i.hasNext()) { + prop = i.next(); + if (prop.endsWith("__pro")) { + i.remove(); + } + } + + // load config's from last save + if (configFile.exists()) + configProps = FileUtils.loadMap(configFile); + else + configProps = new HashMap(); + + // remove all values from config that do not appear in init + configRemoved = new HashMap(); + synchronized (configProps) { + i = configProps.keySet().iterator(); + String key; + while (i.hasNext()) { + key = i.next(); + if (!(initProps.containsKey(key))) { + configRemoved.put(key, this.configProps.get(key)); + i.remove(); + } + } - // the root path for the application - public File getRootPath(); + // doing a config settings migration + //HashMap migratedSettings = migrateSwitchConfigSettings((HashMap) removedProps); + //if (migratedSettings != null) configProps.putAll(migratedSettings); + // merge new props from init to config + // this is necessary for migration, when new properties are attached + initProps.putAll(configProps); + configProps = initProps; + + // save result; this may initially create a config file after + // initialization + saveConfig(); + } + + // other settings + authorization = new HashMap(); + + // init thread control + workerThreads = new TreeMap(); + + // init switch actions + switchActions = new TreeMap(); + + // init busy state control + serverJobs = 0; + + // init server tracking + this.accessTracker = new serverAccessTracker( + getConfigLong("server.maxTrackingTime", 60 * 60 * 1000), + (int) getConfigLong("server.maxTrackingCount", 1000), + (int) getConfigLong("server.maxTrackingHostCount", 100) + ); + } + // a logger for this switchboard - public void setLog(Log log); - public Log getLog(); + public void setLog(final Log log) { + this.log = log; + } + + public Log getLog() { + return log; + } + + public void setConfig(final Map otherConfigs) { + final Iterator> i = otherConfigs.entrySet().iterator(); + Map.Entry entry; + while (i.hasNext()) { + entry = i.next(); + setConfig(entry.getKey(), entry.getValue()); + } + } + + public void setConfig(final String key, final boolean value) { + setConfig(key, (value) ? "true" : "false"); + } + + public void setConfig(final String key, final long value) { + setConfig(key, Long.toString(value)); + } + + public void setConfig(final String key, final double value) { + setConfig(key, Double.toString(value)); + } + + public void setConfig(final String key, final String value) { + // perform action before setting new value + final Iterator bevore = switchActions.values().iterator(); + final Iterator after = switchActions.values().iterator(); + synchronized (configProps) { + serverSwitchAction action; + + while (bevore.hasNext()) { + action = bevore.next(); + try { + action.doBevoreSetConfig(key, value); + } catch (final Exception e) { + log.logSevere("serverAction bevoreSetConfig '" + action.getShortDescription() + "' failed with exception: " + e.getMessage()); + } + } + + // set the value + final Object oldValue = configProps.put(key, value); + saveConfig(); + + // perform actions afterwards + while (after.hasNext()) { + action = after.next(); + try { + action.doAfterSetConfig(key, value, (oldValue == null) ? null : (String) oldValue); + } catch (final Exception e) { + log.logSevere("serverAction afterSetConfig '" + action.getShortDescription() + "' failed with exception: " + e.getMessage()); + } + } + } + } + + public void removeConfig(final String key) { + configProps.remove(key); + } + + /* (non-Javadoc) + * @see de.anomic.server.serverSwitch#getConfig(java.lang.String, java.lang.String) + */ + public String getConfig(final String key, final String dflt) { + final Iterator i = switchActions.values().iterator(); + synchronized (configProps) { + // get the value + final Object s = configProps.get(key); + + // do action + serverSwitchAction action; + while (i.hasNext()) { + action = i.next(); + try { + action.doWhenGetConfig(key, (s == null) ? null : (String) s, dflt); + } catch (final Exception e) { + log.logSevere("serverAction whenGetConfig '" + action.getShortDescription() + "' failed with exception: " + e.getMessage()); + } + } + + // return value + if (s == null) return dflt; + return (String) s; + } + } + + public long getConfigLong(final String key, final long dflt) { + try { + return Long.parseLong(getConfig(key, Long.toString(dflt))); + } catch (final NumberFormatException e) { + return dflt; + } + } + + public double getConfigDouble(final String key, final double dflt) { + try { + return Double.parseDouble(getConfig(key, Double.toString(dflt))); + } catch (final NumberFormatException e) { + return dflt; + } + } + + public boolean getConfigBool(final String key, final boolean dflt) { + return Boolean.valueOf(getConfig(key, Boolean.toString(dflt))).booleanValue(); + } + + /** + * Create a File instance for a configuration setting specifying a path. + * @param key config key + * @param dflt default path value, that is used when there is no value + * key in the configuration. + * @return if the value of the setting is an absolute path String, then the + * returned File is derived from this setting only. Otherwise the path's file + * is constructed from the applications root path + the relative path setting. + */ + public File getConfigPath(final String key, final String dflt) { + File ret; + final String path = getConfig(key, dflt).replace('\\', '/'); + final File f = new File(path); + if (f == null) { + ret = null; + } else { + ret = (f.isAbsolute() ? f : new File(this.rootPath, path)); + } + + return ret; + } + + public Iterator configKeys() { + return configProps.keySet().iterator(); + } + + private void saveConfig() { + try { + synchronized (configProps) { + FileUtils.saveMap(configFile, configProps, configComment); + } + } catch (final IOException e) { + System.out.println("ERROR: cannot write config file " + configFile.toString() + ": " + e.getMessage()); + } + } + + public Map getRemoved() { + // returns configuration that had been removed during initialization + return configRemoved; + } + + // add/remove action listener + public void deployAction(final String actionName, final String actionShortDescription, final String actionLongDescription, + final serverSwitchAction newAction) { + newAction.setLog(log); + newAction.setDescription(actionShortDescription, actionLongDescription); + switchActions.put(actionName, newAction); + log.logInfo("Deployed Action '" + actionShortDescription + "', (" + switchActions.size() + " actions registered)"); + } + + public void undeployAction(final String actionName) { + final serverSwitchAction action = switchActions.get(actionName); + action.close(); + switchActions.remove(actionName); + log.logInfo("Undeployed Action '" + action.getShortDescription() + "', (" + switchActions.size() + " actions registered)"); + } + + public void deployThread( + final String threadName, + final String threadShortDescription, + final String threadLongDescription, + final String threadMonitorURL, + final serverBusyThread newThread, + final long startupDelay) { + deployThread(threadName, threadShortDescription, threadLongDescription, threadMonitorURL, + newThread, startupDelay, + Long.parseLong(getConfig(threadName + "_idlesleep" , "100")), + Long.parseLong(getConfig(threadName + "_busysleep" , "1000")), + Long.parseLong(getConfig(threadName + "_memprereq" , "1000000"))); + } - // access tracker - public void track(String host, String accessPath); // learn that a specific host has accessed a specific path - public SortedMap accessTrack(String host); // returns mapping from Long(accesstime) to path - public Iterator accessHosts(); // returns an iterator of hosts in tracker (String) + public void deployThread( + final String threadName, + final String threadShortDescription, + final String threadLongDescription, + final String threadMonitorURL, + final serverBusyThread newThread, + final long startupDelay, + final long initialIdleSleep, + final long initialBusySleep, + final long initialMemoryPreRequisite) { + if (newThread.isAlive()) throw new RuntimeException("undeployed threads must not live; they are started as part of the deployment"); + newThread.setStartupSleep(startupDelay); + long x; + try { + x = Long.parseLong(getConfig(threadName + "_idlesleep" , "novalue")); + newThread.setIdleSleep(x); + } catch (final NumberFormatException e) { + newThread.setIdleSleep(initialIdleSleep); + setConfig(threadName + "_idlesleep", initialIdleSleep); + } + try { + x = Long.parseLong(getConfig(threadName + "_busysleep" , "novalue")); + newThread.setBusySleep(x); + } catch (final NumberFormatException e) { + newThread.setBusySleep(initialBusySleep); + setConfig(threadName + "_busysleep", initialBusySleep); + } + try { + x = Long.parseLong(getConfig(threadName + "_memprereq" , "novalue")); + newThread.setMemPreReqisite(x); + } catch (final NumberFormatException e) { + newThread.setMemPreReqisite(initialMemoryPreRequisite); + setConfig(threadName + "_memprereq", initialMemoryPreRequisite); + } + newThread.setLog(log); + newThread.setDescription(threadShortDescription, threadLongDescription, threadMonitorURL); + workerThreads.put(threadName, newThread); + // start the thread + if (workerThreads.containsKey(threadName)) newThread.start(); + } + + public serverBusyThread getThread(final String threadName) { + return workerThreads.get(threadName); + } + + public void setThreadPerformance(final String threadName, final long idleMillis, final long busyMillis, final long memprereqBytes) { + final serverBusyThread thread = workerThreads.get(threadName); + if (thread != null) { + thread.setIdleSleep(idleMillis); + thread.setBusySleep(busyMillis); + thread.setMemPreReqisite(memprereqBytes); + } + } + + public synchronized void terminateThread(final String threadName, final boolean waitFor) { + if (workerThreads.containsKey(threadName)) { + ((serverThread) workerThreads.get(threadName)).terminate(waitFor); + workerThreads.remove(threadName); + } + } + + public void intermissionAllThreads(final long pause) { + final Iterator e = workerThreads.keySet().iterator(); + while (e.hasNext()) { + workerThreads.get(e.next()).intermission(pause); + } + } + + public synchronized void terminateAllThreads(final boolean waitFor) { + Iterator e = workerThreads.keySet().iterator(); + while (e.hasNext()) { + ((serverThread) workerThreads.get(e.next())).terminate(false); + } + if (waitFor) { + e = workerThreads.keySet().iterator(); + while (e.hasNext()) { + ((serverThread) workerThreads.get(e.next())).terminate(true); + e.remove(); + } + } + } + + public String[] sessionsOlderThan(String threadName, long timeout) { + ArrayList list = new ArrayList(); + final serverThread st = getThread(threadName); + final Thread[] threadList = new Thread[((serverCore) st).getJobCount()]; + serverCore.sessionThreadGroup.enumerate(threadList); + + for (Thread t: threadList) { + if (t == null) continue; + if (!(t instanceof serverCore.Session)) continue; + if (!t.isAlive()) continue; + if (t == null) continue; + final Session s = (Session) t; + if (s.getTime() > timeout) { + list.add(s.getName()); + } + } + return (String[]) list.toArray(); + } + + public void closeSessions(String threadName, String sessionName) { + if (sessionName == null) return; + final serverThread st = getThread(threadName); + final Thread[] threadList = new Thread[((serverCore) st).getJobCount()]; + serverCore.sessionThreadGroup.enumerate(threadList); - // a switchboard can have action listener - // these listeners are hooks for numerous methods below - public void deployAction(String actionName, - String actionShortDescription, - String actionLongDescription, - serverSwitchAction action); - public void undeployAction(String actionName); - - // the switchboard can manage worker threads - public void deployThread(String threadName, - String threadShortDescription, - String threadLongDescription, - String threadMonitorURL, - serverBusyThread newThread, - long startupDelay, - long initialIdleSleep, long initialBusySleep, - long initialMemoryPreRequisite); - public serverBusyThread getThread(String threadName); - public void setThreadPerformance(String threadName, long idleMillis, long busyMillis, long memprereq); - public void terminateThread(String threadName, boolean waitFor); - public void intermissionAllThreads(long pause); - public void terminateAllThreads(boolean waitFor); - - public Iterator /*of serverThread-Names (String)*/ threadNames(); - - // the switchboard can be used to set and read properties - public void setConfig(Map otherConfigs); - public void setConfig(String key, long value); - public void setConfig(String key, String value); - public String getConfig(String key, String dflt); - public long getConfigLong(String key, long dflt); - public boolean getConfigBool(String key, boolean dflt); - public File getConfigPath(String key, String dflt); - public void removeConfig(String key); - public Iterator configKeys(); - public Map getRemoved(); - - // authentification routines: sets and reads access attributes according to host addresses - public void setAuthentify(InetAddress host, String user, String rigth); - public void removeAuthentify(InetAddress host); - public String getAuthentifyUser(InetAddress host); - public String getAuthentifyRights(InetAddress host); - public void addAuthentifyRight(InetAddress host, String right); - public boolean hasAuthentifyRight(InetAddress host, String right); - - // ask the switchboard to perform an action - // the result is a properties structure with the result of the action - // The actionName selects an action - // the actionInput is an input for the selected action - public serverObjects action(String actionName, serverObjects actionInput); - - // performance control: the server can announce busy and idle status to the switchboard - // these announcements can be used to trigger events or interrupts - public void handleBusyState(int jobs); + for (Thread t: threadList) { + if ( + (t != null) && + (t instanceof serverCore.Session) && + (t.isAlive()) && + (t.getName().equals(sessionName)) + ) { + // try to stop session + ((Session)t).setStopped(true); + try { Thread.sleep(100); } catch (final InterruptedException ex) {} + + // try to interrupt session + if (t.isAlive()) { + t.interrupt(); + try { Thread.sleep(100); } catch (final InterruptedException ex) {} + } + + // try to close socket + if (t.isAlive()) { + ((Session)t).close(); + } + + // wait for session to finish + if (t.isAlive()) { + try { t.join(500); } catch (final InterruptedException ex) {} + } + } + } + } + + public Iterator /*of serverThread-Names (String)*/ threadNames() { + return workerThreads.keySet().iterator(); + } + + // authentification routines: + + public void setAuthentify(final InetAddress host, final String user, final String rights) { + // sets access attributes according to host addresses + authorization.put(host, user + "@" + rights); + } + + public void removeAuthentify(final InetAddress host) { + // remove access attributes according to host addresses + authorization.remove(host); + } + + public String getAuthentifyUser(final InetAddress host) { + // read user name according to host addresses + final String a = authorization.get(host); + if (a == null) return null; + final int p = a.indexOf("@"); + if (p < 0) return null; + return a.substring(0, p); + } + + public String getAuthentifyRights(final InetAddress host) { + // read access rigths according to host addresses + final String a = authorization.get(host); + if (a == null) return null; + final int p = a.indexOf("@"); + if (p < 0) return null; + return a.substring(p + 1); + } + + public void addAuthentifyRight(final InetAddress host, final String right) { + final String rights = getAuthentifyRights(host); + if (rights == null) { + // create new authentification + setAuthentify(host, "unknown", right); + } else { + // add more authentification + final String user = getAuthentifyUser(host); + setAuthentify(host, user, rights + right); + } + } + public boolean hasAuthentifyRight(final InetAddress host, final String right) { + final String rights = getAuthentifyRights(host); + if (rights == null) return false; + return rights.indexOf(right) >= 0; + } + + public File getRootPath() { + return rootPath; + } + + public String toString() { + return configProps.toString(); + } + + public void handleBusyState(final int jobs) { + serverJobs = jobs; + } + + public void track(String host, String accessPath) { + this.accessTracker.track(host, accessPath); + } + + public SortedMap accessTrack(String host) { + return this.accessTracker.accessTrack(host); + } + + public Iterator accessHosts() { + return this.accessTracker.accessHosts(); + } + }