implemented serverSwitchActions - action-hooks

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@105 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 20 years ago
parent 5a651284a2
commit b4030e5023

@ -46,7 +46,7 @@
// javac -classpath .:../Classes Config_p.java // javac -classpath .:../Classes Config_p.java
// if the shell's current path is HTROOT // if the shell's current path is HTROOT
import java.util.Enumeration; import java.util.Iterator;
import de.anomic.http.httpHeader; import de.anomic.http.httpHeader;
import de.anomic.server.serverObjects; import de.anomic.server.serverObjects;
@ -58,7 +58,7 @@ public class Config_p {
// return variable that accumulates replacements // return variable that accumulates replacements
serverObjects prop = new serverObjects(); serverObjects prop = new serverObjects();
int count=0; int count=0;
Enumeration keys = env.configKeys(); Iterator keys = env.configKeys();
String key=""; String key="";
//change a Key //change a Key
@ -70,8 +70,8 @@ public class Config_p {
} }
} }
while(keys.hasMoreElements()){ while(keys.hasNext()){
key=(String)keys.nextElement(); key=(String)keys.next();
prop.put("options_"+count+"_key", key); prop.put("options_"+count+"_key", key);
prop.put("options_"+count+"_value", env.getConfig(key, "ERROR")); prop.put("options_"+count+"_value", env.getConfig(key, "ERROR"));
count++; count++;

@ -123,9 +123,9 @@ public class transferRWI {
while (it.hasNext()) unknownURLs += "," + (String) it.next(); while (it.hasNext()) unknownURLs += "," + (String) it.next();
if (unknownURLs.length() > 0) unknownURLs = unknownURLs.substring(1); if (unknownURLs.length() > 0) unknownURLs = unknownURLs.substring(1);
if (wordhashes.length == 0) if (wordhashes.length == 0)
switchboard.log.logInfo("Received 0 Words from peer " + iam + ", requested " + unknownURL.size() + " URL's"); switchboard.getLog().logInfo("Received 0 Words from peer " + iam + ", requested " + unknownURL.size() + " URL's");
else else
switchboard.log.logInfo("Received " + received + " Words [" + wordhashes[0] + " .. " + wordhashes[wordhashes.length - 1] + "] from peer " + iam + ", requested " + unknownURL.size() + " URL's"); switchboard.getLog().logInfo("Received " + received + " Words [" + wordhashes[0] + " .. " + wordhashes[wordhashes.length - 1] + "] from peer " + iam + ", requested " + unknownURL.size() + " URL's");
result = "ok"; result = "ok";
} else { } else {
result = "error_not_granted"; result = "error_not_granted";

@ -87,8 +87,8 @@ public class transferURL {
// return rewrite properties // return rewrite properties
int more = switchboard.loadedURL.size() - sizeBefore; int more = switchboard.loadedURL.size() - sizeBefore;
doublevalues = "" + (received - more); doublevalues = "" + (received - more);
switchboard.log.logInfo("Received " + received + " URL's from peer " + iam); switchboard.getLog().logInfo("Received " + received + " URL's from peer " + iam);
if ((received - more) > 0) switchboard.log.logError("Received " + doublevalues + " double URL's from peer " + iam); if ((received - more) > 0) switchboard.getLog().logError("Received " + doublevalues + " double URL's from peer " + iam);
result = "ok"; result = "ok";
} else { } else {
result = "error_not_granted"; result = "error_not_granted";

@ -45,7 +45,7 @@
# Contributions and changes to the program code must be marked as such. # Contributions and changes to the program code must be marked as such.
# define variables # define variables
version='0.372' version='0.373'
datestr=`date +%Y%m%d` datestr=`date +%Y%m%d`
#release='yacy_v'$version'_'$datestr #release='yacy_v'$version'_'$datestr
release='yacy_dev_v'$version'_'$datestr release='yacy_dev_v'$version'_'$datestr

@ -360,7 +360,8 @@ public final class plasmaHTCache {
public static boolean isText(httpHeader response) { public static boolean isText(httpHeader response) {
Object ct = response.get("Content-Type"); Object ct = response.get("Content-Type");
if (ct == null) return false; if (ct == null) return false;
return ((String)ct).toUpperCase().startsWith("TEXT"); String t = ((String)ct).toLowerCase();
return ((t.startsWith("text")) || (t.equals("application/xhtml+xml")));
} }
public static boolean noIndexingURL(String urlString) { public static boolean noIndexingURL(String urlString) {

@ -170,7 +170,6 @@ public class plasmaSwitchboard extends serverAbstractSwitch implements serverSwi
public plasmaHTCache cacheManager; public plasmaHTCache cacheManager;
public plasmaCrawlLoader cacheLoader; public plasmaCrawlLoader cacheLoader;
public LinkedList processStack = new LinkedList(); public LinkedList processStack = new LinkedList();
public serverLog log;
public messageBoard messageDB; public messageBoard messageDB;
public wikiBoard wikiDB; public wikiBoard wikiDB;
public String remoteProxyHost; public String remoteProxyHost;
@ -191,9 +190,9 @@ public class plasmaSwitchboard extends serverAbstractSwitch implements serverSwi
public plasmaSwitchboard(String rootPath, String initPath, String configPath) throws IOException { public plasmaSwitchboard(String rootPath, String initPath, String configPath) throws IOException {
super(rootPath, initPath, configPath); super(rootPath, initPath, configPath);
// set loglevel // set loglevel and log
int loglevel = Integer.parseInt(getConfig("plasmaLoglevel", "2")); int loglevel = Integer.parseInt(getConfig("plasmaLoglevel", "2"));
log = new serverLog("PLASMA", loglevel); setLog(new serverLog("PLASMA", loglevel));
// load values from configs // load values from configs
plasmaPath = new File(rootPath, getConfig("dbPath", "DATABASE")); plasmaPath = new File(rootPath, getConfig("dbPath", "DATABASE"));
@ -308,22 +307,22 @@ public class plasmaSwitchboard extends serverAbstractSwitch implements serverSwi
// deploy threads // deploy threads
deployThread("90_cleanup", "Cleanup", "simple cleaning process for monitoring information" , deployThread("90_cleanup", "Cleanup", "simple cleaning process for monitoring information" ,
new serverInstantThread(this, "cleanupJob", "cleanupJobSize"), log, 10000); // all 5 Minutes new serverInstantThread(this, "cleanupJob", "cleanupJobSize"), 10000); // all 5 Minutes
deployThread("80_dequeue", "Indexing Dequeue", "thread that creates database entries from scraped web content and performes indexing" , deployThread("80_dequeue", "Indexing Dequeue", "thread that creates database entries from scraped web content and performes indexing" ,
new serverInstantThread(this, "deQueue", "queueSize"), log, 10000); new serverInstantThread(this, "deQueue", "queueSize"), 10000);
deployThread("70_cachemanager", "Proxy Cache Enqueue", "job takes new proxy files from RAM stack, stores them, and hands over to the Indexing Stack", deployThread("70_cachemanager", "Proxy Cache Enqueue", "job takes new proxy files from RAM stack, stores them, and hands over to the Indexing Stack",
new serverInstantThread(cacheManager, "job", "size"), log, 10000); new serverInstantThread(cacheManager, "job", "size"), 10000);
deployThread("60_globalcrawl", "Global Crawl", "thread that performes a single crawl/indexing step of a web page for global crawling", deployThread("60_globalcrawl", "Global Crawl", "thread that performes a single crawl/indexing step of a web page for global crawling",
new serverInstantThread(this, "globalCrawlJob", "globalCrawlJobSize"), log, 30000); new serverInstantThread(this, "globalCrawlJob", "globalCrawlJobSize"), 30000);
deployThread("50_localcrawl", "Local Crawl", "thread that performes a single crawl step from the local crawl queue", deployThread("50_localcrawl", "Local Crawl", "thread that performes a single crawl step from the local crawl queue",
new serverInstantThread(this, "localCrawlJob", "localCrawlJobSize"), log, 10000); new serverInstantThread(this, "localCrawlJob", "localCrawlJobSize"), 10000);
deployThread("40_peerseedcycle", "Seed-List Upload", "task that a principal peer performes to generate and upload a seed-list to a ftp account", deployThread("40_peerseedcycle", "Seed-List Upload", "task that a principal peer performes to generate and upload a seed-list to a ftp account",
new serverInstantThread(yc, "publishSeedList", null), yc.log, 180000); new serverInstantThread(yc, "publishSeedList", null), 180000);
deployThread("30_peerping", "YaCy Core", "this is the p2p-control and peer-ping task", deployThread("30_peerping", "YaCy Core", "this is the p2p-control and peer-ping task",
new serverInstantThread(yc, "peerPing", null), yc.log, 2000); new serverInstantThread(yc, "peerPing", null), 2000);
indexDistribution = new distributeIndex(100 /*indexCount*/, 8000, 1 /*peerCount*/); indexDistribution = new distributeIndex(100 /*indexCount*/, 8000, 1 /*peerCount*/);
deployThread("20_dhtdistribution", "DHT Distribution (currently by juniors only)", "selection, transfer and deletion of index entries that are not searched on your peer, but on others", deployThread("20_dhtdistribution", "DHT Distribution (currently by juniors only)", "selection, transfer and deletion of index entries that are not searched on your peer, but on others",
new serverInstantThread(indexDistribution, "job", null), log, 120000); new serverInstantThread(indexDistribution, "job", null), 120000);
} }
private static String ppRamString(int bytes) { private static String ppRamString(int bytes) {

@ -48,6 +48,8 @@ import java.io.PrintWriter;
import java.net.InetAddress; import java.net.InetAddress;
import java.util.Enumeration; import java.util.Enumeration;
import java.util.Hashtable; import java.util.Hashtable;
import java.util.Map;
import java.util.HashMap;
import java.util.Iterator; import java.util.Iterator;
import java.util.Properties; import java.util.Properties;
import java.util.TreeMap; import java.util.TreeMap;
@ -56,12 +58,14 @@ public abstract class serverAbstractSwitch implements serverSwitch {
// configuration management // configuration management
private final File configFile; private final File configFile;
private Hashtable configProps; private Map configProps;
private final String configComment; private final String configComment;
private final Hashtable authorization; private final HashMap authorization;
private String rootPath; private String rootPath;
private final TreeMap workerThreads; private final TreeMap workerThreads;
protected int serverJobs; private final TreeMap switchActions;
protected serverLog log;
protected int serverJobs;
public serverAbstractSwitch(String rootPath, String initPath, String configPath) throws IOException { public serverAbstractSwitch(String rootPath, String initPath, String configPath) throws IOException {
// we initialize the switchboard with a property file, // we initialize the switchboard with a property file,
@ -77,20 +81,18 @@ public abstract class serverAbstractSwitch implements serverSwitch {
new File(configFile.getParent()).mkdir(); new File(configFile.getParent()).mkdir();
// predefine init's // predefine init's
Hashtable initProps; Map initProps;
if (initFile.exists()) initProps = loadHashtable(initFile); else initProps = new Hashtable(); if (initFile.exists()) initProps = loadHashMap(initFile); else initProps = new HashMap();
// load config's from last save // load config's from last save
if (configFile.exists()) configProps = loadHashtable(configFile); else configProps = new Hashtable(); if (configFile.exists()) configProps = loadHashMap(configFile); else configProps = new HashMap();
// remove all values from config that do not appear in init (out-dated settings) // remove all values from config that do not appear in init (out-dated settings)
Enumeration e = configProps.keys(); Iterator i = configProps.keySet().iterator();
String key; String key;
while (e.hasMoreElements()) { while (i.hasNext()) {
key = (String) e.nextElement(); key = (String) i.next();
//System.out.println("TESTING " + key);
if (!(initProps.containsKey(key))) { if (!(initProps.containsKey(key))) {
//System.out.println("MIGRATE: removing out-dated property '" + key + "'");
configProps.remove(key); configProps.remove(key);
} }
} }
@ -104,16 +106,28 @@ public abstract class serverAbstractSwitch implements serverSwitch {
saveConfig(); saveConfig();
// other settings // other settings
authorization = new Hashtable(); authorization = new HashMap();
// init thread control // init thread control
workerThreads = new TreeMap(); workerThreads = new TreeMap();
// init switch actions
switchActions = new TreeMap();
// init busy state control // init busy state control
serverJobs = 0; serverJobs = 0;
} }
public static Hashtable loadHashtable(File f) { // a logger for this switchboard
public void setLog(serverLog log) {
this.log = log;
}
public serverLog getLog() {
return log;
}
public static Map loadHashMap(File f) {
// load props // load props
Properties prop = new Properties(); Properties prop = new Properties();
try { try {
@ -125,15 +139,16 @@ public abstract class serverAbstractSwitch implements serverSwitch {
return (Hashtable) prop; return (Hashtable) prop;
} }
public static void saveHashtable(File f, Hashtable props, String comment) throws IOException { public static void saveMap(File f, Map props, String comment) throws IOException {
PrintWriter pw = new PrintWriter(new FileOutputStream(f)); PrintWriter pw = new PrintWriter(new FileOutputStream(f));
pw.println("# " + comment); pw.println("# " + comment);
Enumeration e = props.keys(); Iterator i = props.entrySet().iterator();
String key, value; String key, value;
while (e.hasMoreElements()) { Map.Entry entry;
key = (String) e.nextElement(); while (i.hasNext()) {
//value = (String) props.get(key); entry = (Map.Entry) i.next();
value = ((String) props.get(key)).replaceAll("\n", "\\\\n"); key = (String) entry.getKey();
value = ((String) entry.getValue()).replaceAll("\n", "\\\\n");
pw.println(key + "=" + value); pw.println(key + "=" + value);
} }
pw.println("# EOF"); pw.println("# EOF");
@ -145,35 +160,93 @@ public abstract class serverAbstractSwitch implements serverSwitch {
} }
public void setConfig(String key, String value) { public void setConfig(String key, String value) {
configProps.put(key, value); // perform action before setting new value
Map.Entry entry;
serverSwitchAction action;
Iterator i = switchActions.entrySet().iterator();
while (i.hasNext()) {
entry = (Map.Entry) i.next();
action = (serverSwitchAction) entry.getValue();
try {
action.doBevoreSetConfig(key, value);
} catch (Exception e) {
log.logError("serverAction bevoreSetConfig '" + action.getShortDescription() + "' failed with exception: " + e.getMessage());
}
}
// set the value
String oldValue = (String) configProps.put(key, value);
saveConfig(); saveConfig();
// perform actions afterwards
i = switchActions.entrySet().iterator();
while (i.hasNext()) {
entry = (Map.Entry) i.next();
action = (serverSwitchAction) entry.getValue();
try {
action.doAfterSetConfig(key, value, oldValue);
} catch (Exception e) {
log.logError("serverAction afterSetConfig '" + action.getShortDescription() + "' failed with exception: " + e.getMessage());
}
}
} }
public String getConfig(String key, String dflt) { public String getConfig(String key, String dflt) {
// get the value
String s = (String) configProps.get(key); String s = (String) configProps.get(key);
// do action
Map.Entry entry;
serverSwitchAction action;
Iterator i = switchActions.entrySet().iterator();
while (i.hasNext()) {
entry = (Map.Entry) i.next();
action = (serverSwitchAction) entry.getValue();
try {
action.doWhenGetConfig(key, s, dflt);
} catch (Exception e) {
log.logError("serverAction whenGetConfig '" + action.getShortDescription() + "' failed with exception: " + e.getMessage());
}
}
// return value
if (s == null) return dflt; else return s; if (s == null) return dflt; else return s;
} }
public Enumeration configKeys() { public Iterator configKeys() {
return configProps.keys(); return configProps.keySet().iterator();
} }
private void saveConfig() { private void saveConfig() {
try { try {
saveHashtable(configFile, configProps, configComment); saveMap(configFile, configProps, configComment);
} catch (IOException e) { } catch (IOException e) {
System.out.println("ERROR: cannot write config file " + configFile.toString() + ": " + e.getMessage()); System.out.println("ERROR: cannot write config file " + configFile.toString() + ": " + e.getMessage());
} }
} }
public void deployThread(String threadName, String threadShortDescription, String threadLongDescription, serverThread newThread, serverLog log, long startupDelay) {
// add/remove action listener
public void deployAction(String actionName, String actionShortDescription, String actionLongDescription,
serverSwitchAction newAction) {
newAction.setLog(log);
newAction.setDescription(actionShortDescription, actionLongDescription);
switchActions.put(actionName, newAction);
}
public void undeployAction(String actionName) {
switchActions.remove(actionName);
}
public void deployThread(String threadName, String threadShortDescription, String threadLongDescription, serverThread newThread, long startupDelay) {
deployThread(threadName, threadShortDescription, threadLongDescription, deployThread(threadName, threadShortDescription, threadLongDescription,
newThread, log, startupDelay, newThread, startupDelay,
Long.parseLong(getConfig(threadName + "_idlesleep" , "novalue")), Long.parseLong(getConfig(threadName + "_idlesleep" , "novalue")),
Long.parseLong(getConfig(threadName + "_busysleep" , "novalue"))); Long.parseLong(getConfig(threadName + "_busysleep" , "novalue")));
} }
public void deployThread(String threadName, String threadShortDescription, String threadLongDescription, serverThread newThread, serverLog log, long startupDelay, long initialIdleSleep, long initialBusySleep) { public void deployThread(String threadName, String threadShortDescription, String threadLongDescription, serverThread newThread, long startupDelay, long initialIdleSleep, long initialBusySleep) {
if (newThread.isAlive()) throw new RuntimeException("undeployed threads must not live; they are started as part of the deployment"); if (newThread.isAlive()) throw new RuntimeException("undeployed threads must not live; they are started as part of the deployment");
newThread.setStartupSleep(startupDelay); newThread.setStartupSleep(startupDelay);
long sleep; long sleep;

@ -54,14 +54,23 @@ public interface serverSwitch {
// the root path for the application // the root path for the application
public String getRootPath(); public String getRootPath();
// the switchboard can be used to set and read properties // a logger for this switchboard
public void setConfig(String key, String value); public void setLog(serverLog log);
public String getConfig(String key, String dflt); public serverLog getLog();
public Enumeration configKeys();
// a switchboard can have action listener
// these listeners are hooks for numerous methods below
public void deployAction(String actionName,
String actionShortDescription,
String actionLongDescription,
serverSwitchAction action);
public void undeployAction(String actionName);
// the switchboard can manage worker threads // the switchboard can manage worker threads
public void deployThread(String threadName, String threadShortDescription, String threadLongDescription, public void deployThread(String threadName,
serverThread newThread, serverLog log, String threadShortDescription,
String threadLongDescription,
serverThread newThread,
long startupDelay, long initialIdleSleep, long initialBusySleep); long startupDelay, long initialIdleSleep, long initialBusySleep);
public serverThread getThread(String threadName); public serverThread getThread(String threadName);
public void setThreadSleep(String threadName, long idleMillis, long busyMillis); public void setThreadSleep(String threadName, long idleMillis, long busyMillis);
@ -69,6 +78,11 @@ public interface serverSwitch {
public void terminateAllThreads(boolean waitFor); public void terminateAllThreads(boolean waitFor);
public Iterator /*of serverThread-Names (String)*/ threadNames(); public Iterator /*of serverThread-Names (String)*/ threadNames();
// the switchboard can be used to set and read properties
public void setConfig(String key, String value);
public String getConfig(String key, String dflt);
public Iterator configKeys();
// the switchboard also shall maintain a job list // the switchboard also shall maintain a job list
// jobs can be queued by submitting a job object // jobs can be queued by submitting a job object
// to work off a queue job, use deQueue, which is meant to // to work off a queue job, use deQueue, which is meant to

@ -252,7 +252,7 @@ public final class yacy {
serverLog.logFailure("STARTUP", "Failed to start server. Probably port " + port + " already in use."); serverLog.logFailure("STARTUP", "Failed to start server. Probably port " + port + " already in use.");
} else { } else {
// first start the server // first start the server
sb.deployThread("10_httpd", "HTTPD Server/Proxy", "the HTTPD, used as web server and proxy", server, null, 0, 0, 0); sb.deployThread("10_httpd", "HTTPD Server/Proxy", "the HTTPD, used as web server and proxy", server, 0, 0, 0);
//server.start(); //server.start();
// open the browser window // open the browser window

@ -1,2 +1,2 @@
#plasmaParser configuration file #plasmaParser configuration file
#Sun May 08 22:26:31 CEST 2005 #Wed May 11 13:47:09 CEST 2005

Loading…
Cancel
Save