boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

- removed synchronization in serverSwitch which should improve speed

Browse Source 
- fixed wrong assert in network graph
- enhanced double check method in table class

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7511 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 14 years ago
parent 4bd65532da
commit af87af0d4c
6 changed files with 39 additions and 155 deletions
Show Stats Download Patch File Download Diff File

96
source/de/anomic/server/serverSwitch.java
Unescape View File

@ -63,12 +63,11 @@ public class serverSwitch {
protected boolean firstInit; protected boolean firstInit;
protected Log log; protected Log log;
protected int serverJobs; protected int serverJobs;
private Map<String, String> configProps; private ConcurrentHashMap<String, String> configProps;
private final Map<String, String> configRemoved; private final ConcurrentHashMap<String, String> configRemoved;
private final Map<InetAddress, String> authorization; private final ConcurrentHashMap<InetAddress, String> authorization;
private final TreeMap<String, BusyThread> workerThreads; private final TreeMap<String, BusyThread> workerThreads;
private final TreeMap<String, serverSwitchAction> switchActions; private final serverAccessTracker accessTracker;
private final serverAccessTracker accessTracker;
public serverSwitch(final File dataPath, final File appPath, final String initPath, final String configPath) { public serverSwitch(final File dataPath, final File appPath, final String initPath, final String configPath) {
// we initialize the switchboard with a property file, // we initialize the switchboard with a property file,
@ -86,7 +85,7 @@ public class serverSwitch {
new File(configFile.getParent()).mkdir(); new File(configFile.getParent()).mkdir();
// predefine init's // predefine init's
Map<String, String> initProps; ConcurrentHashMap<String, String> initProps;
if (initFile.exists()) if (initFile.exists())
initProps = FileUtils.loadMap(initFile); initProps = FileUtils.loadMap(initFile);
else else
@ -144,9 +143,6 @@ public class serverSwitch {
// init thread control // init thread control
workerThreads = new TreeMap<String, BusyThread>(); workerThreads = new TreeMap<String, BusyThread>();
// init switch actions
switchActions = new TreeMap<String, serverSwitchAction>();
// init busy state control // init busy state control
serverJobs = 0; serverJobs = 0;
@ -202,35 +198,9 @@ public class serverSwitch {
} }
public void setConfig(final String key, final String value) { public void setConfig(final String key, final String value) {
// perform action before setting new value // set the value
final Iterator<serverSwitchAction> bevore = switchActions.values().iterator(); final String oldValue = configProps.put(key, value);
final Iterator<serverSwitchAction> after = switchActions.values().iterator(); if (oldValue == null || !value.equals(oldValue)) saveConfig();
synchronized (configProps) {
serverSwitchAction action;
while (bevore.hasNext()) {
action = bevore.next();
try {
action.doBevoreSetConfig(key, value);
} catch (final Exception e) {
log.logSevere("serverAction bevoreSetConfig '" + action.getShortDescription() + "' failed with exception: " + e.getMessage());
}
}
// set the value
final Object oldValue = configProps.put(key, value);
saveConfig();
// perform actions afterwards
while (after.hasNext()) {
action = after.next();
try {
action.doAfterSetConfig(key, value, (oldValue == null) ? null : (String) oldValue);
} catch (final Exception e) {
log.logSevere("serverAction afterSetConfig '" + action.getShortDescription() + "' failed with exception: " + e.getMessage());
}
}
}
} }
public void removeConfig(final String key) { public void removeConfig(final String key) {
@ -241,26 +211,12 @@ public class serverSwitch {
* @see de.anomic.server.serverSwitch#getConfig(java.lang.String, java.lang.String) * @see de.anomic.server.serverSwitch#getConfig(java.lang.String, java.lang.String)
*/ */
public String getConfig(final String key, final String dflt) { public String getConfig(final String key, final String dflt) {
final Iterator<serverSwitchAction> i = switchActions.values().iterator(); // get the value
synchronized (configProps) { final String s = configProps.get(key);
// get the value
final Object s = configProps.get(key);
// do action
serverSwitchAction action;
while (i.hasNext()) {
action = i.next();
try {
action.doWhenGetConfig(key, (s == null) ? null : (String) s, dflt);
} catch (final Exception e) {
log.logSevere("serverAction whenGetConfig '" + action.getShortDescription() + "' failed with exception: " + e.getMessage());
}
}
// return value // return value
if (s == null) return dflt; if (s == null) return dflt;
return (String) s; return s;
}
} }
public long getConfigLong(final String key, final long dflt) { public long getConfigLong(final String key, final long dflt) {
@ -314,35 +270,19 @@ public class serverSwitch {
private void saveConfig() { private void saveConfig() {
try { try {
synchronized (configProps) { ConcurrentHashMap<String, String> configPropsCopy = new ConcurrentHashMap<String, String>();
FileUtils.saveMap(configFile, configProps, configComment); configPropsCopy.putAll(configProps); // avoid concurrency problems
} FileUtils.saveMap(configFile, configPropsCopy, configComment);
} catch (final IOException e) { } catch (final IOException e) {
log.logSevere("CONFIG: Cannot write config file " + configFile.toString() + ": " + e.getMessage()); log.logSevere("CONFIG: Cannot write config file " + configFile.toString() + ": " + e.getMessage());
System.out.println("ERROR: cannot write config file " + configFile.toString() + ": " + e.getMessage()); System.out.println("ERROR: cannot write config file " + configFile.toString() + ": " + e.getMessage());
} }
} }
public Map<String, String> getRemoved() { public ConcurrentHashMap<String, String> getRemoved() {
// returns configuration that had been removed during initialization // returns configuration that had been removed during initialization
return configRemoved; return configRemoved;
} }
// add/remove action listener
public void deployAction(final String actionName, final String actionShortDescription, final String actionLongDescription,
final serverSwitchAction newAction) {
newAction.setLog(log);
newAction.setDescription(actionShortDescription, actionLongDescription);
switchActions.put(actionName, newAction);
log.logInfo("Deployed Action '" + actionShortDescription + "', (" + switchActions.size() + " actions registered)");
}
public void undeployAction(final String actionName) {
final serverSwitchAction action = switchActions.get(actionName);
action.close();
switchActions.remove(actionName);
log.logInfo("Undeployed Action '" + action.getShortDescription() + "', (" + switchActions.size() + " actions registered)");
}
public void deployThread( public void deployThread(
final String threadName, final String threadName,

53
source/de/anomic/server/serverSwitchAction.java
Unescape View File

@ -1,53 +0,0 @@
// serverSwitchAction.java
// -------------------------------------
// (C) by Michael Peter Christen; mc@yacy.net
// first published on http://www.anomic.de
// Frankfurt, Germany, 2005
// last major change: 11.05.2005
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package de.anomic.server;
import net.yacy.kelondro.logging.Log;
public interface serverSwitchAction {
// --------------------------------------------------------------------------
// the following methods are implemented by serverSwitchAsbtractAction
// and do not need to be altered
public void setDescription(String shortText, String longText);
// sets a visible description string
public String getShortDescription();
// returns short description string for online display
public String getLongDescription();
// returns long description string for online display
public void setLog(Log log);
// defines a log where process states can be written to
// ---------------------------------------------------------------------
// the following methods are supposed to be implemented
// by extending a serverSwitchAbstractAction object
public void doBevoreSetConfig(String key, String newvalue);
public void doAfterSetConfig(String key, String newvalue, String oldvalue);
public void doWhenGetConfig(String key, String actualvalue, String defaultvalue);
public void close(); // called when an action is undeployed
}

2
source/de/anomic/yacy/graphics/NetworkGraph.java
Unescape View File

@ -253,8 +253,6 @@ public class NetworkGraph {
if (communicationTimeout >= 0) { if (communicationTimeout >= 0) {
Date horizon = new Date(System.currentTimeMillis() - communicationTimeout); Date horizon = new Date(System.currentTimeMillis() - communicationTimeout);
for (Hit event: yacyChannel.channels(yacyChannel.DHTRECEIVE)) { for (Hit event: yacyChannel.channels(yacyChannel.DHTRECEIVE)) {
assert event != null;
assert event.getPubDate() != null;
if (event == null || event.getPubDate() == null) continue; if (event == null || event.getPubDate() == null) continue;
if (event.getPubDate().after(horizon)) { if (event.getPubDate().after(horizon)) {
//System.out.println("*** NETWORK-DHTRECEIVE: " + event.getLink()); //System.out.println("*** NETWORK-DHTRECEIVE: " + event.getLink());

10
source/net/yacy/kelondro/index/HandleMap.java
Unescape View File

@ -248,19 +248,19 @@ public final class HandleMap implements Iterable<Row.Entry> {
return add(key, -1); return add(key, -1);
} }
public final synchronized ArrayList<Long[]> removeDoubles() throws RowSpaceExceededException { public final synchronized ArrayList<long[]> removeDoubles() throws RowSpaceExceededException {
final ArrayList<Long[]> report = new ArrayList<Long[]>(); final ArrayList<long[]> report = new ArrayList<long[]>();
Long[] is; long[] is;
int c; int c;
long l; long l;
final int initialSize = this.size(); final int initialSize = this.size();
for (final RowCollection rowset: index.removeDoubles()) { for (final RowCollection rowset: index.removeDoubles()) {
is = new Long[rowset.size()]; is = new long[rowset.size()];
c = 0; c = 0;
for (Row.Entry e: rowset) { for (Row.Entry e: rowset) {
l = e.getColLong(1); l = e.getColLong(1);
assert l < initialSize : "l = " + l + ", initialSize = " + initialSize; assert l < initialSize : "l = " + l + ", initialSize = " + initialSize;
is[c++] = Long.valueOf(l); is[c++] = l;
} }
report.add(is); report.add(is);
} }

22
source/net/yacy/kelondro/table/Table.java
Unescape View File

@ -205,7 +205,7 @@ public class Table implements Index, Iterable<Row.Entry> {
// remove doubles // remove doubles
if (!freshFile) { if (!freshFile) {
final ArrayList<Long[]> doubles = index.removeDoubles(); final ArrayList<long[]> doubles = index.removeDoubles();
//assert index.size() + doubles.size() + fail == i; //assert index.size() + doubles.size() + fail == i;
//System.out.println(" -removed " + doubles.size() + " doubles- done."); //System.out.println(" -removed " + doubles.size() + " doubles- done.");
if (!doubles.isEmpty()) { if (!doubles.isEmpty()) {
@ -214,15 +214,15 @@ public class Table implements Index, Iterable<Row.Entry> {
// first put back one element each // first put back one element each
final byte[] record = new byte[rowdef.objectsize]; final byte[] record = new byte[rowdef.objectsize];
key = new byte[rowdef.primaryKeyLength]; key = new byte[rowdef.primaryKeyLength];
for (final Long[] ds: doubles) { for (final long[] ds: doubles) {
this.file.get(ds[0].intValue(), record, 0); this.file.get((int) ds[0], record, 0);
System.arraycopy(record, 0, key, 0, rowdef.primaryKeyLength); System.arraycopy(record, 0, key, 0, rowdef.primaryKeyLength);
this.index.putUnique(key, ds[0].intValue()); this.index.putUnique(key, (int) ds[0]);
} }
// then remove the other doubles by removing them from the table, but do a re-indexing while doing that // then remove the other doubles by removing them from the table, but do a re-indexing while doing that
// first aggregate all the delete positions because the elements from the top positions must be removed first // first aggregate all the delete positions because the elements from the top positions must be removed first
final TreeSet<Long> delpos = new TreeSet<Long>(); final TreeSet<Long> delpos = new TreeSet<Long>();
for (final Long[] ds: doubles) { for (final long[] ds: doubles) {
for (int j = 1; j < ds.length; j++) delpos.add(ds[j]); for (int j = 1; j < ds.length; j++) delpos.add(ds[j]);
} }
// now remove the entries in a sorted way (top-down) // now remove the entries in a sorted way (top-down)
@ -381,11 +381,11 @@ public class Table implements Index, Iterable<Row.Entry> {
RowSet rows; RowSet rows;
final TreeSet<Long> d = new TreeSet<Long>(); final TreeSet<Long> d = new TreeSet<Long>();
final byte[] b = new byte[rowdef.objectsize]; final byte[] b = new byte[rowdef.objectsize];
Long L; long L;
Row.Entry inconsistentEntry; Row.Entry inconsistentEntry;
// iterate over all entries that have inconsistent index references // iterate over all entries that have inconsistent index references
long lastlog = System.currentTimeMillis(); long lastlog = System.currentTimeMillis();
ArrayList<Long[]> doubles; ArrayList<long[]> doubles;
try { try {
doubles = index.removeDoubles(); doubles = index.removeDoubles();
} catch (RowSpaceExceededException e) { } catch (RowSpaceExceededException e) {
@ -393,16 +393,16 @@ public class Table implements Index, Iterable<Row.Entry> {
table = null; table = null;
doubles = index.removeDoubles(); doubles = index.removeDoubles();
} }
for (final Long[] is: doubles) { for (final long[] is: doubles) {
// 'is' is the set of all indexes, that have the same reference // 'is' is the set of all indexes, that have the same reference
// we collect that entries now here // we collect that entries now here
rows = new RowSet(this.rowdef, is.length); rows = new RowSet(this.rowdef, is.length);
for (int j = 0; j < is.length; j++) { for (int j = 0; j < is.length; j++) {
L = is[j]; L = is[j];
assert L.intValue() < file.size() : "L.intValue() = " + L.intValue() + ", file.size = " + file.size(); // prevent ooBounds Exception assert (int) L < file.size() : "L.intValue() = " + (int) L + ", file.size = " + file.size(); // prevent ooBounds Exception
d.add(L); d.add(L);
if (L.intValue() >= file.size()) continue; // prevent IndexOutOfBoundsException if ((int) L >= file.size()) continue; // prevent IndexOutOfBoundsException
file.get(L.intValue(), b, 0); // TODO: fix IndexOutOfBoundsException here file.get((int) L, b, 0); // TODO: fix IndexOutOfBoundsException here
inconsistentEntry = rowdef.newEntry(b); inconsistentEntry = rowdef.newEntry(b);
try { try {
rows.addUnique(inconsistentEntry); rows.addUnique(inconsistentEntry);

11
source/net/yacy/kelondro/util/FileUtils.java
Unescape View File

@ -43,7 +43,6 @@ import java.io.Writer;
import java.nio.charset.Charset; import java.nio.charset.Charset;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Comparator; import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet; import java.util.HashSet;
import java.util.Iterator; import java.util.Iterator;
import java.util.LinkedList; import java.util.LinkedList;
@ -52,6 +51,7 @@ import java.util.Map;
import java.util.Set; import java.util.Set;
import java.util.StringTokenizer; import java.util.StringTokenizer;
import java.util.TreeSet; import java.util.TreeSet;
import java.util.concurrent.ConcurrentHashMap;
import java.util.zip.GZIPInputStream; import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream; import java.util.zip.GZIPOutputStream;
import java.util.zip.ZipEntry; import java.util.zip.ZipEntry;
@ -393,7 +393,7 @@ public final class FileUtils {
return set; return set;
} }
public static Map<String, String> loadMap(final File f) { public static ConcurrentHashMap<String, String> loadMap(final File f) {
// load props // load props
try { try {
final byte[] b = read(f); final byte[] b = read(f);
@ -500,15 +500,14 @@ public final class FileUtils {
forceMove(tf, file); forceMove(tf, file);
} }
public static ConcurrentHashMap<String, String> table(Reader r) {
public static Map<String, String> table(Reader r) {
BufferedReader br = new BufferedReader(r); BufferedReader br = new BufferedReader(r);
return table(new StringsIterator(br)); return table(new StringsIterator(br));
} }
public static Map<String, String> table(Iterator<String> li) { public static ConcurrentHashMap<String, String> table(Iterator<String> li) {
String line; String line;
final HashMap<String, String> props = new HashMap<String, String>(); final ConcurrentHashMap<String, String> props = new ConcurrentHashMap<String, String>();
while (li.hasNext()) { while (li.hasNext()) {
int pos = 0; int pos = 0;
line = li.next().trim(); line = li.next().trim();

Write Preview
Loading…
Cancel
Save