boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

some performance hacks (CPU only, not IO)

Browse Source 
this will cause better computation speed for single- and multi-core;
there are enhancements that will speed up old and slow machines as well
as multi-core CPUs. Indexing of surrogates has been speed up
from 4000 PPM to over 20000 PPM on a simple dual core office computer.
Since the enhancements are mostly in core routines, the hack should also
speed up search performance.

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6276 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 16 years ago
parent 92407009b2
commit 1a9cfd8718
14 changed files with 191 additions and 119 deletions
Show Stats Download Patch File Download Diff File

29
htroot/PerformanceSearch_p.java
Unescape View File

@ -24,6 +24,7 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
import java.util.ArrayList;
import java.util.Date;
import java.util.Iterator;
@ -39,25 +40,27 @@ public class PerformanceSearch_p {
// return variable that accumulates replacements
final serverObjects prop = new serverObjects();
final Iterator<serverProfiling.Event> events = serverProfiling.history("SEARCH");
final ArrayList<serverProfiling.Event> events = serverProfiling.history("SEARCH");
int c = 0;
serverProfiling.Event event;
ProfilingGraph.searchEvent search;
long lastt = 0;
while (events.hasNext()) {
event = events.next();
search = (ProfilingGraph.searchEvent) event.payload;
prop.put("table_" + c + "_query", search.queryID);
prop.put("table_" + c + "_event", search.processName);
prop.putNum("table_" + c + "_count", search.resultCount);
prop.putNum("table_" + c + "_delta", event.time - lastt);
prop.put("table_" + c + "_time", (new Date(event.time)).toString());
prop.putNum("table_" + c + "_duration", search.duration);
c++;
lastt = event.time;
if (events != null) synchronized (events) {
Iterator<serverProfiling.Event> i = events.iterator();
while (i.hasNext()) {
event = i.next();
search = (ProfilingGraph.searchEvent) event.payload;
prop.put("table_" + c + "_query", search.queryID);
prop.put("table_" + c + "_event", search.processName);
prop.putNum("table_" + c + "_count", search.resultCount);
prop.putNum("table_" + c + "_delta", event.time - lastt);
prop.put("table_" + c + "_time", (new Date(event.time)).toString());
prop.putNum("table_" + c + "_duration", search.duration);
c++;
lastt = event.time;
}
}
prop.put("table", c);
return prop;
}
}

8
source/de/anomic/document/Condenser.java
Unescape View File

@ -87,7 +87,7 @@ public final class Condenser {
private final static int numlength = 5;
//private Properties analysis;
private TreeMap<String, Word> words; // a string (the words) to (indexWord) - relation
private Map<String, Word> words; // a string (the words) to (indexWord) - relation
private final int wordminsize;
private final int wordcut;
@ -108,7 +108,7 @@ public final class Condenser {
// added media words are flagged with the appropriate media flag
this.wordminsize = 3;
this.wordcut = 2;
this.words = new TreeMap<String, Word>();
this.words = new HashMap<String, Word>();
this.RESULT_FLAGS = new Bitfield(4);
// construct flag set for document
@ -260,7 +260,7 @@ public final class Condenser {
// subtracts the given stopwords from the word list
// the word list shrinkes. This returns the number of shrinked words
final int oldsize = words.size();
words = SetTools.excludeConstructive(words, stopwords);
SetTools.excludeDestructive(words, stopwords);
return oldsize - words.size();
}
@ -738,7 +738,7 @@ public final class Condenser {
}
static StringBuilder readSentence(final Reader reader, final boolean pre) throws IOException {
final StringBuilder s = new StringBuilder(40);
final StringBuilder s = new StringBuilder(80);
int nextChar;
char c, lc = ' '; // starting with ' ' as last character prevents that the result string starts with a ' '

2
source/de/anomic/document/Word.java
Unescape View File

@ -41,7 +41,7 @@ import de.anomic.yacy.yacySeedDB;
public class Word {
public static final int hashCacheSize = (int) (MemoryControl.available() / 10000L);
public static final int hashCacheSize = Math.max(2000, Math.min(100000, (int) (MemoryControl.available() / 20000L)));
private static final SimpleARC<String, byte[]> hashCache = new SimpleARC<String, byte[]>(hashCacheSize);
// object carries statistics for words and sentences

2
source/de/anomic/kelondro/index/ObjectIndexCache.java
Unescape View File

@ -54,7 +54,7 @@ public class ObjectIndexCache implements ObjectIndex {
public synchronized void reset(final int initialspace) {
this.index0 = null; // first flush RAM to make room
this.index0 = new RowSet(rowdef, initialspace);
this.index0 = new RowSet(rowdef, initialspace);
this.index1 = null; // to show that this is the initialization phase
}

36
source/de/anomic/kelondro/index/RowCollection.java
Unescape View File

@ -49,7 +49,7 @@ import de.anomic.yacy.logging.Log;
public class RowCollection implements Iterable<Row.Entry> {
public static final double growfactor = 1.4;
public static final long growfactor100 = 140L;
private static final int isortlimit = 20;
static final Integer dummy = 0;
@ -142,8 +142,10 @@ public class RowCollection implements Iterable<Row.Entry> {
return (int) (time / day) - 10957;
}
private static Column exportColumn0, exportColumn1, exportColumn2, exportColumn3, exportColumn4;
private static Row exportRow(final int chunkcachelength) {
// find out the size of this collection
/*
return new Row(
"int size-4 {b256}," +
"short lastread-2 {b256}," + // as daysSince2000
@ -153,6 +155,23 @@ public class RowCollection implements Iterable<Row.Entry> {
"byte[] collection-" + chunkcachelength,
NaturalOrder.naturalOrder
);
*/
if (exportColumn0 == null) exportColumn0 = new Column("int size-4 {b256}");
if (exportColumn1 == null) exportColumn1 = new Column("short lastread-2 {b256}");
if (exportColumn2 == null) exportColumn2 = new Column("short lastwrote-2 {b256}");
if (exportColumn3 == null) exportColumn3 = new Column("byte[] orderkey-2");
if (exportColumn4 == null) exportColumn4 = new Column("int orderbound-4 {b256}");
/*
* because of a strange bug these objects cannot be initialized as normal
* static final. If I try that, they are not initialized and are assigned null. why?
*/
return new Row(new Column[]{
exportColumn0, exportColumn1, exportColumn2, exportColumn3, exportColumn4,
new Column("byte[] collection-" + chunkcachelength)
},
NaturalOrder.naturalOrder
);
}
public static final int exportOverheadSize = 14;
@ -183,9 +202,12 @@ public class RowCollection implements Iterable<Row.Entry> {
}
protected final void ensureSize(final int elements) {
final int needed = elements * rowdef.objectsize;
assert elements > 0 : "elements = " + elements;
final long needed = elements * rowdef.objectsize;
if (chunkcache.length >= needed) return;
byte[] newChunkcache = new byte[(int) (needed * growfactor)]; // increase space
assert needed > 0 : "needed = " + needed;
assert needed * growfactor100 / 100L > 0 : "elements = " + elements + ", new = " + (needed * growfactor100 / 100L);
byte[] newChunkcache = new byte[(int) (needed * growfactor100 / 100L)]; // increase space
System.arraycopy(chunkcache, 0, newChunkcache, 0, chunkcache.length);
chunkcache = newChunkcache;
}
@ -193,17 +215,17 @@ public class RowCollection implements Iterable<Row.Entry> {
/**
* compute the needed memory in case of a cache extension. That is, if the cache is full and must
* be copied into a new cache which is larger. In such a case the Collection needs more than the double size
* than is necessary to store the data. This method coputes the extra memory that is needed to perform this task.
* than is necessary to store the data. This method computes the extra memory that is needed to perform this task.
* @return
*/
public final long memoryNeededForGrow() {
return (long) ((((long) (chunkcount + 1)) * ((long) rowdef.objectsize)) * growfactor);
return (long) ((((long) (chunkcount + 1)) * ((long) rowdef.objectsize)) * growfactor100 / 100L);
}
public synchronized void trim(final boolean plusGrowFactor) {
if (chunkcache.length == 0) return;
int needed = chunkcount * rowdef.objectsize;
if (plusGrowFactor) needed = (int) (needed * growfactor);
if (plusGrowFactor) needed = (int) (((long) needed) * growfactor100 / 100L);
if (needed >= chunkcache.length)
return; // in case that the growfactor causes that the cache would
// grow instead of shrink, simply ignore the growfactor

57
source/de/anomic/kelondro/table/SplitTable.java
Unescape View File

@ -114,14 +114,6 @@ public class SplitTable implements ObjectIndex {
public void init() {
current = null;
// init the thread pool for the keeperOf executor service
this.executor = new ThreadPoolExecutor(
Runtime.getRuntime().availableProcessors() + 1,
Runtime.getRuntime().availableProcessors() + 1, 10,
TimeUnit.SECONDS,
new LinkedBlockingQueue<Runnable>(),
new NamePrefixThreadFactory(prefix));
// initialized tables map
this.tables = new HashMap<String, ObjectIndex>();
if (!(path.exists())) path.mkdirs();
@ -199,6 +191,16 @@ public class SplitTable implements ObjectIndex {
tables.put(maxf, table);
}
}
// init the thread pool for the keeperOf executor service
this.executor = new ThreadPoolExecutor(
Math.max(tables.size(), Runtime.getRuntime().availableProcessors()) + 1,
Math.max(tables.size(), Runtime.getRuntime().availableProcessors()) + 1, 10,
TimeUnit.SECONDS,
new LinkedBlockingQueue<Runnable>(),
new NamePrefixThreadFactory(prefix));
}
public void clear() throws IOException {
@ -304,20 +306,46 @@ public class SplitTable implements ObjectIndex {
keeper.put(row);
}
private static final class ReadyCheck {
private boolean r;
public ReadyCheck() {
this.r = false;
}
public void isReady() {
this.r = true;
}
public boolean check() {
return this.r;
}
}
public synchronized ObjectIndex keeperOf(final byte[] key) {
if (tables.size() < 2) {
// no concurrency if not needed
for (final ObjectIndex table : tables.values()) {
if (table.has(key)) return table;
}
return null;
}
// because the index is stored only in one table,
// and the index is completely in RAM, a concurrency will create
// not concurrent File accesses
//long start = System.currentTimeMillis();
// start a concurrent query to database tables
final CompletionService<ObjectIndex> cs = new ExecutorCompletionService<ObjectIndex>(executor);
int accepted = 0;
final ReadyCheck ready = new ReadyCheck();
for (final ObjectIndex table : tables.values()) {
if (ready.check()) break; // found already a table
try {
cs.submit(new Callable<ObjectIndex>() {
public ObjectIndex call() {
if (table.has(key)) return table;
if (table.has(key)) {
ready.isReady();
return table;
}
return dummyIndex;
}
});
@ -333,15 +361,11 @@ public class SplitTable implements ObjectIndex {
try {
for (int i = 0; i < accepted; i++) {
final Future<ObjectIndex> f = cs.take();
//hash(System.out.println("**********accepted = " + accepted + ", i =" + i);
if (f == null) continue;
final ObjectIndex index = f.get();
if (index != dummyIndex) {
//System.out.println("*DEBUG SplitTable success.time = " + (System.currentTimeMillis() - start) + " ms");
return index;
}
if (index == dummyIndex) continue;
return index;
}
//System.out.println("*DEBUG SplitTable fail.time = " + (System.currentTimeMillis() - start) + " ms");
return null;
} catch (final InterruptedException e) {
Thread.currentThread().interrupt();
@ -349,7 +373,6 @@ public class SplitTable implements ObjectIndex {
e.printStackTrace();
throw new RuntimeException(e.getCause());
}
//System.out.println("*DEBUG SplitTable fail.time = " + (System.currentTimeMillis() - start) + " ms");
return null;
}

6
source/de/anomic/kelondro/table/Table.java
Unescape View File

@ -266,9 +266,9 @@ public class Table implements ObjectIndex {
final HashMap<String, String> map = new HashMap<String, String>();
map.put("tableSize", Integer.toString(index.size()));
map.put("tableKeyChunkSize", Integer.toString(index.row().objectsize));
map.put("tableKeyMem", Integer.toString((int) (index.row().objectsize * index.size() * RowCollection.growfactor)));
map.put("tableKeyMem", Integer.toString((int) (((long) index.row().objectsize) * ((long) index.size()) * RowCollection.growfactor100 / 100L)));
map.put("tableValueChunkSize", (table == null) ? "0" : Integer.toString(table.row().objectsize));
map.put("tableValueMem", (table == null) ? "0" : Integer.toString((int) (table.row().objectsize * table.size() * RowCollection.growfactor)));
map.put("tableValueMem", (table == null) ? "0" : Integer.toString((int) (((long) index.row().objectsize) * ((long) index.size()) * RowCollection.growfactor100 / 100L)));
return map;
}
@ -277,7 +277,7 @@ public class Table implements ObjectIndex {
}
public static int staticRAMIndexNeed(final File f, final Row rowdef) {
return (int) ((rowdef.primaryKeyLength + 4) * tableSize(f, rowdef.objectsize) * RowCollection.growfactor);
return (int) (((long)(rowdef.primaryKeyLength + 4)) * ((long) tableSize(f, rowdef.objectsize)) * RowCollection.growfactor100 / 100L);
}
public synchronized void addUnique(final Entry row) throws IOException {

6
source/de/anomic/kelondro/text/IndexCell.java
Unescape View File

@ -52,7 +52,7 @@ import de.anomic.server.serverProfiling;
public final class IndexCell<ReferenceType extends Reference> extends AbstractBufferedIndex<ReferenceType> implements BufferedIndex<ReferenceType> {
private static final long cleanupCycle = 10000;
private static final long cleanupCycle = 60000;
// class variables
private final ReferenceContainerArray<ReferenceType> array;
@ -101,7 +101,7 @@ public final class IndexCell<ReferenceType extends Reference> extends AbstractBu
*/
public void add(ReferenceContainer<ReferenceType> newEntries) throws IOException {
this.ram.add(newEntries);
if (this.ram.size() % 100 == 0) {
if (this.ram.size() % 1000 == 0 || this.lastCleanup + cleanupCycle < System.currentTimeMillis()) {
serverProfiling.update("wordcache", Long.valueOf(this.ram.size()), true);
cleanCache();
}
@ -109,7 +109,7 @@ public final class IndexCell<ReferenceType extends Reference> extends AbstractBu
public void add(byte[] termHash, ReferenceType entry) throws IOException {
this.ram.add(termHash, entry);
if (this.ram.size() % 100 == 0) {
if (this.ram.size() % 1000 == 0 || this.lastCleanup + cleanupCycle < System.currentTimeMillis()) {
serverProfiling.update("wordcache", Long.valueOf(this.ram.size()), true);
cleanCache();
}

56
source/de/anomic/kelondro/text/ReferenceContainerCache.java
Unescape View File

@ -28,11 +28,11 @@ package de.anomic.kelondro.text;
import java.io.File;
import java.io.IOException;
import java.util.Arrays;
import java.util.Comparator;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
import java.util.SortedMap;
import java.util.TreeMap;
import java.util.concurrent.ConcurrentHashMap;
import de.anomic.kelondro.blob.HeapReader;
@ -48,8 +48,15 @@ import de.anomic.yacy.logging.Log;
public final class ReferenceContainerCache<ReferenceType extends Reference> extends AbstractIndex<ReferenceType> implements Index<ReferenceType>, IndexReader<ReferenceType>, Iterable<ReferenceContainer<ReferenceType>> {
public class ContainerOrder implements Comparator<ReferenceContainer<ReferenceType>> {
public int compare(ReferenceContainer<ReferenceType> arg0, ReferenceContainer<ReferenceType> arg1) {
return termOrder.compare(arg0.getTermHash(), arg1.getTermHash());
}
}
private final Row payloadrow;
private final ByteOrder termOrder;
protected final ByteOrder termOrder;
private final ContainerOrder containerOrder;
protected Map<ByteArray, ReferenceContainer<ReferenceType>> cache;
/**
@ -63,6 +70,7 @@ public final class ReferenceContainerCache<ReferenceType extends Reference> exte
super(factory);
this.payloadrow = payloadrow;
this.termOrder = termOrder;
this.containerOrder = new ContainerOrder();
this.cache = null;
}
@ -102,17 +110,15 @@ public final class ReferenceContainerCache<ReferenceType extends Reference> exte
final long startTime = System.currentTimeMillis();
// sort the map
SortedMap<byte[], ReferenceContainer<ReferenceType>> cachecopy = sortedClone();
ReferenceContainer<ReferenceType>[] cachecopy = sortedClone();
// write wCache
long wordcount = 0, urlcount = 0;
byte[] wordHash = null, lwh;
ReferenceContainer<ReferenceType> container;
for (final Map.Entry<byte[], ReferenceContainer<ReferenceType>> entry: cachecopy.entrySet()) {
for (final ReferenceContainer<ReferenceType> container: cachecopy) {
// get entries
lwh = wordHash;
wordHash = entry.getKey();
container = entry.getValue();
wordHash = container.getTermHash();
// check consistency: entries must be ordered
assert (lwh == null || this.ordering().compare(wordHash, lwh) > 0);
@ -139,6 +145,19 @@ public final class ReferenceContainerCache<ReferenceType extends Reference> exte
}
}
@SuppressWarnings("unchecked")
public ReferenceContainer<ReferenceType>[] sortedClone() {
ReferenceContainer<ReferenceType>[] cachecopy = new ReferenceContainer[cache.size()];
synchronized (cache) {
int p = 0;
for (final Map.Entry<ByteArray, ReferenceContainer<ReferenceType>> entry: cache.entrySet()) {
cachecopy[p++] = entry.getValue();
}
}
Arrays.sort(cachecopy, this.containerOrder);
return cachecopy;
}
/*
public SortedMap<byte[], ReferenceContainer<ReferenceType>> sortedClone() {
SortedMap<byte[], ReferenceContainer<ReferenceType>> cachecopy;
synchronized (cache) {
@ -149,7 +168,7 @@ public final class ReferenceContainerCache<ReferenceType extends Reference> exte
}
return cachecopy;
}
*/
public int size() {
return (this.cache == null) ? 0 : this.cache.size();
}
@ -252,14 +271,15 @@ public final class ReferenceContainerCache<ReferenceType extends Reference> exte
// plus the mentioned features
private final boolean rot;
private Iterator<ReferenceContainer<ReferenceType>> iterator;
private ReferenceContainer<ReferenceType>[] cachecopy;
private int p;
private byte[] latestTermHash;
public heapCacheIterator(byte[] startWordHash, final boolean rot) {
this.rot = rot;
if (startWordHash != null && startWordHash.length == 0) startWordHash = null;
SortedMap<byte[], ReferenceContainer<ReferenceType>> cachecopy = sortedClone();
this.iterator = (startWordHash == null) ? cachecopy.values().iterator() : cachecopy.tailMap(startWordHash).values().iterator();
this.cachecopy = sortedClone();
this.p = 0;
this.latestTermHash = null;
// The collection's iterator will return the values in the order that their corresponding keys appear in the tree.
}
@ -270,12 +290,12 @@ public final class ReferenceContainerCache<ReferenceType extends Reference> exte
public boolean hasNext() {
if (rot) return true;
return iterator.hasNext();
return this.p < this.cachecopy.length;
}
public ReferenceContainer<ReferenceType> next() {
if (iterator.hasNext()) {
ReferenceContainer<ReferenceType> c = iterator.next();
if (this.p < this.cachecopy.length) {
ReferenceContainer<ReferenceType> c = this.cachecopy[this.p++];
this.latestTermHash = c.getTermHash();
return c.topLevelClone();
}
@ -283,14 +303,14 @@ public final class ReferenceContainerCache<ReferenceType extends Reference> exte
if (!rot) {
return null;
}
iterator = cache.values().iterator();
ReferenceContainer<ReferenceType> c = iterator.next();
p = 0;
ReferenceContainer<ReferenceType> c = this.cachecopy[this.p++];
this.latestTermHash = c.getTermHash();
return c.topLevelClone();
}
public void remove() {
iterator.remove();
System.arraycopy(this.cachecopy, this.p, this.cachecopy, this.p - 1, this.cachecopy.length - p);
cache.remove(new ByteArray(this.latestTermHash));
}

11
source/de/anomic/kelondro/text/ReferenceOrder.java
Unescape View File

@ -43,11 +43,12 @@ import de.anomic.search.RankingProcess;
import de.anomic.yacy.yacyURL;
public class ReferenceOrder {
private WordReferenceVars min, max;
private final RankingProfile ranking;
private final ScoreCluster<String> doms; // collected for "authority" heuristic
private int maxdomcount;
private String language;
protected int maxdomcount;
protected WordReferenceVars min, max;
protected final ScoreCluster<String> doms; // collected for "authority" heuristic
private final RankingProfile ranking;
private String language;
public ReferenceOrder(final RankingProfile profile, String language) {
this.min = null;

28
source/de/anomic/kelondro/util/SetTools.java
Unescape View File

@ -38,7 +38,6 @@ import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.TreeSet;
import java.util.Map.Entry;
public class SetTools {
@ -294,17 +293,18 @@ public class SetTools {
// ------------------------------------------------------------------------------------------------
// exclude
public static <A, B> TreeMap<A, B> excludeConstructive(final TreeMap<A, B> map, final TreeSet<A> set) {
// comparators must be equal
/*
public static <A, B> TreeMap<A, B> excludeConstructive(final TreeMap<A, B> map, final Set<A> set) {
if (map == null) return null;
if (set == null) return map;
if ((map.size() == 0) || (set.size() == 0)) return map;
if (map.comparator() != set.comparator()) return excludeConstructiveByTestMapInSet(map, set);
assert !(set instanceof TreeSet) || map.comparator() == ((TreeSet<A>) set).comparator();
// if (map.comparator() != set.comparator()) return excludeConstructiveByTestMapInSet(map, set);
return excludeConstructiveByTestMapInSet(map, set);
// return excludeConstructiveByEnumeration(map, set);
}
private static <A, B> TreeMap<A, B> excludeConstructiveByTestMapInSet(final TreeMap<A, B> map, final TreeSet<A> set) {
private static <A, B> TreeMap<A, B> excludeConstructiveByTestMapInSet(final TreeMap<A, B> map, final Set<A> set) {
final TreeMap<A, B> result = new TreeMap<A, B>(map.comparator());
A o;
for (Entry<A, B> entry: map.entrySet()) {
@ -313,12 +313,13 @@ public class SetTools {
}
return result;
}
*/
public static <A, B> void excludeDestructive(final TreeMap<A, B> map, final TreeSet<A> set) {
public static <A, B> void excludeDestructive(final Map<A, B> map, final Set<A> set) {
// comparators must be equal
if (map == null) return;
if (set == null) return;
if (map.comparator() != set.comparator()) return;
assert !(map instanceof TreeMap && set instanceof TreeSet) || ((TreeMap<A, B>) map).comparator() == ((TreeSet<A>) set).comparator();
if ((map.size() == 0) || (set.size() == 0)) return;
if (map.size() < set.size())
@ -327,22 +328,21 @@ public class SetTools {
excludeDestructiveByTestSetInMap(map, set);
}
private static <A, B> void excludeDestructiveByTestMapInSet(final TreeMap<A, B> map, final TreeSet<A> set) {
private static <A, B> void excludeDestructiveByTestMapInSet(final Map<A, B> map, final Set<A> set) {
final Iterator<A> mi = map.keySet().iterator();
while (mi.hasNext()) if (set.contains(mi.next())) mi.remove();
}
private static <A, B> void excludeDestructiveByTestSetInMap(final TreeMap<A, B> map, final TreeSet<A> set) {
private static <A, B> void excludeDestructiveByTestSetInMap(final Map<A, B> map, final Set<A> set) {
final Iterator<A> si = set.iterator();
while (si.hasNext()) map.remove(si.next());
}
// and the same again with set-set
public static <A> void excludeDestructive(final TreeSet<A> set1, final TreeSet<A> set2) {
// comparators must be equal
public static <A> void excludeDestructive(final Set<A> set1, final Set<A> set2) {
if (set1 == null) return;
if (set2 == null) return;
if (set1.comparator() != set2.comparator()) return;
assert !(set1 instanceof TreeSet && set2 instanceof TreeSet) || ((TreeSet<A>) set1).comparator() == ((TreeSet<A>) set2).comparator();
if ((set1.size() == 0) || (set2.size() == 0)) return;
if (set1.size() < set2.size())
@ -351,12 +351,12 @@ public class SetTools {
excludeDestructiveByTestLargeInSmall(set1, set2);
}
private static <A> void excludeDestructiveByTestSmallInLarge(final TreeSet<A> small, final TreeSet<A> large) {
private static <A> void excludeDestructiveByTestSmallInLarge(final Set<A> small, final Set<A> large) {
final Iterator<A> mi = small.iterator();
while (mi.hasNext()) if (large.contains(mi.next())) mi.remove();
}
private static <A> void excludeDestructiveByTestLargeInSmall(final TreeSet<A> large, final TreeSet<A> small) {
private static <A> void excludeDestructiveByTestLargeInSmall(final Set<A> large, final Set<A> small) {
final Iterator<A> si = small.iterator();
while (si.hasNext()) large.remove(si.next());
}

2
source/de/anomic/search/Switchboard.java
Unescape View File

@ -1208,6 +1208,8 @@ public final class Switchboard extends serverAbstractSwitch implements serverSwi
Response response;
while ((surrogate = reader.take()) != DCEntry.poison) {
// check if url is in accepted domain
assert surrogate != null;
assert crawlStacker != null;
final String urlRejectReason = crawlStacker.urlInAcceptedDomain(surrogate.url());
if (urlRejectReason != null) {
if (this.log.isFine()) this.log.logInfo("Rejected URL '" + surrogate.url() + "': " + urlRejectReason);

30
source/de/anomic/server/serverProfiling.java
Unescape View File

@ -26,16 +26,16 @@
package de.anomic.server;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentLinkedQueue;
import de.anomic.kelondro.util.MemoryControl;
public class serverProfiling extends Thread {
private static final Map<String, ConcurrentLinkedQueue<Event>> historyMaps = new ConcurrentHashMap<String, ConcurrentLinkedQueue<Event>>();
private static final Map<String, ArrayList<Event>> historyMaps = new ConcurrentHashMap<String, ArrayList<Event>>();
private static final Map<String, Long> eventAccess = new ConcurrentHashMap<String, Long>(); // value: last time when this was accessed
private static serverProfiling systemProfiler = null;
@ -84,26 +84,26 @@ public class serverProfiling extends Thread {
return; // protect against too heavy load
}
}
ConcurrentLinkedQueue<Event> history = historyMaps.get(eventName);
if (history != null) {
ArrayList<Event> history = historyMaps.get(eventName);
if (history != null) synchronized (history) {
// update entry
history.add(new Event(eventPayload));
// clean up too old entries
int tp = history.size() - 30000;
while (tp-- > 0) history.poll();
while (tp-- > 0) history.remove(0);
if (history.size() % 10 == 0) { // reduce number of System.currentTimeMillis() calls
Event e;
final long now = System.currentTimeMillis();
while (history.size() > 0) {
e = history.peek();
e = history.get(0);
if (now - e.time < 600000) break;
history.poll();
history.remove(0);
}
}
} else {
history = new ConcurrentLinkedQueue<Event>();
history = new ArrayList<Event>(100);
// update entry
history.add(new Event(eventPayload));
@ -113,16 +113,20 @@ public class serverProfiling extends Thread {
}
}
public static Iterator<Event> history(final String eventName) {
return (historyMaps.containsKey(eventName) ? (historyMaps.get(eventName)) : new ConcurrentLinkedQueue<Event>()).iterator();
public static ArrayList<Event> history(final String eventName) {
return historyMaps.get(eventName);
}
public static int countEvents(final String eventName, long time) {
Iterator<Event> i = history(eventName);
ArrayList<Event> event = history(eventName);
if (event == null) return 0;
long now = System.currentTimeMillis();
int count = 0;
while (i.hasNext()) {
if (now - i.next().time < time) count++;
synchronized (event) {
Iterator<Event> i = event.iterator();
while (i.hasNext()) {
if (now - i.next().time < time) count++;
}
}
return count;
}

37
source/de/anomic/ymage/ProfilingGraph.java
Unescape View File

@ -26,8 +26,8 @@
package de.anomic.ymage;
import java.util.ArrayList;
import java.util.ConcurrentModificationException;
import java.util.Iterator;
import de.anomic.server.serverProfiling;
import de.anomic.server.serverProfiling.Event;
@ -37,13 +37,14 @@ public class ProfilingGraph {
private static ymageChart bufferChart = null;
public static long maxPayload(final String eventname, final long min) {
final Iterator<Event> i = serverProfiling.history(eventname);
serverProfiling.Event event;
final ArrayList<Event> list = serverProfiling.history(eventname);
if (list == null) return min;
long max = min, l;
while (i.hasNext()) {
event = i.next();
l = ((Long) event.payload).longValue();
if (l > max) max = l;
synchronized (list) {
for (serverProfiling.Event event: list) {
l = ((Long) event.payload).longValue();
if (l > max) max = l;
}
}
return max;
}
@ -79,7 +80,6 @@ public class ProfilingGraph {
final long now = System.currentTimeMillis();
long bytes;
int x0, x1, y0, y1, ppm, words;
serverProfiling.Event event;
try {
// draw urls
/*
@ -98,10 +98,9 @@ public class ProfilingGraph {
}
*/
// draw memory
Iterator<Event> i = serverProfiling.history("memory");
ArrayList<Event> events = serverProfiling.history("memory");
x0 = 1; y0 = 0;
while (i.hasNext()) {
event = i.next();
if (events != null) synchronized (events) {for (serverProfiling.Event event: events) {
time = event.time - now;
bytes = ((Long) event.payload).longValue();
x1 = (int) (time/1000);
@ -111,13 +110,12 @@ public class ProfilingGraph {
chart.setColor("0000FF");
if (x0 < 0) chart.chartLine(ymageChart.DIMENSION_BOTTOM, ymageChart.DIMENSION_RIGHT, x0, y0, x1, y1);
x0 = x1; y0 = y1;
}
}}
// draw wordcache
i = serverProfiling.history("wordcache");
events = serverProfiling.history("wordcache");
x0 = 1; y0 = 0;
while (i.hasNext()) {
event = i.next();
if (events != null) synchronized (events) {for (serverProfiling.Event event: events) {
time = event.time - now;
words = (int) ((Long) event.payload).longValue();
x1 = (int) (time/1000);
@ -127,13 +125,12 @@ public class ProfilingGraph {
chart.setColor("008800");
if (x0 < 0) chart.chartLine(ymageChart.DIMENSION_BOTTOM, ymageChart.DIMENSION_LEFT, x0, y0, x1, y1);
x0 = x1; y0 = y1;
}
}}
// draw ppm
i = serverProfiling.history("ppm");
events = serverProfiling.history("ppm");
x0 = 1; y0 = 0;
while (i.hasNext()) {
event = i.next();
if (events != null) synchronized (events) {for (serverProfiling.Event event: events) {
time = event.time - now;
ppm = (int) ((Long) event.payload).longValue();
x1 = (int) (time/1000);
@ -143,7 +140,7 @@ public class ProfilingGraph {
chart.setColor("AA2222");
chart.chartDot(ymageChart.DIMENSION_BOTTOM, ymageChart.DIMENSION_ANOT0, x1, y1, 2, ppm + " PPM", 0);
x0 = x1; y0 = y1;
}
}}
bufferChart = chart;
} catch (final ConcurrentModificationException cme) {

Write Preview
Loading…
Cancel
Save