boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

less synchronization, better thread dump tool

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7556 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 14 years ago
parent 8d14916c74
commit 7138f4036b
7 changed files with 223 additions and 43 deletions
Show Stats Download Patch File Download Diff File

34
htroot/Threaddump_p.java
Unescape View File

@ -32,7 +32,7 @@ import java.util.Map;
import java.util.ArrayList;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.kelondro.logging.ThreadDumpGenerator;
import net.yacy.kelondro.logging.ThreadDump;
import net.yacy.kelondro.util.MemoryControl;
import de.anomic.search.Switchboard;
@ -58,14 +58,14 @@ public class Threaddump_p {
final String versionstring = yacyBuildProperties.getVersion() + "/" + yacyBuildProperties.getSVNRevision();
Runtime runtime = Runtime.getRuntime();
ThreadDumpGenerator.bufferappend(buffer, plain, "************* Start Thread Dump " + dt + " *******************");
ThreadDumpGenerator.bufferappend(buffer, plain, "");
ThreadDumpGenerator.bufferappend(buffer, plain, "YaCy Version: " + versionstring);
ThreadDumpGenerator.bufferappend(buffer, plain, "Assigned   Memory = " + (runtime.maxMemory()));
ThreadDumpGenerator.bufferappend(buffer, plain, "Used       Memory = " + (runtime.totalMemory() - runtime.freeMemory()));
ThreadDumpGenerator.bufferappend(buffer, plain, "Available  Memory = " + (runtime.maxMemory() - runtime.totalMemory() + runtime.freeMemory()));
ThreadDumpGenerator.bufferappend(buffer, plain, "");
ThreadDumpGenerator.bufferappend(buffer, plain, "");
ThreadDump.bufferappend(buffer, plain, "************* Start Thread Dump " + dt + " *******************");
ThreadDump.bufferappend(buffer, plain, "");
ThreadDump.bufferappend(buffer, plain, "YaCy Version: " + versionstring);
ThreadDump.bufferappend(buffer, plain, "Assigned   Memory = " + (runtime.maxMemory()));
ThreadDump.bufferappend(buffer, plain, "Used       Memory = " + (runtime.totalMemory() - runtime.freeMemory()));
ThreadDump.bufferappend(buffer, plain, "Available  Memory = " + (runtime.maxMemory() - runtime.totalMemory() + runtime.freeMemory()));
ThreadDump.bufferappend(buffer, plain, "");
ThreadDump.bufferappend(buffer, plain, "");
int multipleCount = 100;
File appPath = sb.getAppPath();
@ -76,7 +76,7 @@ public class Threaddump_p {
traces.add(Thread.getAllStackTraces());
if (MemoryControl.available() < 20 * 1024 * 1024) break;
}
ThreadDumpGenerator.appendStackTraceStats(appPath, buffer, traces, plain, null);
ThreadDump.appendStackTraceStats(appPath, buffer, traces, plain, null);
/*
ThreadDumpGenerator.appendStackTraceStats(appPath, buffer, traces, plain, Thread.State.BLOCKED);
ThreadDumpGenerator.appendStackTraceStats(appPath, buffer, traces, plain, Thread.State.RUNNABLE);
@ -88,15 +88,15 @@ public class Threaddump_p {
} else {
// generate a single thread dump
final Map<Thread,StackTraceElement[]> stackTraces = Thread.getAllStackTraces();
ThreadDumpGenerator.appendStackTraces(appPath, buffer, stackTraces, plain, Thread.State.BLOCKED);
ThreadDumpGenerator.appendStackTraces(appPath, buffer, stackTraces, plain, Thread.State.RUNNABLE);
ThreadDumpGenerator.appendStackTraces(appPath, buffer, stackTraces, plain, Thread.State.TIMED_WAITING);
ThreadDumpGenerator.appendStackTraces(appPath, buffer, stackTraces, plain, Thread.State.WAITING);
ThreadDumpGenerator.appendStackTraces(appPath, buffer, stackTraces, plain, Thread.State.NEW);
ThreadDumpGenerator.appendStackTraces(appPath, buffer, stackTraces, plain, Thread.State.TERMINATED);
ThreadDump.appendStackTraces(appPath, buffer, stackTraces, plain, Thread.State.BLOCKED);
ThreadDump.appendStackTraces(appPath, buffer, stackTraces, plain, Thread.State.RUNNABLE);
ThreadDump.appendStackTraces(appPath, buffer, stackTraces, plain, Thread.State.TIMED_WAITING);
ThreadDump.appendStackTraces(appPath, buffer, stackTraces, plain, Thread.State.WAITING);
ThreadDump.appendStackTraces(appPath, buffer, stackTraces, plain, Thread.State.NEW);
ThreadDump.appendStackTraces(appPath, buffer, stackTraces, plain, Thread.State.TERMINATED);
}
ThreadDumpGenerator.bufferappend(buffer, plain, "************* End Thread Dump " + dt + " *******************");
ThreadDump.bufferappend(buffer, plain, "************* End Thread Dump " + dt + " *******************");
prop.put("plain_count", multipleCount);
prop.put("plain_content", buffer.toString());

3
source/de/anomic/search/ReferenceOrder.java
Unescape View File

@ -33,6 +33,7 @@ import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.Semaphore;
import net.yacy.cora.document.UTF8;
import net.yacy.cora.storage.DynamicScore;
import net.yacy.cora.storage.ScoreCluster;
import net.yacy.document.Condenser;
@ -152,7 +153,7 @@ public class ReferenceOrder {
if (max == null) max = iEntry.clone(); else max.max(iEntry);
out.put(iEntry); // must be after the min/max check to prevent that min/max is null in cardinal()
// update domcount
dom = new String(iEntry.metadataHash(), 6, 6);
dom = UTF8.String(iEntry.metadataHash(), 6, 6);
count = doms0.get(dom);
if (count == null) {
doms0.put(dom, int1);

2
source/de/anomic/yacy/yacySeed.java
Unescape View File

@ -234,7 +234,7 @@ public class yacySeed implements Cloneable, Comparable<yacySeed>, Comparator<yac
/**
* check the peer name: protect against usage as XSS hack
* @param name
* @param id
* @return a checked name without "<" and ">"
*/
final static Pattern ltp = Pattern.compile("<");

14
source/net/yacy/cora/document/UTF8.java
Unescape View File

@ -48,4 +48,18 @@ public class UTF8 {
return null;
}
}
/**
* using the string method with the default charset given as argument should prevent using the charset cache
* in FastCharsetProvider.java:118 which locks all concurrent threads using a new String() method
* @param bytes
* @return
*/
public static String String(byte[] bytes) {
return new String(bytes, charset);
}
public static String String(byte[] bytes, int offset, int length) {
return new String(bytes, charset);
}
}

22
source/net/yacy/kelondro/data/meta/URIMetadataRow.java
Unescape View File

@ -106,8 +106,6 @@ public class URIMetadataRow implements URIMetadata {
private final long ranking; // during generation of a search result this value is set
private Components comp;
private static final GenericFormatter mySHORT_DAY_FORMATTER = new GenericFormatter(GenericFormatter.FORMAT_SHORT_DAY, GenericFormatter.time_minute);
public URIMetadataRow() {
// create a dummy entry, good to produce poison objects
this.entry = rowdef.newEntry();
@ -229,18 +227,22 @@ public class URIMetadataRow implements URIMetadata {
this.entry = rowdef.newEntry();
this.entry.setCol(col_hash, url.hash()); // FIXME potential null pointer access
this.entry.setCol(col_comp, encodeComp(url, descr, dc_creator, tags, dc_publisher));
// create new formatters to make concurrency possible
GenericFormatter formatter = new GenericFormatter(GenericFormatter.FORMAT_SHORT_DAY, GenericFormatter.time_minute);
try {
encodeDate(col_mod, mySHORT_DAY_FORMATTER.parse(prop.getProperty("mod", "20000101")));
encodeDate(col_mod, formatter.parse(prop.getProperty("mod", "20000101")));
} catch (final ParseException e) {
encodeDate(col_mod, new Date());
}
try {
encodeDate(col_load, mySHORT_DAY_FORMATTER.parse(prop.getProperty("load", "20000101")));
encodeDate(col_load, formatter.parse(prop.getProperty("load", "20000101")));
} catch (final ParseException e) {
encodeDate(col_load, new Date());
}
try {
encodeDate(col_fresh, mySHORT_DAY_FORMATTER.parse(prop.getProperty("fresh", "20000101")));
encodeDate(col_fresh, formatter.parse(prop.getProperty("fresh", "20000101")));
} catch (final ParseException e) {
encodeDate(col_fresh, new Date());
}
@ -295,6 +297,10 @@ public class URIMetadataRow implements URIMetadata {
final StringBuilder s = new StringBuilder(300);
if (metadata == null) return null;
//System.out.println("author=" + comp.author());
// create new formatters to make concurrency possible
GenericFormatter formatter = new GenericFormatter(GenericFormatter.FORMAT_SHORT_DAY, GenericFormatter.time_minute);
try {
s.append("hash=").append(new String(hash()));
assert (s.toString().indexOf(0) < 0);
@ -308,11 +314,11 @@ public class URIMetadataRow implements URIMetadata {
assert (s.toString().indexOf(0) < 0);
s.append(",publisher=").append(crypt.simpleEncode(metadata.dc_publisher()));
assert (s.toString().indexOf(0) < 0);
s.append(",mod=").append(mySHORT_DAY_FORMATTER.format(moddate()));
s.append(",mod=").append(formatter.format(moddate()));
assert (s.toString().indexOf(0) < 0);
s.append(",load=").append(mySHORT_DAY_FORMATTER.format(loaddate()));
s.append(",load=").append(formatter.format(loaddate()));
assert (s.toString().indexOf(0) < 0);
s.append(",fresh=").append(mySHORT_DAY_FORMATTER.format(freshdate()));
s.append(",fresh=").append(formatter.format(freshdate()));
assert (s.toString().indexOf(0) < 0);
s.append(",referrer=").append(referrerHash() == null ? "" : new String(referrerHash()));
assert (s.toString().indexOf(0) < 0);

8
source/net/yacy/kelondro/index/RowCollection.java
Unescape View File

@ -345,11 +345,9 @@ public class RowCollection implements Iterable<Row.Entry>, Cloneable {
if ((chunkcache == null) || (rowdef == null)) return null; // case may appear during shutdown
Row.Entry entry;
final int addr = index * rowdef.objectsize;
synchronized (this) {
if (index >= chunkcount) return null;
if (addr + rowdef.objectsize > chunkcache.length) return null; // the whole chunk does not fit into the chunkcache
entry = rowdef.newEntry(chunkcache, addr, clone);
}
if (index >= chunkcount) return null;
if (addr + rowdef.objectsize > chunkcache.length) return null; // the whole chunk does not fit into the chunkcache
entry = rowdef.newEntry(chunkcache, addr, clone);
return entry;
}

183
source/net/yacy/kelondro/logging/ThreadDumpGenerator.java → source/net/yacy/kelondro/logging/ThreadDump.java
Unescape View File

@ -25,22 +25,105 @@
package net.yacy.kelondro.logging;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.SortedSet;
import java.util.TreeSet;
import java.util.Map.Entry;
import net.yacy.document.parser.html.CharacterCoding;
import net.yacy.kelondro.util.FileUtils;
import de.anomic.tools.nxTools;
public class ThreadDumpGenerator {
public class ThreadDump extends HashMap<ThreadDump.Thread, List<String>> implements Map<ThreadDump.Thread, List<String>> {
public static void appendStackTraces(final File rootPath, final StringBuilder buffer, final Map<Thread,StackTraceElement[]> stackTraces, final boolean plain, final Thread.State stateIn) {
private static final long serialVersionUID = -5587850671040354397L;
public class Thread {
public String name;
public Thread(String name) {
this.name = name;
}
public boolean equals(Object a) {
if (a == null) return false;
return this.name.equals(((Thread) a).name);
}
public boolean equals(Thread a) {
if (a == null) return false;
return this.name.equals(a.name);
}
public int hashCode() {
return this.name.hashCode();
}
public String toString() {
return this.name;
}
}
public class Lock {
public String id;
public Lock(String name) {
this.id = name;
}
public boolean equals(Object a) {
if (a == null) return false;
return this.id.equals(((Lock) a).id);
}
public boolean equals(Lock a) {
if (a == null) return false;
return this.id.equals(a.id);
}
public int hashCode() {
return this.id.hashCode();
}
public String toString() {
return this.id;
}
}
public static Map<java.lang.Thread, StackTraceElement[]> getAllStackTraces() {
return java.lang.Thread.getAllStackTraces();
}
public ThreadDump(File f) throws IOException {
this(new FileInputStream(f));
}
public ThreadDump(InputStream is) throws IOException {
super();
BufferedReader br = new BufferedReader(new InputStreamReader(is));
String line;
String thread = null;
int p;
List<String> list = new ArrayList<String>();
while ((line = br.readLine()) != null) {
if (line.isEmpty()) {
if (thread != null) {
this.put(new ThreadDump.Thread(thread), list);
}
list = new ArrayList<String>();
thread = null;
continue;
}
if (line.charAt(0) == '"' && (p = line.indexOf("\" prio=")) > 0) {
// start a new thread
thread = line.substring(1, p);
continue;
}
if (thread != null) {
list.add(line);
}
}
}
public static void appendStackTraces(final File rootPath, final StringBuilder buffer, final Map<java.lang.Thread, StackTraceElement[]> stackTraces, final boolean plain, final java.lang.Thread.State stateIn) {
bufferappend(buffer, plain, "THREADS WITH STATES: " + stateIn.toString());
bufferappend(buffer, plain, "");
// collect single dumps
@ -56,7 +139,7 @@ public class ThreadDumpGenerator {
bufferappend(buffer, plain, "");
}
public static void appendStackTraceStats(final File rootPath, final StringBuilder buffer, final ArrayList<Map<Thread,StackTraceElement[]>> traces, final boolean plain, final Thread.State stateIn) {
public static void appendStackTraceStats(final File rootPath, final StringBuilder buffer, final ArrayList<Map<java.lang.Thread, StackTraceElement[]>> traces, final boolean plain, final java.lang.Thread.State stateIn) {
if (stateIn != null) {
bufferappend(buffer, plain, "THREADS WITH STATES: " + stateIn.toString());
bufferappend(buffer, plain, "");
@ -85,13 +168,11 @@ public class ThreadDumpGenerator {
return max;
}
private static HashMap<String, Integer> dumpStatistic(final File rootPath, final ArrayList<Map<Thread,StackTraceElement[]>> stackTraces, final boolean plain, final Thread.State stateIn) {
Map<Thread,StackTraceElement[]> trace;
private static HashMap<String, Integer> dumpStatistic(final File rootPath, final ArrayList<Map<java.lang.Thread, StackTraceElement[]>> stackTraces, final boolean plain, final java.lang.Thread.State stateIn) {
HashMap<String, Integer> result = new HashMap<String, Integer>();
HashMap<String, SortedSet<String>> x;
int count;
for (int i = 0; i < stackTraces.size(); i++) {
trace = stackTraces.get(i);
for (Map<java.lang.Thread,StackTraceElement[]> trace: stackTraces) {
x = dumpCollection(rootPath, trace, plain, stateIn);
for (final Entry<String, SortedSet<String>> e: x.entrySet()) {
Integer c = result.get(e.getKey());
@ -106,13 +187,13 @@ public class ThreadDumpGenerator {
return result;
}
private static HashMap<String, SortedSet<String>> dumpCollection(final File appPath, final Map<Thread,StackTraceElement[]> stackTraces, final boolean plain, final Thread.State stateIn) {
private static HashMap<String, SortedSet<String>> dumpCollection(final File appPath, final Map<java.lang.Thread,StackTraceElement[]> stackTraces, final boolean plain, final java.lang.Thread.State stateIn) {
final File classPath = new File(appPath, "source");
Thread thread;
java.lang.Thread thread;
// collect single dumps
HashMap<String, SortedSet<String>> dumps = new HashMap<String, SortedSet<String>>();
for (final Entry<Thread, StackTraceElement[]> entry: stackTraces.entrySet()) {
for (final Entry<java.lang.Thread, StackTraceElement[]> entry: stackTraces.entrySet()) {
thread = entry.getKey();
final StackTraceElement[] stackTraceElements = entry.getValue();
StackTraceElement ste;
@ -189,4 +270,84 @@ public class ThreadDumpGenerator {
}
}
/**
* find all locks in this dump
* @return a map from lock ids to the name of the thread where the lock occurs
*/
public Map<Lock, Thread> locks() {
int p;
Map<Lock, Thread> locks = new HashMap<Lock, Thread>();
for (Map.Entry<Thread, List<String>> entry: this.entrySet()) {
for (String s: entry.getValue()) {
if ((p = s.indexOf("locked <")) > 0) {
locks.put(new Lock(s.substring(p + 8, s.indexOf('>'))), entry.getKey());
}
}
}
return locks;
}
/**
* check if a thread is locked by another thread
* @param threadName
* @return the thread id if there is a lock or null if there is none
*/
public Lock lockedBy(Thread threadName) {
int p;
List<String> list = this.get(threadName);
if (list == null) return null;
for (String s: list) {
if ((p = s.indexOf("<")) > 0 && s.indexOf("locked <") < 0) {
return new Lock(s.substring(p + 1, s.indexOf('>')));
}
}
return null;
}
public Map<Thread, Integer> countLocks() {
Map<Lock, Thread> locks = locks();
Map<Thread, Integer> count = new HashMap<Thread, Integer>();
for (Map.Entry<Lock, Thread> entry: locks.entrySet()) {
// look where the lock has an effect
int c = 0;
for (Thread thread: this.keySet()) if (entry.getKey().equals(lockedBy(thread))) c++;
if (c > 0) count.put(entry.getValue(), c);
}
return count;
}
public void print() {
for (Thread thread: this.keySet()) print(thread);
}
public void print(Thread thread) {
List<String> list = this.get(thread);
if (list == null) return;
System.out.println("Thread: " + thread);
for (String s: list) System.out.println(" " + s);
System.out.println("");
}
public static void main(String[] args) {
if (args.length == 2 && args[0].equals("-f")) {
File dumpfile = new File(args[1]);
ThreadDump dump = null;
try {
dump = new ThreadDump(dumpfile);
} catch (IOException e) {
e.printStackTrace();
}
//dump.print();
Map<Thread, Integer> locks = dump.countLocks();
for (int i = 0; i < dump.size() + 10; i++) {
for (Map.Entry<Thread, Integer> entry: locks.entrySet()) {
if (entry.getValue().intValue() == i) {
System.out.println("holds lock for " + i + " threads:");
dump.print(entry.getKey());
}
}
}
}
}
}
Write Preview
Loading…
Cancel
Save