boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

performance hacks

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7977 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 14 years ago
parent 5ad7f9612b
commit 57d5529a01
5 changed files with 31 additions and 25 deletions
Show Stats Download Patch File Download Diff File

4
source/net/yacy/kelondro/blob/HeapModifier.java
Unescape View File

@ -109,7 +109,7 @@ public class HeapModifier extends HeapReader implements BLOB {
if (seek < 0) return;
// check consistency of the index
assert (checkKey(key, seek)) : "key compare failed; key = " + UTF8.String(key) + ", seek = " + seek;
//assert (checkKey(key, seek)) : "key compare failed; key = " + UTF8.String(key) + ", seek = " + seek;
// access the file and read the container
this.file.seek(seek);
@ -262,7 +262,7 @@ public class HeapModifier extends HeapReader implements BLOB {
if (pos < 0) return 0;
// check consistency of the index
assert checkKey(key, pos) : "key compare failed; key = " + UTF8.String(key) + ", seek = " + pos;
//assert checkKey(key, pos) : "key compare failed; key = " + UTF8.String(key) + ", seek = " + pos;
// access the file and read the container
file.seek(pos);

4
source/net/yacy/kelondro/blob/MapDataMining.java
Unescape View File

@ -259,14 +259,14 @@ public class MapDataMining extends MapHeap {
if (key == null) return;
// update elementCount
if ((sortfields != null) || (longaccfields != null) || (floataccfields != null)) {
if (sortfields != null || longaccfields != null || floataccfields != null) {
Map<String, String> map;
try {
map = super.get(key, false);
if (map != null) {
// update accumulators (subtract)
if ((longaccfields != null) || (floataccfields != null)) updateAcc(map, false);
if (longaccfields != null || floataccfields != null) updateAcc(map, false);
// remove from sortCluster
if (sortfields != null) deleteSortCluster(UTF8.String(key));

8
source/net/yacy/kelondro/logging/ThreadDump.java
Unescape View File

@ -117,14 +117,16 @@ public class ThreadDump extends HashMap<ThreadDump.StackTrace, List<String>> imp
// try to get the thread dump from yacy.log which is available when YaCy is started with
// startYACY.sh -l
if (!canProduceLockedBy(logFile)) return;
long sizeBefore = logFile.length();
long sizeBefore = 0;
if (canProduceLockedBy(logFile)) {
sizeBefore = logFile.length();
// get the current process PID
int pid = OS.getPID();
// call kill -3 on the pid
if (pid >= 0) try {OS.execSynchronous("kill -3 " + pid);} catch (IOException e) {}
}
// read the log from the dump
long sizeAfter = logFile.length();
@ -412,7 +414,9 @@ public class ThreadDump extends HashMap<ThreadDump.StackTrace, List<String>> imp
}
}
//dump.print();
assert dump != null;
Map<StackTrace, Integer> locks = dump.countLocks();
assert locks != null;
System.out.println("*** Thread Dump Lock report; dump size = " + dump.size() + ", locks = " + locks.size());
for (int i = 0; i < dump.size() + 10; i++) {
for (Map.Entry<StackTrace, Integer> entry: locks.entrySet()) {

18
source/net/yacy/kelondro/rwi/ReferenceContainerCache.java
Unescape View File

@ -61,7 +61,7 @@ public final class ReferenceContainerCache<ReferenceType extends Reference> exte
private final int termSize;
private final ByteOrder termOrder;
private final ContainerOrder<ReferenceType> containerOrder;
private Map<ByteArray, ReferenceContainer<ReferenceType>> cache;
private ConcurrentHashMap<ByteArray, ReferenceContainer<ReferenceType>> cache;
/**
* open an existing heap file in undefined mode
@ -511,20 +511,22 @@ public final class ReferenceContainerCache<ReferenceType extends Reference> exte
container.put(newEntry);
// synchronization: check if the entry is still empty and set new value
synchronized (this.cache) {
final ReferenceContainer<ReferenceType> containerNew = this.cache.put(tha, container);
final ReferenceContainer<ReferenceType> container0 = this.cache.put(tha, container);
if (container0 != null) synchronized (this.cache) {
// no luck here, we get a lock exclusively to sort this out
final ReferenceContainer<ReferenceType> containerNew = this.cache.put(tha, container0);
if (containerNew == null) return;
if (container == containerNew) {
if (container0 == containerNew) {
// The containers are the same, so nothing needs to be done
return;
}
// Now merge the smaller container into the lager.
// The other way around can become very slow
if (container.size() >= containerNew.size()) {
container.putAllRecent(containerNew);
this.cache.put(tha, container);
if (container0.size() >= containerNew.size()) {
container0.putAllRecent(containerNew);
this.cache.put(tha, container0);
} else {
containerNew.putAllRecent(container);
containerNew.putAllRecent(container0);
this.cache.put(tha, containerNew);
}
}

8
source/net/yacy/peers/yacySeed.java
Unescape View File

@ -178,9 +178,6 @@ public class yacySeed implements Cloneable, Comparable<yacySeed>, Comparator<yac
private String alternativeIP = null;
private long birthdate; // keep this value in ram since it is often used and may cause lockings in concurrent situations.
// use our own formatter to prevent concurrency locks with other processes
private final static GenericFormatter my_SHORT_SECOND_FORMATTER = new GenericFormatter(GenericFormatter.FORMAT_SHORT_SECOND, GenericFormatter.time_second);
public yacySeed(final String theHash, final ConcurrentMap<String, String> theDna) {
// create a seed with a pre-defined hash map
assert theHash != null;
@ -234,7 +231,7 @@ public class yacySeed implements Cloneable, Comparable<yacySeed>, Comparator<yac
this.dna.put(yacySeed.URL_IN, yacySeed.ZERO); // received URLs
// default first filling
this.dna.put(yacySeed.BDATE, my_SHORT_SECOND_FORMATTER.format());
this.dna.put(yacySeed.BDATE, GenericFormatter.SHORT_SECOND_FORMATTER.format());
this.dna.put(yacySeed.LASTSEEN, this.dna.get(yacySeed.BDATE)); // just as initial setting
this.dna.put(yacySeed.UTC, GenericFormatter.UTCDiffString());
this.dna.put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_VIRGIN); // virgin/junior/senior/principal
@ -524,6 +521,7 @@ public class yacySeed implements Cloneable, Comparable<yacySeed>, Comparator<yac
// because java thinks it must apply the UTC offset to the current time,
// to create a string that looks like our current time, it adds the local UTC offset to the
// time. To create a corrected UTC Date string, we first subtract the local UTC offset.
GenericFormatter my_SHORT_SECOND_FORMATTER = new GenericFormatter(GenericFormatter.FORMAT_SHORT_SECOND, GenericFormatter.time_second); // use our own formatter to prevent concurrency locks with other processes
final String ls = my_SHORT_SECOND_FORMATTER.format(new Date(System.currentTimeMillis() /*- DateFormatter.UTCDiff()*/));
//System.out.println("SETTING LAST-SEEN of " + this.getName() + " to " + ls);
this.dna.put(yacySeed.LASTSEEN, ls );
@ -534,6 +532,7 @@ public class yacySeed implements Cloneable, Comparable<yacySeed>, Comparator<yac
*/
public final long getLastSeenUTC() {
try {
GenericFormatter my_SHORT_SECOND_FORMATTER = new GenericFormatter(GenericFormatter.FORMAT_SHORT_SECOND, GenericFormatter.time_second); // use our own formatter to prevent concurrency locks with other processes
final long t = my_SHORT_SECOND_FORMATTER.parse(get(yacySeed.LASTSEEN, "20040101000000")).getTime();
// getTime creates a UTC time number. But in this case java thinks, that the given
// time string is a local time, which has a local UTC offset applied.
@ -563,6 +562,7 @@ public class yacySeed implements Cloneable, Comparable<yacySeed>, Comparator<yac
if (this.birthdate > 0) return this.birthdate;
long b;
try {
GenericFormatter my_SHORT_SECOND_FORMATTER = new GenericFormatter(GenericFormatter.FORMAT_SHORT_SECOND, GenericFormatter.time_second); // use our own formatter to prevent concurrency locks with other processes
b = my_SHORT_SECOND_FORMATTER.parse(get(yacySeed.BDATE, "20040101000000")).getTime();
} catch (final ParseException e) {
b = System.currentTimeMillis();

Write Preview
Loading…
Cancel
Save