boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

added more info to DNS cache in /PerformanceMemory_p.html

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7798 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 14 years ago
parent f803da8aae
commit d8072d1866
3 changed files with 65 additions and 27 deletions
Show Stats Download Patch File Download Diff File

18
htroot/PerformanceMemory_p.html
Unescape View File

@ -191,20 +191,28 @@
<td>Delete</td> <td>Delete</td>
</tr> </tr>
<tr class="TableCellLight"> <tr class="TableCellLight">
<td>DNSCache</td> <td>DNSCache/Hit</td>
<td></td>
<td>#[namecacheHit.size]#</td> <td>#[namecacheHit.size]#</td>
<td>#[namecacheHit.Hit]#</td>
<td>#[namecacheHit.Miss]#</td>
<td>#[namecacheHit.Insert]#</td>
<td>(ARC)</td>
</tr>
<tr class="TableCellLight">
<td>DNSCache/Miss</td>
<td>#[namecacheMiss.size]#</td> <td>#[namecacheMiss.size]#</td>
<td></td> <td>#[namecacheMiss.Hit]#</td>
<td></td> <td>#[namecacheMiss.Miss]#</td>
<td>#[namecacheMiss.Insert]#</td>
<td>(ARC)</td>
</tr> </tr>
<tr class="TableCellDark"> <tr class="TableCellDark">
<td>DNSNoCache</td> <td>DNSNoCache</td>
<td></td>
<td>#[namecache.noCache]#</td> <td>#[namecache.noCache]#</td>
<td></td> <td></td>
<td></td> <td></td>
<td></td> <td></td>
<td></td>
</tr> </tr>
<tr class="TableCellLight"> <tr class="TableCellLight">
<td>HashBlacklistedCache</td> <td>HashBlacklistedCache</td>

45
htroot/PerformanceMemory_p.java
Unescape View File

@ -38,25 +38,24 @@ import net.yacy.kelondro.table.Table;
import net.yacy.kelondro.util.FileUtils; import net.yacy.kelondro.util.FileUtils;
import net.yacy.kelondro.util.Formatter; import net.yacy.kelondro.util.Formatter;
import net.yacy.kelondro.util.MemoryControl; import net.yacy.kelondro.util.MemoryControl;
import de.anomic.search.SearchEventCache; import de.anomic.search.SearchEventCache;
import de.anomic.search.Switchboard; import de.anomic.search.Switchboard;
import de.anomic.server.serverObjects; import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch; import de.anomic.server.serverSwitch;
public class PerformanceMemory_p { public class PerformanceMemory_p {
private static final long KB = 1024; private static final long KB = 1024;
private static final long MB = 1024 * KB; private static final long MB = 1024 * KB;
private static Map<String, String> defaultSettings = null; private static Map<String, String> defaultSettings = null;
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
// return variable that accumulates replacements // return variable that accumulates replacements
final serverObjects prop = new serverObjects(); final serverObjects prop = new serverObjects();
if (defaultSettings == null) { if (defaultSettings == null) {
defaultSettings = FileUtils.loadMap(new File(env.getAppPath(), "defaults/yacy.init")); defaultSettings = FileUtils.loadMap(new File(env.getAppPath(), "defaults/yacy.init"));
} }
prop.put("gc", "0"); prop.put("gc", "0");
if (post != null) { if (post != null) {
if (post.containsKey("gc")) { if (post.containsKey("gc")) {
@ -64,7 +63,7 @@ public class PerformanceMemory_p {
prop.put("gc", "1"); prop.put("gc", "1");
} }
} }
final long memoryFreeNow = MemoryControl.free(); final long memoryFreeNow = MemoryControl.free();
final long memoryFreeAfterInitBGC = env.getConfigLong("memoryFreeAfterInitBGC", 0L); final long memoryFreeAfterInitBGC = env.getConfigLong("memoryFreeAfterInitBGC", 0L);
final long memoryFreeAfterInitAGC = env.getConfigLong("memoryFreeAfterInitAGC", 0L); final long memoryFreeAfterInitAGC = env.getConfigLong("memoryFreeAfterInitAGC", 0L);
@ -73,7 +72,7 @@ public class PerformanceMemory_p {
final long memoryTotalAfterInitBGC = env.getConfigLong("memoryTotalAfterInitBGC", 0L); final long memoryTotalAfterInitBGC = env.getConfigLong("memoryTotalAfterInitBGC", 0L);
final long memoryTotalAfterInitAGC = env.getConfigLong("memoryTotalAfterInitAGC", 0L); final long memoryTotalAfterInitAGC = env.getConfigLong("memoryTotalAfterInitAGC", 0L);
final long memoryTotalAfterStartup = env.getConfigLong("memoryTotalAfterStartup", 0L); final long memoryTotalAfterStartup = env.getConfigLong("memoryTotalAfterStartup", 0L);
prop.putNum("memoryMax", MemoryControl.maxMemory / MB); prop.putNum("memoryMax", MemoryControl.maxMemory / MB);
prop.putNum("memoryAvailAfterStartup", (MemoryControl.maxMemory - memoryTotalAfterStartup + memoryFreeAfterStartup) / MB); prop.putNum("memoryAvailAfterStartup", (MemoryControl.maxMemory - memoryTotalAfterStartup + memoryFreeAfterStartup) / MB);
prop.putNum("memoryAvailAfterInitBGC", (MemoryControl.maxMemory - memoryTotalAfterInitBGC + memoryFreeAfterInitBGC) / MB); prop.putNum("memoryAvailAfterInitBGC", (MemoryControl.maxMemory - memoryTotalAfterInitBGC + memoryFreeAfterInitBGC) / MB);
@ -91,7 +90,7 @@ public class PerformanceMemory_p {
prop.putNum("memoryUsedAfterInitBGC", (memoryTotalAfterInitBGC - memoryFreeAfterInitBGC) / KB); prop.putNum("memoryUsedAfterInitBGC", (memoryTotalAfterInitBGC - memoryFreeAfterInitBGC) / KB);
prop.putNum("memoryUsedAfterInitAGC", (memoryTotalAfterInitAGC - memoryFreeAfterInitAGC) / KB); prop.putNum("memoryUsedAfterInitAGC", (memoryTotalAfterInitAGC - memoryFreeAfterInitAGC) / KB);
prop.putNum("memoryUsedNow", (memoryTotalNow - memoryFreeNow) / MB); prop.putNum("memoryUsedNow", (memoryTotalNow - memoryFreeNow) / MB);
// write table for Table index sizes // write table for Table index sizes
Iterator<String> i = Table.filenames(); Iterator<String> i = Table.filenames();
String filename; String filename;
@ -109,19 +108,19 @@ public class PerformanceMemory_p {
totalmem += mem; totalmem += mem;
prop.put("EcoList_" + c + "_tableKeyMem", Formatter.bytesToString(mem)); prop.put("EcoList_" + c + "_tableKeyMem", Formatter.bytesToString(mem));
prop.put("EcoList_" + c + "_tableKeyChunkSize", map.get("tableKeyChunkSize")); prop.put("EcoList_" + c + "_tableKeyChunkSize", map.get("tableKeyChunkSize"));
mem = Long.parseLong(map.get("tableValueMem")); mem = Long.parseLong(map.get("tableValueMem"));
totalmem += mem; totalmem += mem;
prop.put("EcoList_" + c + "_tableValueMem", Formatter.bytesToString(mem)); prop.put("EcoList_" + c + "_tableValueMem", Formatter.bytesToString(mem));
prop.put("EcoList_" + c + "_tableValueChunkSize", map.get("tableValueChunkSize")); prop.put("EcoList_" + c + "_tableValueChunkSize", map.get("tableValueChunkSize"));
c++; c++;
} }
prop.put("EcoList", c); prop.put("EcoList", c);
prop.putNum("EcoIndexTotalMem", totalmem / (1024 * 1024d)); prop.putNum("EcoIndexTotalMem", totalmem / (1024 * 1024d));
// write object cache table // write object cache table
Iterator<Map.Entry<String, RAMIndex>> oi = RAMIndex.objects(); final Iterator<Map.Entry<String, RAMIndex>> oi = RAMIndex.objects();
c = 0; c = 0;
mem = 0; mem = 0;
Map.Entry<String, RAMIndex> oie; Map.Entry<String, RAMIndex> oie;
@ -130,25 +129,25 @@ public class PerformanceMemory_p {
while (oi.hasNext()) { while (oi.hasNext()) {
try { try {
oie = oi.next(); oie = oi.next();
} catch (ConcurrentModificationException e) { } catch (final ConcurrentModificationException e) {
break; break;
} }
filename = oie.getKey(); filename = oie.getKey();
cache = oie.getValue(); cache = oie.getValue();
prop.put("indexcache_" + c + "_Name", ((p = filename.indexOf("DATA")) < 0) ? filename : filename.substring(p)); prop.put("indexcache_" + c + "_Name", ((p = filename.indexOf("DATA")) < 0) ? filename : filename.substring(p));
hitmem = cache.mem(); hitmem = cache.mem();
totalhitmem += hitmem; totalhitmem += hitmem;
prop.put("indexcache_" + c + "_ChunkSize", cache.row().objectsize); prop.put("indexcache_" + c + "_ChunkSize", cache.row().objectsize);
prop.putNum("indexcache_" + c + "_Count", cache.size()); prop.putNum("indexcache_" + c + "_Count", cache.size());
prop.put("indexcache_" + c + "_NeededMem", cache.size() * cache.row().objectsize); prop.put("indexcache_" + c + "_NeededMem", cache.size() * cache.row().objectsize);
prop.put("indexcache_" + c + "_UsedMem", hitmem); prop.put("indexcache_" + c + "_UsedMem", hitmem);
c++; c++;
} }
prop.put("indexcache", c); prop.put("indexcache", c);
prop.putNum("indexcacheTotalMem", totalhitmem / (1024 * 1024d)); prop.putNum("indexcacheTotalMem", totalhitmem / (1024 * 1024d));
// write object cache table // write object cache table
i = Cache.filenames(); i = Cache.filenames();
c = 0; c = 0;
@ -158,7 +157,7 @@ public class PerformanceMemory_p {
filename = i.next(); filename = i.next();
map = Cache.memoryStats(filename); map = Cache.memoryStats(filename);
prop.put("ObjectList_" + c + "_objectCachePath", ((p = filename.indexOf("DATA")) < 0) ? filename : filename.substring(p)); prop.put("ObjectList_" + c + "_objectCachePath", ((p = filename.indexOf("DATA")) < 0) ? filename : filename.substring(p));
// hit cache // hit cache
hitmem = Long.parseLong(map.get("objectHitMem")); hitmem = Long.parseLong(map.get("objectHitMem"));
totalhitmem += hitmem; totalhitmem += hitmem;
@ -171,7 +170,7 @@ public class PerformanceMemory_p {
prop.putNum("ObjectList_" + c + "_objectHitCacheWriteDouble", map.get("objectHitCacheWriteDouble")); prop.putNum("ObjectList_" + c + "_objectHitCacheWriteDouble", map.get("objectHitCacheWriteDouble"));
prop.putNum("ObjectList_" + c + "_objectHitCacheDeletes", map.get("objectHitCacheDeletes")); prop.putNum("ObjectList_" + c + "_objectHitCacheDeletes", map.get("objectHitCacheDeletes"));
prop.putNum("ObjectList_" + c + "_objectHitCacheFlushes", map.get("objectHitCacheFlushes")); prop.putNum("ObjectList_" + c + "_objectHitCacheFlushes", map.get("objectHitCacheFlushes"));
// miss cache // miss cache
missmem = Long.parseLong(map.get("objectMissMem")); missmem = Long.parseLong(map.get("objectMissMem"));
totalmissmem += missmem; totalmissmem += missmem;
@ -184,7 +183,7 @@ public class PerformanceMemory_p {
prop.putNum("ObjectList_" + c + "_objectMissCacheWriteDouble", map.get("objectMissCacheWriteDouble")); prop.putNum("ObjectList_" + c + "_objectMissCacheWriteDouble", map.get("objectMissCacheWriteDouble"));
prop.putNum("ObjectList_" + c + "_objectMissCacheDeletes", map.get("objectMissCacheDeletes")); prop.putNum("ObjectList_" + c + "_objectMissCacheDeletes", map.get("objectMissCacheDeletes"));
//prop.put("ObjectList_" + c + "_objectMissCacheFlushes", map.get("objectMissCacheFlushes")); //prop.put("ObjectList_" + c + "_objectMissCacheFlushes", map.get("objectMissCacheFlushes"));
c++; c++;
} }
prop.put("ObjectList", c); prop.put("ObjectList", c);
@ -192,11 +191,17 @@ public class PerformanceMemory_p {
prop.putNum("objectCacheStartShrink", Cache.getMemStartShrink() / (1024 * 1024d)); prop.putNum("objectCacheStartShrink", Cache.getMemStartShrink() / (1024 * 1024d));
prop.putNum("objectHitCacheTotalMem", totalhitmem / (1024 * 1024d)); prop.putNum("objectHitCacheTotalMem", totalhitmem / (1024 * 1024d));
prop.putNum("objectMissCacheTotalMem", totalmissmem / (1024 * 1024d)); prop.putNum("objectMissCacheTotalMem", totalmissmem / (1024 * 1024d));
// other caching structures // other caching structures
prop.putNum("namecacheHit.size", Domains.nameCacheHitSize()); prop.putNum("namecacheHit.size", Domains.nameCacheHitSize());
prop.putNum("namecacheHit.Hit", Domains.cacheHit_Hit);
prop.putNum("namecacheHit.Miss", Domains.cacheHit_Miss);
prop.putNum("namecacheHit.Insert", Domains.cacheHit_Insert);
prop.putNum("namecacheMiss.size", Domains.nameCacheMissSize()); prop.putNum("namecacheMiss.size", Domains.nameCacheMissSize());
prop.putNum("namecache.noCache", 0); prop.putNum("namecacheMiss.Hit", Domains.cacheMiss_Hit);
prop.putNum("namecacheMiss.Miss", Domains.cacheMiss_Miss);
prop.putNum("namecacheMiss.Insert", Domains.cacheMiss_Insert);
prop.putNum("namecache.noCache", Domains.nameCacheNoCachingPatternsSize());
prop.putNum("blacklistcache.size", Switchboard.urlBlacklist.blacklistCacheSize()); prop.putNum("blacklistcache.size", Switchboard.urlBlacklist.blacklistCacheSize());
prop.putNum("searchevent.size", SearchEventCache.size()); prop.putNum("searchevent.size", SearchEventCache.size());
prop.putNum("searchevent.hit", SearchEventCache.cacheHit); prop.putNum("searchevent.hit", SearchEventCache.cacheHit);

29
source/net/yacy/cora/protocol/Domains.java
Unescape View File

@ -61,6 +61,8 @@ public class Domains {
private static final ConcurrentHashMap<String, Object> LOOKUP_SYNC = new ConcurrentHashMap<String, Object>(); private static final ConcurrentHashMap<String, Object> LOOKUP_SYNC = new ConcurrentHashMap<String, Object>();
private static List<Pattern> nameCacheNoCachingPatterns = Collections.synchronizedList(new LinkedList<Pattern>()); private static List<Pattern> nameCacheNoCachingPatterns = Collections.synchronizedList(new LinkedList<Pattern>());
private static final List<Pattern> LOCALHOST_PATTERNS = makePatterns(LOCAL_PATTERNS); private static final List<Pattern> LOCALHOST_PATTERNS = makePatterns(LOCAL_PATTERNS);
public static long cacheHit_Hit = 0, cacheHit_Miss = 0, cacheHit_Insert = 0; // for statistics only; do not write
public static long cacheMiss_Hit = 0, cacheMiss_Miss = 0, cacheMiss_Insert = 0; // for statistics only; do not write
/** /**
* ! ! ! A T T E N T I O N A T T E N T I O N A T T E N T I O N ! ! ! * ! ! ! A T T E N T I O N A T T E N T I O N A T T E N T I O N ! ! !
@ -470,9 +472,17 @@ public class Domains {
// trying to resolve host by doing a name cache lookup // trying to resolve host by doing a name cache lookup
ip = NAME_CACHE_HIT.get(host); ip = NAME_CACHE_HIT.get(host);
if (ip != null) return ip; if (ip != null) {
cacheHit_Hit++;
return ip;
}
cacheHit_Miss++;
if (NAME_CACHE_MISS.containsKey(host)) return null; if (NAME_CACHE_MISS.containsKey(host)) {
cacheMiss_Hit++;
return null;
}
cacheMiss_Miss++;
throw new UnknownHostException("host not in cache"); throw new UnknownHostException("host not in cache");
} }
@ -501,6 +511,7 @@ public class Domains {
if (!hosts.isEmpty()) return hosts.iterator().next(); if (!hosts.isEmpty()) return hosts.iterator().next();
final String host = i.getHostName(); final String host = i.getHostName();
NAME_CACHE_HIT.insertIfAbsent(host, i); NAME_CACHE_HIT.insertIfAbsent(host, i);
cacheHit_Insert++;
return host; return host;
/* /*
// call i.getHostName() using concurrency to interrupt execution in case of a time-out // call i.getHostName() using concurrency to interrupt execution in case of a time-out
@ -530,12 +541,16 @@ public class Domains {
ip = NAME_CACHE_HIT.get(host); ip = NAME_CACHE_HIT.get(host);
if (ip != null) { if (ip != null) {
//System.out.println("DNSLOOKUP-CACHE-HIT(CONC) " + host); //System.out.println("DNSLOOKUP-CACHE-HIT(CONC) " + host);
cacheHit_Hit++;
return ip; return ip;
} }
cacheHit_Miss++;
if (NAME_CACHE_MISS.containsKey(host)) { if (NAME_CACHE_MISS.containsKey(host)) {
//System.out.println("DNSLOOKUP-CACHE-MISS(CONC) " + host); //System.out.println("DNSLOOKUP-CACHE-MISS(CONC) " + host);
cacheMiss_Hit++;
return null; return null;
} }
cacheMiss_Miss++;
// call dnsResolveNetBased(host) using concurrency to interrupt execution in case of a time-out // call dnsResolveNetBased(host) using concurrency to interrupt execution in case of a time-out
final Object sync_obj_new = new Object(); final Object sync_obj_new = new Object();
@ -547,13 +562,17 @@ public class Domains {
if (ip != null) { if (ip != null) {
//System.out.println("DNSLOOKUP-CACHE-HIT(SYNC) " + host); //System.out.println("DNSLOOKUP-CACHE-HIT(SYNC) " + host);
LOOKUP_SYNC.remove(host); LOOKUP_SYNC.remove(host);
cacheHit_Hit++;
return ip; return ip;
} }
cacheHit_Miss++;
if (NAME_CACHE_MISS.containsKey(host)) { if (NAME_CACHE_MISS.containsKey(host)) {
//System.out.println("DNSLOOKUP-CACHE-MISS(SYNC) " + host); //System.out.println("DNSLOOKUP-CACHE-MISS(SYNC) " + host);
LOOKUP_SYNC.remove(host); LOOKUP_SYNC.remove(host);
cacheMiss_Hit++;
return null; return null;
} }
cacheMiss_Miss++;
// do the dns lookup on the dns server // do the dns lookup on the dns server
//if (!matchesList(host, nameCacheNoCachingPatterns)) System.out.println("DNSLOOKUP " + host); //if (!matchesList(host, nameCacheNoCachingPatterns)) System.out.println("DNSLOOKUP " + host);
@ -563,6 +582,7 @@ public class Domains {
} catch (final UnknownHostException e) { } catch (final UnknownHostException e) {
// add new entries // add new entries
NAME_CACHE_MISS.insertIfAbsent(host, PRESENT); NAME_CACHE_MISS.insertIfAbsent(host, PRESENT);
cacheMiss_Insert++;
LOOKUP_SYNC.remove(host); LOOKUP_SYNC.remove(host);
return null; return null;
} }
@ -570,6 +590,7 @@ public class Domains {
if (ip != null && !ip.isLoopbackAddress() && !matchesList(host, nameCacheNoCachingPatterns)) { if (ip != null && !ip.isLoopbackAddress() && !matchesList(host, nameCacheNoCachingPatterns)) {
// add new ip cache entries // add new ip cache entries
NAME_CACHE_HIT.insertIfAbsent(host, ip); NAME_CACHE_HIT.insertIfAbsent(host, ip);
cacheHit_Insert++;
// add also the isLocal host name caches // add also the isLocal host name caches
final boolean localp = ip.isAnyLocalAddress() || ip.isLinkLocalAddress() || ip.isSiteLocalAddress(); final boolean localp = ip.isAnyLocalAddress() || ip.isLinkLocalAddress() || ip.isSiteLocalAddress();
@ -620,6 +641,10 @@ public class Domains {
return NAME_CACHE_MISS.size(); return NAME_CACHE_MISS.size();
} }
public static int nameCacheNoCachingPatternsSize() {
return nameCacheNoCachingPatterns.size();
}
private static String localHostName = "127.0.0.1"; private static String localHostName = "127.0.0.1";
private static Set<InetAddress> localHostAddresses = new HashSet<InetAddress>(); private static Set<InetAddress> localHostAddresses = new HashSet<InetAddress>();
private static Set<String> localHostNames = new HashSet<String>(); private static Set<String> localHostNames = new HashSet<String>();

Write Preview
Loading…
Cancel
Save