yacy_search_server/htroot/PerformanceMemory_p.java

//PerformaceMemory_p.java
//-----------------------
//part of YaCy
//(C) by Michael Peter Christen; mc@yacy.net
//first published on http://www.anomic.de
//Frankfurt, Germany, 2005
//last major change: 19.09.2005
//
//This program is free software; you can redistribute it and/or modify
//it under the terms of the GNU General Public License as published by
//the Free Software Foundation; either version 2 of the License, or
//(at your option) any later version.
//
//This program is distributed in the hope that it will be useful,
//but WITHOUT ANY WARRANTY; without even the implied warranty of
//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
//GNU General Public License for more details.
//
//You should have received a copy of the GNU General Public License
//along with this program; if not, write to the Free Software
//Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

//You must compile this file with
//javac -classpath .:../classes PerformanceMemory_p.java
//if the shell's current path is HTROOT

//import java.util.Iterator;
import java.util.ConcurrentModificationException;
import java.util.Iterator;
import java.util.Map;
import java.util.TreeMap;

import org.apache.solr.core.SolrInfoMBean;
import org.apache.solr.search.SolrCache;

import net.yacy.cora.protocol.Domains;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.cora.util.ConcurrentLog;
import net.yacy.kelondro.index.RAMIndex;
import net.yacy.kelondro.table.Table;
import net.yacy.kelondro.table.Table.TableStatistics;
import net.yacy.kelondro.util.Formatter;
import net.yacy.kelondro.util.MemoryControl;
import net.yacy.search.Switchboard;
import net.yacy.search.query.SearchEventCache;
import net.yacy.server.serverObjects;
import net.yacy.server.serverSwitch;

public class PerformanceMemory_p {

    private static final long KB = 1024;
    private static final long MB = 1024 * KB;
    
    public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, final serverObjects post, final serverSwitch env) {
        // return variable that accumulates replacements
        Switchboard sb = (Switchboard) env;
        
        final serverObjects prop = new serverObjects();

        prop.put("gc", "0");
        prop.put("autoreload.checked", "0");
        if (post != null) {
            if (post.containsKey("gc")) {
                System.gc();
                prop.put("gc", "1");
                prop.put("autoreload.checked", "1");
            } else if (post.containsKey("dummy")) {
                boolean simulatedshortmemory = post.containsKey("simulatedshortmemory");
                MemoryControl.setSimulatedShortStatus(simulatedshortmemory);
                if (simulatedshortmemory) prop.put("autoreload.checked", "1");
                final boolean std = post.containsKey("useStandardmemoryStrategy");
                env.setConfig("memory.standardStrategy", std);
                MemoryControl.setStandardStrategy(std);
            }
        }
        
        prop.put("simulatedshortmemory.checked", MemoryControl.getSimulatedShortStatus() ? 1 : 0);
        prop.put("useStandardmemoryStrategy.checked", env.getConfigBool("memory.standardStrategy", true) ? 1 : 0);
        prop.put("memoryStrategy", MemoryControl.getStrategyName());

        final long memoryFreeAfterInitBGC = env.getConfigLong("memoryFreeAfterInitBGC", 0L);
        final long memoryFreeAfterInitAGC = env.getConfigLong("memoryFreeAfterInitAGC", 0L);
        final long memoryFreeAfterStartup = env.getConfigLong("memoryFreeAfterStartup", 0L);
        final long memoryTotalAfterInitBGC = env.getConfigLong("memoryTotalAfterInitBGC", 0L);
        final long memoryTotalAfterInitAGC = env.getConfigLong("memoryTotalAfterInitAGC", 0L);
        final long memoryTotalAfterStartup = env.getConfigLong("memoryTotalAfterStartup", 0L);

        prop.putNum("memoryMax", MemoryControl.maxMemory() / MB);
        prop.putNum("memoryAvailAfterStartup", (MemoryControl.maxMemory() - memoryTotalAfterStartup + memoryFreeAfterStartup) / MB);
        prop.putNum("memoryAvailAfterInitBGC", (MemoryControl.maxMemory() - memoryTotalAfterInitBGC + memoryFreeAfterInitBGC) / MB);
        prop.putNum("memoryAvailAfterInitAGC", (MemoryControl.maxMemory() - memoryTotalAfterInitAGC + memoryFreeAfterInitAGC) / MB);
        prop.putNum("memoryAvailNow", MemoryControl.available() / MB);
        prop.putNum("memoryTotalAfterStartup", memoryTotalAfterStartup / KB);
        prop.putNum("memoryTotalAfterInitBGC", memoryTotalAfterInitBGC / KB);
        prop.putNum("memoryTotalAfterInitAGC", memoryTotalAfterInitAGC / KB);
        prop.putNum("memoryTotalNow", MemoryControl.total() / MB);
        prop.putNum("memoryFreeAfterStartup", memoryFreeAfterStartup / KB);
        prop.putNum("memoryFreeAfterInitBGC", memoryFreeAfterInitBGC / KB);
        prop.putNum("memoryFreeAfterInitAGC", memoryFreeAfterInitAGC / KB);
        prop.putNum("memoryFreeNow", MemoryControl.free() / MB);
        prop.putNum("memoryUsedAfterStartup", (memoryTotalAfterStartup - memoryFreeAfterStartup) / KB);
        prop.putNum("memoryUsedAfterInitBGC", (memoryTotalAfterInitBGC - memoryFreeAfterInitBGC) / KB);
        prop.putNum("memoryUsedAfterInitAGC", (memoryTotalAfterInitAGC - memoryFreeAfterInitAGC) / KB);
        prop.putNum("memoryUsedNow", MemoryControl.used() / MB);

        
        final Map<String, SolrInfoMBean> solrInfoMBeans = sb.index.fulltext().getSolrInfoBeans();
        final TreeMap<String, Map.Entry<String, SolrInfoMBean>> solrBeanOM = new TreeMap<String, Map.Entry<String, SolrInfoMBean>>();
        int c = 0;
        for (Map.Entry<String, SolrInfoMBean> sc: solrInfoMBeans.entrySet()) solrBeanOM.put(sc.getValue().getName() + "$" + sc.getKey() + "$" + c++, sc);
        c = 0;
        int scc = 0;
        for (Map.Entry<String, SolrInfoMBean> sc: solrBeanOM.values()) {
            prop.put("SolrList_" + c + "_class", sc.getValue().getName());
            prop.put("SolrList_" + c + "_type", sc.getKey());
            prop.put("SolrList_" + c + "_description", sc.getValue().getDescription());
            prop.put("SolrList_" + c + "_statistics", sc.getValue().getStatistics() == null ? "" : sc.getValue().getStatistics().toString().replaceAll(",", ", "));
            prop.put("SolrList_" + c + "_size", sc.getValue() instanceof SolrCache ? Integer.toString(((SolrCache<?,?>)sc.getValue()).size()) : "");
            if (sc.getValue() instanceof SolrCache) scc++;
            c++;
        }
        prop.put("SolrList", c);
        prop.put("SolrCacheCount", scc);
        
        // write table for Table index sizes
        Iterator<String> i = Table.filenames();
        String filename;
        TableStatistics stats;
        int p;
        c = 0;
        long totalmem = 0;
        while (i.hasNext()) {
            filename = i.next();
            stats = Table.memoryStats(filename);
        	totalmem += stats.getTotalMem();
        	
            prop.put("EcoList_" + c + "_tableIndexPath", ((p = filename.indexOf("DATA",0)) < 0) ? filename : filename.substring(p));
            prop.putNum("EcoList_" + c + "_tableSize", stats.getTableSize());

            prop.put("EcoList_" + c + "_tableKeyMem", Formatter.bytesToString(stats.getKeyMem()));
            prop.put("EcoList_" + c + "_tableKeyChunkSize", Formatter.bytesToString(stats.getKeyChunkSize()));

            prop.put("EcoList_" + c + "_tableValueMem", Formatter.bytesToString(stats.getValueMem()));
            prop.put("EcoList_" + c + "_tableValueChunkSize",  Formatter.bytesToString(stats.getValueChunkSize()));

            c++;
        }
        prop.put("EcoList", c);
        prop.putNum("EcoIndexTotalMem", totalmem / (1024d * 1024d));

        // write object cache table
        final Iterator<Map.Entry<String, RAMIndex>> oi = RAMIndex.objects();
        c = 0;
        Map.Entry<String, RAMIndex> oie;
        RAMIndex cache;
        long hitmem, totalhitmem = 0;
        while (oi.hasNext()) {
            try {
                oie = oi.next();
            } catch (final ConcurrentModificationException e) {
                // we don't want to synchronize this
                ConcurrentLog.logException(e);
                break;
            }
            filename = oie.getKey();
            cache = oie.getValue();
            prop.put("indexcache_" + c + "_Name", ((p = filename.indexOf("DATA",0)) < 0) ? filename : filename.substring(p));

            hitmem = cache.mem();
            totalhitmem += hitmem;
            prop.put("indexcache_" + c + "_ChunkSize", cache.row().objectsize);
            prop.putNum("indexcache_" + c + "_Count", cache.size());
            prop.put("indexcache_" + c + "_NeededMem", cache.size() * cache.row().objectsize);
            prop.put("indexcache_" + c + "_UsedMem", hitmem);

            c++;
        }
        prop.put("indexcache", c);
        prop.putNum("indexcacheTotalMem", totalhitmem / (1024d * 1024d));

//        this cache table wasn't used for years
//        // write object cache table
//        i = Cache.filenames();
//        c = 0;
//        long missmem, totalmissmem = 0;
//        totalhitmem = 0;
//        Map<Cache.StatKeys, String> mapy;
//        while (i.hasNext()) {
//            filename = i.next();
//            mapy = Cache.memoryStats(filename);
//            prop.put("ObjectList_" + c + "_objectCachePath", ((p = filename.indexOf("DATA",0)) < 0) ? filename : filename.substring(p));
//
//            // hit cache
//            hitmem = Long.parseLong(mapy.get(Cache.StatKeys.objectHitMem));
//            totalhitmem += hitmem;
//            prop.put("ObjectList_" + c + "_objectHitChunkSize", mapy.get(Cache.StatKeys.objectHitChunkSize));
//            prop.putNum("ObjectList_" + c + "_objectHitCacheCount", mapy.get(Cache.StatKeys.objectHitCacheCount));
//            prop.put("ObjectList_" + c + "_objectHitCacheMem", Formatter.bytesToString(hitmem));
//            prop.putNum("ObjectList_" + c + "_objectHitCacheReadHit", mapy.get(Cache.StatKeys.objectHitCacheReadHit));
//            prop.putNum("ObjectList_" + c + "_objectHitCacheReadMiss", mapy.get(Cache.StatKeys.objectHitCacheReadMiss));
//            prop.putNum("ObjectList_" + c + "_objectHitCacheWriteUnique", mapy.get(Cache.StatKeys.objectHitCacheWriteUnique));
//            prop.putNum("ObjectList_" + c + "_objectHitCacheWriteDouble", mapy.get(Cache.StatKeys.objectHitCacheWriteDouble));
//            prop.putNum("ObjectList_" + c + "_objectHitCacheDeletes", mapy.get(Cache.StatKeys.objectHitCacheDeletes));
//            prop.putNum("ObjectList_" + c + "_objectHitCacheFlushes", mapy.get(Cache.StatKeys.objectHitCacheFlushes));
//
//            // miss cache
//            missmem = Long.parseLong(mapy.get(Cache.StatKeys.objectMissMem));
//            totalmissmem += missmem;
//            prop.put("ObjectList_" + c + "_objectMissChunkSize", mapy.get(Cache.StatKeys.objectMissChunkSize));
//            prop.putNum("ObjectList_" + c + "_objectMissCacheCount", mapy.get(Cache.StatKeys.objectMissCacheCount));
//            prop.putHTML("ObjectList_" + c + "_objectMissCacheMem", Formatter.bytesToString(missmem));
//            prop.putNum("ObjectList_" + c + "_objectMissCacheReadHit", mapy.get(Cache.StatKeys.objectMissCacheReadHit));
//            prop.putNum("ObjectList_" + c + "_objectMissCacheReadMiss", mapy.get(Cache.StatKeys.objectMissCacheReadMiss));
//            prop.putNum("ObjectList_" + c + "_objectMissCacheWriteUnique", mapy.get(Cache.StatKeys.objectMissCacheWriteUnique));
//            prop.putNum("ObjectList_" + c + "_objectMissCacheWriteDouble", mapy.get(Cache.StatKeys.objectMissCacheWriteDouble));
//            prop.putNum("ObjectList_" + c + "_objectMissCacheDeletes", mapy.get(Cache.StatKeys.objectMissCacheDeletes));
//            //prop.put("ObjectList_" + c + "_objectMissCacheFlushes", mapy.get(Cache.StatKeys.objectMissCacheFlushes));
//
//            c++;
//        }
//        prop.put("ObjectList", c);
//        prop.putNum("objectCacheStopGrow", Cache.getMemStopGrow() / (1024d * 1024d));
//        prop.putNum("objectCacheStartShrink", Cache.getMemStartShrink() / (1024d * 1024d));
//        prop.putNum("objectHitCacheTotalMem", totalhitmem / (1024d * 1024d));
//        prop.putNum("objectMissCacheTotalMem", totalmissmem / (1024d * 1024d));
        
        prop.putNum("namecacheHit.size", Domains.nameCacheHitSize());
        prop.putNum("namecacheHit.Hit", Domains.cacheHit_Hit);
        prop.putNum("namecacheHit.Miss", Domains.cacheHit_Miss);
        prop.putNum("namecacheHit.Insert", Domains.cacheHit_Insert);
        prop.putNum("namecacheMiss.size", Domains.nameCacheMissSize());
        prop.putNum("namecacheMiss.Hit", Domains.cacheMiss_Hit);
        prop.putNum("namecacheMiss.Miss", Domains.cacheMiss_Miss);
        prop.putNum("namecacheMiss.Insert", Domains.cacheMiss_Insert);
        prop.putNum("namecache.noCache", Domains.nameCacheNoCachingPatternsSize());
        prop.putNum("blacklistcache.size", Switchboard.urlBlacklist.blacklistCacheSize());
        prop.putNum("searchevent.size", SearchEventCache.size());
        prop.putNum("searchevent.hit", SearchEventCache.cacheHit);
        prop.putNum("searchevent.miss", SearchEventCache.cacheMiss);
        prop.putNum("searchevent.insert", SearchEventCache.cacheInsert);
        prop.putNum("searchevent.delete", SearchEventCache.cacheDelete);
        // return rewrite values for templates
        return prop;
    }
}