yacy_search_server/source/dbtest.java

// test application
// shall be used to compare the kelondroDB with other databases
// with relevance to yacy-specific use cases

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Random;

import javax.imageio.ImageIO;

import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroCache;
import de.anomic.kelondro.kelondroFlexSplitTable;
import de.anomic.kelondro.kelondroFlexTable;
import de.anomic.kelondro.kelondroIndex;
import de.anomic.kelondro.kelondroIntBytesMap;
import de.anomic.kelondro.kelondroProfile;
import de.anomic.kelondro.kelondroRow;
import de.anomic.kelondro.kelondroRowSet;
import de.anomic.kelondro.kelondroSQLTable;
import de.anomic.kelondro.kelondroSplittedTree;
import de.anomic.kelondro.kelondroTree;
import de.anomic.server.serverInstantThread;
import de.anomic.server.serverMemory;
import de.anomic.server.logging.serverLog;
import de.anomic.ymage.ymageChart;

public class dbtest {

    public final static int keylength = 12;
    public final static int valuelength = 223; // sum of all data length as defined in plasmaURL
    public final static long preload = 1000; // 1 second
    public static byte[] dummyvalue2 = new byte[valuelength];
    static {
        // fill the dummy value
        for (int i = 0; i < valuelength; i++) dummyvalue2[i] = '-';
        dummyvalue2[0] = '{';
        dummyvalue2[valuelength - 1] = '}';
    }
    
    public static byte[] randomHash(final long r0, final long r1) {
        // a long can have 64 bit, but a 12-byte hash can have 6 * 12 = 72 bits
        // so we construct a generic Hash using two long values
        String s = (kelondroBase64Order.enhancedCoder.encodeLong(Math.abs(r0), 6) +
                    kelondroBase64Order.enhancedCoder.encodeLong(Math.abs(r1), 6));
        return s.getBytes();
    }
    
    public static byte[] randomHash(Random r) {
        return randomHash(r.nextLong(), r.nextLong());
    }
    
    public final static class STEntry {
        private final byte[] key;
        private final byte[] value;

        public STEntry(final long aSource) {
            this.key = randomHash(aSource, aSource);
            this.value = new byte[valuelength];
            for (int i = 0; i < valuelength; i++) this.value[i] = 0;
            final byte[] tempKey = String.valueOf(aSource).getBytes();
            System.arraycopy(tempKey, 0, this.value, 0, tempKey.length);
        }

        public STEntry(final byte[] aKey, final byte[] aValue) {
            this.key = aKey;
            this.value = aValue;
        }

        public boolean isValid() {
            String s = new String(this.value).trim();
            if (s.length() == 0) return false;
            final long source = new Long(s).longValue();
            return new String(this.key).equals(new String(randomHash(source, source)));
        }

        public byte[] getKey() {
            return this.key;
        }

        public byte[] getValue() {
            return this.value;
        }

        public String toString() {
            return new String(this.key) + "#" + new String(this.value);
        }

    }

    public static abstract class STJob implements Runnable {
        private final kelondroIndex table;

        private final long source;

        public STJob(final kelondroIndex aTable, final long aSource) {
            this.table = aTable;
            this.source = aSource;
        }

        public kelondroIndex getTable() {
            return this.table;
        }

        public abstract void run();

        public long getSource() {
            return this.source;
        }
    }

    public static final class WriteJob extends STJob {
        public WriteJob(final kelondroIndex aTable, final long aSource) {
            super(aTable, aSource);
        }

        public void run() {
            final STEntry entry = new STEntry(this.getSource());
            System.out.println("write:  " + serverLog.arrayList(entry.getKey(), 0, entry.getKey().length));
            try {
                getTable().put(getTable().row().newEntry(new byte[][] { entry.getKey(), entry.getValue() , entry.getValue() }));
            } catch (IOException e) {
                System.err.println(e);
                e.printStackTrace();
                System.exit(0);
            }
        }
    }

    public static final class RemoveJob extends STJob {
        public RemoveJob(final kelondroIndex aTable, final long aSource) {
            super(aTable, aSource);
        }

        public void run() {
            final STEntry entry = new STEntry(this.getSource());
            System.out.println("remove: " + serverLog.arrayList(entry.getKey(), 0, entry.getKey().length));
            try {
                getTable().remove(entry.getKey(), false);
            } catch (IOException e) {
                System.err.println(e);
                e.printStackTrace();
                System.exit(0);
            }
        }
    }

    public static final class ReadJob extends STJob {
        public ReadJob(final kelondroIndex aTable, final long aSource) {
            super(aTable, aSource);
        }

        public void run() {
            final STEntry entry = new STEntry(this.getSource());
            try {
                final kelondroRow.Entry entryBytes = getTable().get(entry.getKey());
                if (entryBytes != null) {
                    //System.out.println("ENTRY=" + entryBytes.getColString(1, null));
                    final STEntry dbEntry = new STEntry(entryBytes.getColBytes(0), entryBytes.getColBytes(1));
                    if (!dbEntry.isValid()) {
                        System.out.println("INVALID: " + dbEntry);
                    }/* else {
                        System.out.println("_VALID_: " + dbEntry);
                        getTable().remove(entry.getKey());
                    }*/
                }
            } catch (IOException e) {
                System.err.println(e);
                e.printStackTrace();
                System.exit(0);
            }
        }
    }
    
    public static void main(String[] args) {
        System.setProperty("java.awt.headless", "true");
        
        String dbe = args[0];       // the database engine
        String command = args[1];   // test command
        String tablename = args[2]; // name of test-table
        long startup = System.currentTimeMillis();
        
        try {
            kelondroIndex table = null;
            // create a memory profiler
            memprofiler profiler = new memprofiler(1024, 320, 120, new File(tablename + ".profile.png"));
            profiler.start();
            
            // create the database access
            kelondroRow testRow = new kelondroRow("byte[] key-" + keylength + ", byte[] dummy-" + keylength + ", value-" + valuelength, kelondroBase64Order.enhancedCoder, 0);
            if (dbe.equals("kelondroRowSet")) {
                table = new kelondroRowSet(testRow, 0);
            }
            if (dbe.equals("kelondroTree")) {
                File tablefile = new File(tablename + ".kelondro.db");
                table = new kelondroCache(new kelondroTree(tablefile, true, preload, testRow), true, false);
            }
            if (dbe.equals("kelondroSplittedTree")) {
                File tablepath = new File(tablename).getParentFile();
                tablename = new File(tablename).getName();
                table = new kelondroSplittedTree(tablepath, tablename, kelondroBase64Order.enhancedCoder,
                                preload, 8, testRow, 1, 80);
            }
            if (dbe.equals("kelondroFlexTable")) {
                File tablepath = new File(tablename).getParentFile();
                table = new kelondroFlexTable(tablepath, new File(tablename).getName(), preload, testRow, true);
            }
            if (dbe.equals("kelondroFlexSplitTable")) {
                File tablepath = new File(tablename).getParentFile();
                table = new kelondroFlexSplitTable(tablepath, new File(tablename).getName(), preload, testRow, true);
            }
            if (dbe.equals("mysql")) {
                table = new kelondroSQLTable("mysql", testRow);
            }
            
            if (dbe.equals("pgsql")) {
                table = new kelondroSQLTable("pgsql", testRow);
            }
            
            long afterinit = System.currentTimeMillis();
            System.out.println("Test for db-engine " + dbe +  " started to create file " + tablename + " with test " + command);
            
            // execute command
            if (command.equals("create")) {
                // do nothing, since opening of the database access must handle this
                System.out.println("Database created");
            }
            
            if (command.equals("fill")) {
                // fill database with random entries;
                // args: <number-of-entries> <random-startpoint>
                long count = Long.parseLong(args[3]);
                long randomstart = Long.parseLong(args[4]);
                Random random = new Random(randomstart);
                byte[] key;
                kelondroProfile ioProfileAcc = new kelondroProfile();
                kelondroProfile cacheProfileAcc = new kelondroProfile();
                kelondroProfile[] profiles;
                for (int i = 0; i < count; i++) {
                    key = randomHash(random);
                    table.put(table.row().newEntry(new byte[][]{key, key, dummyvalue2}));
                    if (i % 1000 == 0) {
                        System.out.println(i + " entries. ");
                        if (table instanceof kelondroTree) {
                            profiles = ((kelondroTree) table).profiles();
                            System.out.println("Cache Delta: " + kelondroProfile.delta(profiles[0], cacheProfileAcc).toString());
                            System.out.println("IO    Delta: " + kelondroProfile.delta(profiles[1],    ioProfileAcc).toString());
                            cacheProfileAcc = (kelondroProfile) profiles[0].clone();
                               ioProfileAcc = (kelondroProfile) profiles[1].clone();
                        }
                    }
                }
            }
            
            if (command.equals("benchfill")) {
                // fill database with random entries;
                // args: <number-of-entries> <random-startpoint>
                long count = Long.parseLong(args[3]);
                long time = System.currentTimeMillis();
                long randomstart = Long.parseLong(args[4]);
                Random random = new Random(randomstart);
                byte[] key;
                for (int i = 0; i < count; i++) {
                    key = randomHash(random);
                    table.put(table.row().newEntry(new byte[][]{key, key, dummyvalue2}));
                    if (i % 10000 == 0) {
                        System.out.println(System.currentTimeMillis() - time);
                    }
                }
            }
            
            if (command.equals("read")) {
                // read the database and compare with random entries;
                // args: <number-of-entries> <random-startpoint>
                long count = Long.parseLong(args[3]);
                long randomstart = Long.parseLong(args[4]);
                Random random = new Random(randomstart);
                kelondroRow.Entry entry;
                byte[] key;
                for (int i = 0; i < count; i++) {
                    key = randomHash(random);
                    entry = table.get(key);
                    if (entry == null) System.out.println("missing value for entry " + new String(key)); else
                    if (!(new String(entry.getColBytes(1)).equals(new String(key)))) System.out.println("wrong value for entry " + new String(key) + ": " + new String(entry.getColBytes(1)));
                    if (i % 1000 == 0) {
                        System.out.println(i + " entries processed so far.");
                    }
                }
            }
            
            if (command.equals("benchread")) {
                // read the database and compare with random entries;
                // args: <number-of-entries> <random-startpoint>
                long count = Long.parseLong(args[3]);
                long time = System.currentTimeMillis();
                long randomstart = Long.parseLong(args[4]);
                Random random = new Random(randomstart);
                kelondroRow.Entry entry;
                byte[] key;
                for (int i = 0; i < count; i++) {
                    key = randomHash(random);
                    entry = table.get(key);
                    if (entry == null) System.out.println("missing value for entry " + new String(key)); else
                    if (!(new String(entry.getColBytes(1)).equals(new String(key)))) System.out.println("wrong value for entry " + new String(key) + ": " + new String(entry.getColBytes(1)));
                    if (i % 10000 == 0) {
                        System.out.println(System.currentTimeMillis() - time);
                    }
                }
            }
            
            if (command.equals("ramtest")) {
                // fill database with random entries and delete them again;
                // this is repeated without termination; after each loop
                // the current ram is printed out
                // args: <number-of-entries> <random-startpoint>
                long count = Long.parseLong(args[3]);
                long randomstart = Long.parseLong(args[4]);
                byte[] key;
                Random random;
                long start, write, remove;
                int loop = 0;
                while (true) {
                    // write
                    random = new Random(randomstart);
                    start = System.currentTimeMillis();
                    for (int i = 0; i < count; i++) {
                        key = randomHash(random);
                        table.put(table.row().newEntry(new byte[][]{key, key, dummyvalue2}));
                    }
                    write = System.currentTimeMillis() - start;

                    // delete
                    random = new Random(randomstart);
                    start = System.currentTimeMillis();
                    for (int i = 0; i < count; i++) {
                        key = randomHash(random);
                        table.remove(key, false);
                    }
                    remove = System.currentTimeMillis() - start;
                    
                    System.out.println("Loop " + loop + ": Write = " + write + ", Remove = " + remove);
                    System.out.println(" bevore GC: " +
                              "free = " + Runtime.getRuntime().freeMemory() +
                            ", max = " + Runtime.getRuntime().maxMemory() +
                            ", total = " + Runtime.getRuntime().totalMemory());
                    System.gc();
                    System.out.println(" after  GC: " +
                            "free = " + Runtime.getRuntime().freeMemory() +
                          ", max = " + Runtime.getRuntime().maxMemory() +
                          ", total = " + Runtime.getRuntime().totalMemory());
                  loop++;
                }
            }
            
            if (command.equals("list")) {
                Iterator i = null;
                if (table instanceof kelondroSplittedTree) i = ((kelondroSplittedTree) table).rows(true, null);
                if (table instanceof kelondroTree) i = ((kelondroTree) table).rows(true, null);
                if (table instanceof kelondroSQLTable) i = ((kelondroSQLTable) table).rows(true, null);
                byte[][] row;
                while (i.hasNext()) {
                    row = (byte[][]) i.next();
                    for (int j = 0; j < row.length; j++) System.out.print(new String(row[j]) + ",");
                    System.out.println();
                }
            }
            
            if (command.equals("stressThreaded")) {
                // 
                // args: <number-of-writes> <number-of-reads-per-write> <random-startpoint>
                long writeCount = Long.parseLong(args[3]);
                long readCount = Long.parseLong(args[4]);
                long randomstart = Long.parseLong(args[5]);
                final Random random = new Random(randomstart);
                long r;
                Long R;
                int p, rc=0;
                ArrayList ra = new ArrayList();
                HashSet jcontrol = new HashSet();
                kelondroIntBytesMap kcontrol = new kelondroIntBytesMap(1, 0);
                for (int i = 0; i < writeCount; i++) {
                	r = Math.abs(random.nextLong() % 1000);
                	jcontrol.add(new Long(r));
                	kcontrol.putb((int) r, "x".getBytes());
                    serverInstantThread.oneTimeJob(new WriteJob(table, r), 0, 50);
                    if (random.nextLong() % 5 == 0) ra.add(new Long(r));
                    for (int j = 0; j < readCount; j++) {
                        serverInstantThread.oneTimeJob(new ReadJob(table, random.nextLong() % writeCount), random.nextLong() % 1000, 20);
                    }
                    if ((ra.size() > 0) && (random.nextLong() % 7 == 0)) {
                    	rc++;
                    	p = Math.abs(random.nextInt()) % ra.size();
                    	R = (Long) ra.get(p);
                    	jcontrol.remove(R);
                    	kcontrol.removeb((int) R.longValue());
                    	System.out.println("remove: " + R.longValue());
                    	serverInstantThread.oneTimeJob(new RemoveJob(table, ((Long) ra.remove(p)).longValue()), 0, 50);
                    }
                }
                System.out.println("removed: " + rc + ", size of jcontrol set: " + jcontrol.size() + ", size of kcontrol set: " + kcontrol.size());
                while (serverInstantThread.instantThreadCounter > 0) {
                    try {Thread.sleep(1000);} catch (InterruptedException e) {} // wait for all tasks to finish
                    System.out.println("count: "  + serverInstantThread.instantThreadCounter + ", jobs: " + serverInstantThread.jobs.toString());
                }
                try {Thread.sleep(6000);} catch (InterruptedException e) {}
            }
            
            if (command.equals("stressSequential")) {
                // 
                // args: <number-of-writes> <number-of-reads> <random-startpoint>
                long writeCount = Long.parseLong(args[3]);
                long readCount = Long.parseLong(args[4]);
                long randomstart = Long.parseLong(args[5]);
                Random random = new Random(randomstart);
                long r;
                Long R;
                int p, rc=0;
                ArrayList ra = new ArrayList();
                HashSet jcontrol = new HashSet();
                kelondroIntBytesMap kcontrol = new kelondroIntBytesMap(1, 0);
                for (int i = 0; i < writeCount; i++) {
                	//if (i == 30) random = new Random(randomstart);
                	r = Math.abs(random.nextLong() % 1000);
                	jcontrol.add(new Long(r));
                	kcontrol.putb((int) r, "x".getBytes());
                    new WriteJob(table, r).run();
                    if (random.nextLong() % 5 == 0) ra.add(new Long(r));
                    for (int j = 0; j < readCount; j++) {
                        new ReadJob(table, random.nextLong() % writeCount).run();
                    }
                    if ((ra.size() > 0) && (random.nextLong() % 7 == 0)) {
                    	rc++;
                    	p = Math.abs(random.nextInt()) % ra.size();
                    	R = (Long) ra.get(p);
                    	jcontrol.remove(R);
                    	kcontrol.removeb((int) R.longValue());
                    	new RemoveJob(table, ((Long) ra.remove(p)).longValue()).run();
                    }
                }
                try {Thread.sleep(1000);} catch (InterruptedException e) {}
                System.out.println("removed: " + rc + ", size of jcontrol set: " + jcontrol.size() + ", size of kcontrol set: " + kcontrol.size());
            }
            
            long aftercommand = System.currentTimeMillis();
            // final report
            System.out.println("Database size = " + table.size() + " unique entries.");
            
            // finally close the database/table
            table.close();
            
            long afterclose = System.currentTimeMillis();
            
            System.out.println("Execution time: open=" + (afterinit - startup) + ", command=" + (aftercommand - afterinit) + ", close=" + (afterclose - aftercommand) + ", total=" + (afterclose - startup));
            profiler.terminate();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}

final class memprofiler extends Thread {
    
    ymageChart memChart;
    boolean run;
    File outputFile;
    long start;
    
    public memprofiler(int width, int height, int expectedTimeSeconds, File outputFile) {
        this.outputFile = outputFile;
        int expectedKilobytes = 20 * 1024;//(Runtime.getRuntime().totalMemory() / 1024);
        memChart = new ymageChart(width, height, "000010", 50, 20, 20, 20, "MEMORY CHART FROM EXECUTION AT " + new Date());
        int timescale = 10; // steps with each 10 seconds
        int memscale = 1024;
        memChart.declareDimension(ymageChart.DIMENSION_BOTTOM, timescale, (width - 40) * timescale / expectedTimeSeconds, "FFFFFF", "555555", "SECONDS");
        memChart.declareDimension(ymageChart.DIMENSION_LEFT, memscale, (height - 40) * memscale / expectedKilobytes , "FFFFFF", "555555", "KILOBYTES");
        run = true;
        start = System.currentTimeMillis();
    }
    
    public void run() {
        int seconds0 = 0, kilobytes0 = 0;
        int seconds1 = 0, kilobytes1 = 0;
        while(run) {
            memChart.setColor("FF0000");
            seconds1 = (int) ((System.currentTimeMillis() - start) / 1000);
            kilobytes1 = (int) (serverMemory.used() / 1024);
            memChart.chartLine(ymageChart.DIMENSION_BOTTOM, ymageChart.DIMENSION_LEFT, seconds0, kilobytes0, seconds1, kilobytes1);
            seconds0 = seconds1;
            kilobytes0 = kilobytes1;
            try {Thread.sleep(100);} catch (InterruptedException e) {}
        }
        try {
            ImageIO.write(memChart.getImage(), "png", outputFile);
        } catch (IOException e) {}
    }
    
    public void terminate() {
        run = false;
        while (this.isAlive()) {
            try {Thread.sleep(1000);} catch (InterruptedException e) {}
        }
    }
}