From 733385cdd7ebcb00fae165e10f4781444c3bc223 Mon Sep 17 00:00:00 2001 From: orbiter Date: Wed, 10 Jun 2009 23:02:42 +0000 Subject: [PATCH] enahnced database access times by removal of unnecessary synchronization. added also more hacks that resulted from high-volum query testing git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6047 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- htroot/yacysearch.java | 10 +++- source/de/anomic/kelondro/blob/BLOBArray.java | 2 +- source/de/anomic/kelondro/text/IndexCell.java | 46 +++++++++---------- .../text/ReferenceContainerArray.java | 2 +- .../de/anomic/plasma/plasmaSwitchboard.java | 7 +-- 5 files changed, 36 insertions(+), 31 deletions(-) diff --git a/htroot/yacysearch.java b/htroot/yacysearch.java index b1d7aa6fe..ed967a4e9 100644 --- a/htroot/yacysearch.java +++ b/htroot/yacysearch.java @@ -467,8 +467,14 @@ public class yacysearch { sb.localSearches.add(theQuery); // update the search tracker - trackerHandles.add(theQuery.handle); - sb.localSearchTracker.put(client, trackerHandles); + try { + trackerHandles.add(theQuery.handle); + if (trackerHandles.size() > 1000) trackerHandles.remove(trackerHandles.first()); + sb.localSearchTracker.put(client, trackerHandles); + if (sb.localSearchTracker.size() > 1000) sb.localSearchTracker.remove(sb.localSearchTracker.keys().nextElement()); + } catch (Exception e) { + e.printStackTrace(); + } final int totalcount = theSearch.getRankingResult().getLocalResourceSize() + theSearch.getRankingResult().getRemoteResourceSize(); prop.put("num-results_offset", offset); diff --git a/source/de/anomic/kelondro/blob/BLOBArray.java b/source/de/anomic/kelondro/blob/BLOBArray.java index d19e25ad0..c05e9842c 100755 --- a/source/de/anomic/kelondro/blob/BLOBArray.java +++ b/source/de/anomic/kelondro/blob/BLOBArray.java @@ -480,7 +480,7 @@ public class BLOBArray implements BLOB { * @return * @throws IOException */ - public synchronized Iterable getAll(byte[] key) throws IOException { + public Iterable getAll(byte[] key) throws IOException { /* byte[] b; ArrayList l = new ArrayList(blobs.size()); diff --git a/source/de/anomic/kelondro/text/IndexCell.java b/source/de/anomic/kelondro/text/IndexCell.java index e39782f37..15b6cb132 100644 --- a/source/de/anomic/kelondro/text/IndexCell.java +++ b/source/de/anomic/kelondro/text/IndexCell.java @@ -276,32 +276,30 @@ public final class IndexCell extends AbstractBu * cache control methods */ - private synchronized void cleanCache() { + private void cleanCache() { + // dump the cache if necessary - if (this.ram.size() >= this.maxRamEntries || (this.ram.size() > 3000 && !MemoryControl.request(80L * 1024L * 1024L, false))) { - try { - cacheDump(); - } catch (IOException e) { - e.printStackTrace(); - } - } - + synchronized (this) { + if (this.ram.size() >= this.maxRamEntries || (this.ram.size() > 3000 && !MemoryControl.request(80L * 1024L * 1024L, false))) { + // dump the ram + File dumpFile = this.array.newContainerBLOBFile(); + //this.ram.dump(dumpFile, true); + //this.array.mountBLOBContainer(dumpFile); + merger.dump(this.ram, dumpFile, array); + // get a fresh ram cache + this.ram = new ReferenceContainerCache(factory, this.array.rowdef(), this.array.ordering()); + this.ram.initWriteMode(); + } + } + // clean-up the cache - if (this.array.entries() < 50 && (this.lastCleanup + cleanupCycle > System.currentTimeMillis())) return; - //System.out.println("----cleanup check"); - this.array.shrink(this.targetFileSize, this.maxFileSize); - this.lastCleanup = System.currentTimeMillis(); - } - - private synchronized void cacheDump() throws IOException { - // dump the ram - File dumpFile = this.array.newContainerBLOBFile(); - //this.ram.dump(dumpFile, true); - //this.array.mountBLOBContainer(dumpFile); - merger.dump(this.ram, dumpFile, array); - // get a fresh ram cache - this.ram = new ReferenceContainerCache(factory, this.array.rowdef(), this.array.ordering()); - this.ram.initWriteMode(); + synchronized (this) { + if (this.array.entries() > 50 || (this.lastCleanup + cleanupCycle < System.currentTimeMillis())) { + //System.out.println("----cleanup check"); + this.array.shrink(this.targetFileSize, this.maxFileSize); + this.lastCleanup = System.currentTimeMillis(); + } + } } public File newContainerBLOBFile() { diff --git a/source/de/anomic/kelondro/text/ReferenceContainerArray.java b/source/de/anomic/kelondro/text/ReferenceContainerArray.java index 4a5177d76..642c9a649 100644 --- a/source/de/anomic/kelondro/text/ReferenceContainerArray.java +++ b/source/de/anomic/kelondro/text/ReferenceContainerArray.java @@ -195,7 +195,7 @@ public final class ReferenceContainerArray { * @return the indexContainer if one exist, null otherwise * @throws IOException */ - public synchronized ReferenceContainer get(final byte[] termHash) throws IOException { + public ReferenceContainer get(final byte[] termHash) throws IOException { long timeout = System.currentTimeMillis() + 1000; Iterator entries = this.array.getAll(termHash).iterator(); if (entries == null || !entries.hasNext()) return null; diff --git a/source/de/anomic/plasma/plasmaSwitchboard.java b/source/de/anomic/plasma/plasmaSwitchboard.java index ba945fa5c..3ecaa8bbc 100644 --- a/source/de/anomic/plasma/plasmaSwitchboard.java +++ b/source/de/anomic/plasma/plasmaSwitchboard.java @@ -109,6 +109,7 @@ import java.util.Properties; import java.util.Set; import java.util.TreeMap; import java.util.TreeSet; +import java.util.concurrent.ConcurrentHashMap; import java.util.regex.Pattern; import de.anomic.content.DCEntry; @@ -247,7 +248,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch localSearches; // array of search result properties as HashMaps public ArrayList remoteSearches; // array of search result properties as HashMaps - public HashMap> localSearchTracker, remoteSearchTracker; // mappings from requesting host to a TreeSet of Long(access time) + public ConcurrentHashMap> localSearchTracker, remoteSearchTracker; // mappings from requesting host to a TreeSet of Long(access time) public long indexedPages = 0; public double requestedQueries = 0d; public double totalQPM = 0d; @@ -551,8 +552,8 @@ public final class plasmaSwitchboard extends serverAbstractSwitch(); // init search history trackers - this.localSearchTracker = new HashMap>(); // String:TreeSet - IP:set of Long(accessTime) - this.remoteSearchTracker = new HashMap>(); + this.localSearchTracker = new ConcurrentHashMap>(); // String:TreeSet - IP:set of Long(accessTime) + this.remoteSearchTracker = new ConcurrentHashMap>(); this.localSearches = new ArrayList(); // contains search result properties as HashMaps this.remoteSearches = new ArrayList();