From b7881829544aeb16b7db6ba4df4f39858e35069f Mon Sep 17 00:00:00 2001 From: orbiter Date: Wed, 13 Apr 2011 15:17:00 +0000 Subject: [PATCH] some enhancements to scoring speed git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7652 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- .../yacy/cora/storage/ConcurrentScoreMap.java | 6 +- source/net/yacy/cora/storage/IntScore.java | 105 ------------ .../yacy/cora/storage/OrderedScoreMap.java | 150 ++++++------------ source/net/yacy/cora/storage/ScoreMap.java | 2 - source/net/yacy/document/WordCache.java | 6 +- 5 files changed, 54 insertions(+), 215 deletions(-) delete mode 100644 source/net/yacy/cora/storage/IntScore.java diff --git a/source/net/yacy/cora/storage/ConcurrentScoreMap.java b/source/net/yacy/cora/storage/ConcurrentScoreMap.java index 8ec89bb97..087d60aa9 100644 --- a/source/net/yacy/cora/storage/ConcurrentScoreMap.java +++ b/source/net/yacy/cora/storage/ConcurrentScoreMap.java @@ -174,11 +174,11 @@ public class ConcurrentScoreMap implements ScoreMap { public Iterator keys(boolean up) { // re-organize entries - TreeMap> m = new TreeMap>(); + TreeMap> m = new TreeMap>(); Set s; - IntScore is; + Integer is; for (Map.Entry entry: this.map.entrySet()) { - is = new IntScore(entry.getValue().intValue()); + is = new Integer(entry.getValue().intValue()); s = m.get(is); if (s == null) { s = new HashSet(); diff --git a/source/net/yacy/cora/storage/IntScore.java b/source/net/yacy/cora/storage/IntScore.java deleted file mode 100644 index 43fe9251e..000000000 --- a/source/net/yacy/cora/storage/IntScore.java +++ /dev/null @@ -1,105 +0,0 @@ -/** - * IntScore - * Copyright 2010 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany - * First released 14.10.2010 at http://yacy.net - * - * $LastChangedDate$ - * $LastChangedRevision$ - * $LastChangedBy$ - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program in the file lgpl21.txt - * If not, see . - */ - -package net.yacy.cora.storage; - -import java.util.Comparator; - -/** - * This class acts as a replacement for Long and shall be used as counter object in Object-Counter relations - * The use case of this class is given when an value element of a map must be increased or decreased. If - * the normal Long class is used, the new value must be rewritten to the map with an increased and newly allocated number object - * When using this class, then only the score of the Number object can be changed without the need of - * rewriting the new key value to a map. - */ -public class IntScore implements Comparable, Comparator { - - private int value; - - public IntScore(int value) { - this.value = value; - } - - public final static IntScore valueOf(final int n) { - return new IntScore(n); - } - - public int intValue() { - return this.value; - } - - public void inc() { - this.value++; - } - - public void inc(int n) { - this.value += n; - } - - public void dec() { - this.value--; - } - - public void dec(int n) { - this.value -= n; - } - - public void set(int n) { - this.value = n; - } - - public void min(int n) { - if (n < this.value) this.value = n; - } - - public void max(int n) { - if (n > this.value) this.value = n; - } - - @Override - public boolean equals(Object o) { - return (o instanceof IntScore) && this.value == ((IntScore) o).value; - } - - @Override - public int hashCode() { - return this.value; - // return (int) (this.value ^ (this.value >>> 32)); // hash code for long values - } - - public int compareTo(IntScore o) { - int thisVal = this.value; - int anotherVal = o.value; - return thisVal < anotherVal ? -1 : (thisVal == anotherVal ? 0 : 1); - } - - public int compare(IntScore o1, IntScore o2) { - return o1.compareTo(o2); - } - - @Override - public String toString() { - return Integer.toString(this.value); - } -} diff --git a/source/net/yacy/cora/storage/OrderedScoreMap.java b/source/net/yacy/cora/storage/OrderedScoreMap.java index 61ccce30b..4cdca3397 100644 --- a/source/net/yacy/cora/storage/OrderedScoreMap.java +++ b/source/net/yacy/cora/storage/OrderedScoreMap.java @@ -35,25 +35,23 @@ import java.util.Set; import java.util.SortedMap; import java.util.TreeMap; import java.util.TreeSet; +import java.util.concurrent.atomic.AtomicInteger; public class OrderedScoreMap implements ScoreMap { - protected final Map map; // a mapping from a reference to the cluster key - private long gcount; + protected final Map map; // a mapping from a reference to the cluster key public OrderedScoreMap(Comparator comparator) { if (comparator == null) { - map = new HashMap(); + map = new HashMap(); } else { - map = new TreeMap(comparator); + map = new TreeMap(comparator); } - gcount = 0; } public synchronized void clear() { map.clear(); - gcount = 0; } /** @@ -74,9 +72,9 @@ public class OrderedScoreMap implements ScoreMap { * @param minScore */ public void shrinkToMinScore(int minScore) { - synchronized (this) { - Iterator> i = this.map.entrySet().iterator(); - Map.Entry entry; + synchronized (map) { + Iterator> i = this.map.entrySet().iterator(); + Map.Entry entry; while (i.hasNext()) { entry = i.next(); if (entry.getValue().intValue() < minScore) i.remove(); @@ -84,10 +82,6 @@ public class OrderedScoreMap implements ScoreMap { } } - public synchronized long totalCount() { - return gcount; - } - public synchronized int size() { return map.size(); } @@ -98,59 +92,53 @@ public class OrderedScoreMap implements ScoreMap { public void inc(final E obj) { if (obj == null) return; - synchronized (this) { - IntScore score = this.map.get(obj); + AtomicInteger score; + synchronized (map) { + score = this.map.get(obj); if (score == null) { - this.map.put(obj, new IntScore(1)); - } else { - score.inc(); + this.map.put(obj, new AtomicInteger(1)); + return; } - } - // increase overall counter - gcount++; + } + score.incrementAndGet(); } public void dec(final E obj) { if (obj == null) return; - synchronized (this) { - IntScore score = this.map.get(obj); + AtomicInteger score; + synchronized (map) { + score = this.map.get(obj); if (score == null) { - this.map.put(obj, IntScore.valueOf(-1)); - } else { - score.dec(); + this.map.put(obj, new AtomicInteger(-1)); + return; } - } - // increase overall counter - gcount--; + } + score.decrementAndGet(); } public void set(final E obj, final int newScore) { if (obj == null) return; - synchronized (this) { - IntScore score = this.map.get(obj); + AtomicInteger score; + synchronized (map) { + score = this.map.get(obj); if (score == null) { - this.map.put(obj, new IntScore(1)); - } else { - gcount -= score.intValue(); - score.set(newScore); + this.map.put(obj, new AtomicInteger(newScore)); + return; } - } - // increase overall counter - gcount += newScore; + } + score.getAndSet(newScore); } public void inc(final E obj, final int incrementScore) { if (obj == null) return; - synchronized (this) { - IntScore score = this.map.get(obj); + AtomicInteger score; + synchronized (map) { + score = this.map.get(obj); if (score == null) { - this.map.put(obj, IntScore.valueOf(incrementScore)); - } else { - score.inc(incrementScore); + this.map.put(obj, new AtomicInteger(incrementScore)); } - } - // increase overall counter - gcount += incrementScore; + } + score.addAndGet(incrementScore); } public void dec(final E obj, final int incrementScore) { @@ -160,14 +148,11 @@ public class OrderedScoreMap implements ScoreMap { public int delete(final E obj) { // deletes entry and returns previous score if (obj == null) return 0; - final IntScore score; - synchronized (this) { + final AtomicInteger score; + synchronized (map) { score = map.remove(obj); if (score == null) return 0; } - - // decrease overall counter - gcount -= score.intValue(); return score.intValue(); } @@ -177,17 +162,17 @@ public class OrderedScoreMap implements ScoreMap { public int get(final E obj) { if (obj == null) return 0; - final IntScore score; - synchronized (this) { + final AtomicInteger score; + synchronized (map) { score = map.get(obj); } if (score == null) return 0; return score.intValue(); } - public SortedMap tailMap(E obj) { + public SortedMap tailMap(E obj) { if (this.map instanceof TreeMap) { - return ((TreeMap) this.map).tailMap(obj); + return ((TreeMap) this.map).tailMap(obj); } throw new UnsupportedOperationException("map must have comparator"); } @@ -195,69 +180,30 @@ public class OrderedScoreMap implements ScoreMap { private int getMinScore() { if (map.isEmpty()) return -1; int minScore = Integer.MAX_VALUE; - synchronized (this) { - for (Map.Entry entry: this.map.entrySet()) if (entry.getValue().intValue() < minScore) { + synchronized (map) { + for (Map.Entry entry: this.map.entrySet()) if (entry.getValue().intValue() < minScore) { minScore = entry.getValue().intValue(); } } return minScore; } - /* - public int getMaxScore() { - if (map.isEmpty()) return -1; - int maxScore = Integer.MIN_VALUE; - synchronized (this) { - for (Map.Entry entry: this.map.entrySet()) if (entry.getValue().intValue() > maxScore) { - maxScore = entry.getValue().intValue(); - } - } - return maxScore; - } - - public E getMaxKey() { - if (map.isEmpty()) return null; - E maxObject = null; - int maxScore = Integer.MIN_VALUE; - synchronized (this) { - for (Map.Entry entry: this.map.entrySet()) if (entry.getValue().intValue() > maxScore) { - maxScore = entry.getValue().intValue(); - maxObject = entry.getKey(); - } - } - return maxObject; - } - - public E getMinKey() { - if (map.isEmpty()) return null; - E minObject = null; - int minScore = Integer.MAX_VALUE; - synchronized (this) { - for (Map.Entry entry: this.map.entrySet()) if (entry.getValue().intValue() < minScore) { - minScore = entry.getValue().intValue(); - minObject = entry.getKey(); - } - } - return minObject; - } - */ - @Override public String toString() { return map.toString(); } public Iterator keys(boolean up) { - synchronized (this) { + synchronized (map) { // re-organize entries - TreeMap> m = new TreeMap>(); + TreeMap> m = new TreeMap>(); Set s; - for (Map.Entry entry: this.map.entrySet()) { - s = m.get(entry.getValue()); + for (Map.Entry entry: this.map.entrySet()) { + s = m.get(entry.getValue().intValue()); if (s == null) { - s = this.map instanceof TreeMap ? new TreeSet(((TreeMap) this.map).comparator()) : new HashSet(); + s = this.map instanceof TreeMap ? new TreeSet(((TreeMap) this.map).comparator()) : new HashSet(); s.add(entry.getKey()); - m.put(entry.getValue(), s); + m.put(entry.getValue().intValue(), s); } else { s.add(entry.getKey()); } diff --git a/source/net/yacy/cora/storage/ScoreMap.java b/source/net/yacy/cora/storage/ScoreMap.java index 9a87f1c8c..49a60ee37 100644 --- a/source/net/yacy/cora/storage/ScoreMap.java +++ b/source/net/yacy/cora/storage/ScoreMap.java @@ -42,8 +42,6 @@ public interface ScoreMap { */ public void shrinkToMinScore(int minScore); - public long totalCount(); - public int size(); public boolean isEmpty(); diff --git a/source/net/yacy/document/WordCache.java b/source/net/yacy/document/WordCache.java index 2eec00aef..a917728cf 100644 --- a/source/net/yacy/document/WordCache.java +++ b/source/net/yacy/document/WordCache.java @@ -33,9 +33,9 @@ import java.util.Set; import java.util.SortedMap; import java.util.SortedSet; import java.util.TreeSet; +import java.util.concurrent.atomic.AtomicInteger; import java.util.zip.GZIPInputStream; -import net.yacy.cora.storage.IntScore; import net.yacy.cora.storage.OrderedScoreMap; import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.util.MemoryControl; @@ -138,10 +138,10 @@ public class WordCache { for (final String r: t) { if (r.startsWith(string) && r.length() > string.length()) ret.add(r); else break; } - SortedMap u = this.commonWords.tailMap(string); + SortedMap u = this.commonWords.tailMap(string); String vv; try { - for (final Map.Entry v: u.entrySet()) { + for (final Map.Entry v: u.entrySet()) { vv = v.getKey(); if (vv.startsWith(string) && vv.length() > string.length()) ret.add(vv); else break; }