boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

some enhancements to scoring speed

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7652 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 14 years ago
parent 01690eab86
commit b788182954
5 changed files with 54 additions and 215 deletions
Show Stats Download Patch File Download Diff File

6
source/net/yacy/cora/storage/ConcurrentScoreMap.java
Unescape View File

@ -174,11 +174,11 @@ public class ConcurrentScoreMap<E> implements ScoreMap<E> {
public Iterator<E> keys(boolean up) { public Iterator<E> keys(boolean up) {
// re-organize entries // re-organize entries
TreeMap<IntScore, Set<E>> m = new TreeMap<IntScore, Set<E>>(); TreeMap<Integer, Set<E>> m = new TreeMap<Integer, Set<E>>();
Set<E> s; Set<E> s;
IntScore is; Integer is;
for (Map.Entry<E, AtomicLong> entry: this.map.entrySet()) { for (Map.Entry<E, AtomicLong> entry: this.map.entrySet()) {
is = new IntScore(entry.getValue().intValue()); is = new Integer(entry.getValue().intValue());
s = m.get(is); s = m.get(is);
if (s == null) { if (s == null) {
s = new HashSet<E>(); s = new HashSet<E>();

105
source/net/yacy/cora/storage/IntScore.java
Unescape View File

@ -1,105 +0,0 @@
/**
* IntScore
* Copyright 2010 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany
* First released 14.10.2010 at http://yacy.net
*
* $LastChangedDate$
* $LastChangedRevision$
* $LastChangedBy$
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program in the file lgpl21.txt
* If not, see <http://www.gnu.org/licenses/>.
*/
package net.yacy.cora.storage;
import java.util.Comparator;
/**
* This class acts as a replacement for Long and shall be used as counter object in Object-Counter relations
* The use case of this class is given when an value element of a map must be increased or decreased. If
* the normal Long class is used, the new value must be rewritten to the map with an increased and newly allocated number object
* When using this class, then only the score of the Number object can be changed without the need of
* rewriting the new key value to a map.
*/
public class IntScore implements Comparable<IntScore>, Comparator<IntScore> {
private int value;
public IntScore(int value) {
this.value = value;
}
public final static IntScore valueOf(final int n) {
return new IntScore(n);
}
public int intValue() {
return this.value;
}
public void inc() {
this.value++;
}
public void inc(int n) {
this.value += n;
}
public void dec() {
this.value--;
}
public void dec(int n) {
this.value -= n;
}
public void set(int n) {
this.value = n;
}
public void min(int n) {
if (n < this.value) this.value = n;
}
public void max(int n) {
if (n > this.value) this.value = n;
}
@Override
public boolean equals(Object o) {
return (o instanceof IntScore) && this.value == ((IntScore) o).value;
}
@Override
public int hashCode() {
return this.value;
// return (int) (this.value ^ (this.value >>> 32)); // hash code for long values
}
public int compareTo(IntScore o) {
int thisVal = this.value;
int anotherVal = o.value;
return thisVal < anotherVal ? -1 : (thisVal == anotherVal ? 0 : 1);
}
public int compare(IntScore o1, IntScore o2) {
return o1.compareTo(o2);
}
@Override
public String toString() {
return Integer.toString(this.value);
}
}

142
source/net/yacy/cora/storage/OrderedScoreMap.java
Unescape View File

@ -35,25 +35,23 @@ import java.util.Set;
import java.util.SortedMap; import java.util.SortedMap;
import java.util.TreeMap; import java.util.TreeMap;
import java.util.TreeSet; import java.util.TreeSet;
import java.util.concurrent.atomic.AtomicInteger;
public class OrderedScoreMap<E> implements ScoreMap<E> { public class OrderedScoreMap<E> implements ScoreMap<E> {
protected final Map<E, IntScore> map; // a mapping from a reference to the cluster key protected final Map<E, AtomicInteger> map; // a mapping from a reference to the cluster key
private long gcount;
public OrderedScoreMap(Comparator<? super E> comparator) { public OrderedScoreMap(Comparator<? super E> comparator) {
if (comparator == null) { if (comparator == null) {
map = new HashMap<E, IntScore>(); map = new HashMap<E, AtomicInteger>();
} else { } else {
map = new TreeMap<E, IntScore>(comparator); map = new TreeMap<E, AtomicInteger>(comparator);
} }
gcount = 0;
} }
public synchronized void clear() { public synchronized void clear() {
map.clear(); map.clear();
gcount = 0;
} }
/** /**
@ -74,9 +72,9 @@ public class OrderedScoreMap<E> implements ScoreMap<E> {
* @param minScore * @param minScore
*/ */
public void shrinkToMinScore(int minScore) { public void shrinkToMinScore(int minScore) {
synchronized (this) { synchronized (map) {
Iterator<Map.Entry<E, IntScore>> i = this.map.entrySet().iterator(); Iterator<Map.Entry<E, AtomicInteger>> i = this.map.entrySet().iterator();
Map.Entry<E, IntScore> entry; Map.Entry<E, AtomicInteger> entry;
while (i.hasNext()) { while (i.hasNext()) {
entry = i.next(); entry = i.next();
if (entry.getValue().intValue() < minScore) i.remove(); if (entry.getValue().intValue() < minScore) i.remove();
@ -84,10 +82,6 @@ public class OrderedScoreMap<E> implements ScoreMap<E> {
} }
} }
public synchronized long totalCount() {
return gcount;
}
public synchronized int size() { public synchronized int size() {
return map.size(); return map.size();
} }
@ -98,59 +92,53 @@ public class OrderedScoreMap<E> implements ScoreMap<E> {
public void inc(final E obj) { public void inc(final E obj) {
if (obj == null) return; if (obj == null) return;
synchronized (this) { AtomicInteger score;
IntScore score = this.map.get(obj); synchronized (map) {
score = this.map.get(obj);
if (score == null) { if (score == null) {
this.map.put(obj, new IntScore(1)); this.map.put(obj, new AtomicInteger(1));
} else { return;
score.inc();
} }
} }
// increase overall counter score.incrementAndGet();
gcount++;
} }
public void dec(final E obj) { public void dec(final E obj) {
if (obj == null) return; if (obj == null) return;
synchronized (this) { AtomicInteger score;
IntScore score = this.map.get(obj); synchronized (map) {
score = this.map.get(obj);
if (score == null) { if (score == null) {
this.map.put(obj, IntScore.valueOf(-1)); this.map.put(obj, new AtomicInteger(-1));
} else { return;
score.dec();
} }
} }
// increase overall counter score.decrementAndGet();
gcount--;
} }
public void set(final E obj, final int newScore) { public void set(final E obj, final int newScore) {
if (obj == null) return; if (obj == null) return;
synchronized (this) { AtomicInteger score;
IntScore score = this.map.get(obj); synchronized (map) {
score = this.map.get(obj);
if (score == null) { if (score == null) {
this.map.put(obj, new IntScore(1)); this.map.put(obj, new AtomicInteger(newScore));
} else { return;
gcount -= score.intValue();
score.set(newScore);
} }
} }
// increase overall counter score.getAndSet(newScore);
gcount += newScore;
} }
public void inc(final E obj, final int incrementScore) { public void inc(final E obj, final int incrementScore) {
if (obj == null) return; if (obj == null) return;
synchronized (this) { AtomicInteger score;
IntScore score = this.map.get(obj); synchronized (map) {
score = this.map.get(obj);
if (score == null) { if (score == null) {
this.map.put(obj, IntScore.valueOf(incrementScore)); this.map.put(obj, new AtomicInteger(incrementScore));
} else {
score.inc(incrementScore);
} }
} }
// increase overall counter score.addAndGet(incrementScore);
gcount += incrementScore;
} }
public void dec(final E obj, final int incrementScore) { public void dec(final E obj, final int incrementScore) {
@ -160,14 +148,11 @@ public class OrderedScoreMap<E> implements ScoreMap<E> {
public int delete(final E obj) { public int delete(final E obj) {
// deletes entry and returns previous score // deletes entry and returns previous score
if (obj == null) return 0; if (obj == null) return 0;
final IntScore score; final AtomicInteger score;
synchronized (this) { synchronized (map) {
score = map.remove(obj); score = map.remove(obj);
if (score == null) return 0; if (score == null) return 0;
} }
// decrease overall counter
gcount -= score.intValue();
return score.intValue(); return score.intValue();
} }
@ -177,17 +162,17 @@ public class OrderedScoreMap<E> implements ScoreMap<E> {
public int get(final E obj) { public int get(final E obj) {
if (obj == null) return 0; if (obj == null) return 0;
final IntScore score; final AtomicInteger score;
synchronized (this) { synchronized (map) {
score = map.get(obj); score = map.get(obj);
} }
if (score == null) return 0; if (score == null) return 0;
return score.intValue(); return score.intValue();
} }
public SortedMap<E, IntScore> tailMap(E obj) { public SortedMap<E, AtomicInteger> tailMap(E obj) {
if (this.map instanceof TreeMap) { if (this.map instanceof TreeMap) {
return ((TreeMap<E, IntScore>) this.map).tailMap(obj); return ((TreeMap<E, AtomicInteger>) this.map).tailMap(obj);
} }
throw new UnsupportedOperationException("map must have comparator"); throw new UnsupportedOperationException("map must have comparator");
} }
@ -195,69 +180,30 @@ public class OrderedScoreMap<E> implements ScoreMap<E> {
private int getMinScore() { private int getMinScore() {
if (map.isEmpty()) return -1; if (map.isEmpty()) return -1;
int minScore = Integer.MAX_VALUE; int minScore = Integer.MAX_VALUE;
synchronized (this) { synchronized (map) {
for (Map.Entry<E, IntScore> entry: this.map.entrySet()) if (entry.getValue().intValue() < minScore) { for (Map.Entry<E, AtomicInteger> entry: this.map.entrySet()) if (entry.getValue().intValue() < minScore) {
minScore = entry.getValue().intValue(); minScore = entry.getValue().intValue();
} }
} }
return minScore; return minScore;
} }
/*
public int getMaxScore() {
if (map.isEmpty()) return -1;
int maxScore = Integer.MIN_VALUE;
synchronized (this) {
for (Map.Entry<E, IntScore> entry: this.map.entrySet()) if (entry.getValue().intValue() > maxScore) {
maxScore = entry.getValue().intValue();
}
}
return maxScore;
}
public E getMaxKey() {
if (map.isEmpty()) return null;
E maxObject = null;
int maxScore = Integer.MIN_VALUE;
synchronized (this) {
for (Map.Entry<E, IntScore> entry: this.map.entrySet()) if (entry.getValue().intValue() > maxScore) {
maxScore = entry.getValue().intValue();
maxObject = entry.getKey();
}
}
return maxObject;
}
public E getMinKey() {
if (map.isEmpty()) return null;
E minObject = null;
int minScore = Integer.MAX_VALUE;
synchronized (this) {
for (Map.Entry<E, IntScore> entry: this.map.entrySet()) if (entry.getValue().intValue() < minScore) {
minScore = entry.getValue().intValue();
minObject = entry.getKey();
}
}
return minObject;
}
*/
@Override @Override
public String toString() { public String toString() {
return map.toString(); return map.toString();
} }
public Iterator<E> keys(boolean up) { public Iterator<E> keys(boolean up) {
synchronized (this) { synchronized (map) {
// re-organize entries // re-organize entries
TreeMap<IntScore, Set<E>> m = new TreeMap<IntScore, Set<E>>(); TreeMap<Integer, Set<E>> m = new TreeMap<Integer, Set<E>>();
Set<E> s; Set<E> s;
for (Map.Entry<E, IntScore> entry: this.map.entrySet()) { for (Map.Entry<E, AtomicInteger> entry: this.map.entrySet()) {
s = m.get(entry.getValue()); s = m.get(entry.getValue().intValue());
if (s == null) { if (s == null) {
s = this.map instanceof TreeMap ? new TreeSet<E>(((TreeMap<E, IntScore>) this.map).comparator()) : new HashSet<E>(); s = this.map instanceof TreeMap ? new TreeSet<E>(((TreeMap<E, AtomicInteger>) this.map).comparator()) : new HashSet<E>();
s.add(entry.getKey()); s.add(entry.getKey());
m.put(entry.getValue(), s); m.put(entry.getValue().intValue(), s);
} else { } else {
s.add(entry.getKey()); s.add(entry.getKey());
} }

2
source/net/yacy/cora/storage/ScoreMap.java
Unescape View File

@ -42,8 +42,6 @@ public interface ScoreMap<E> {
*/ */
public void shrinkToMinScore(int minScore); public void shrinkToMinScore(int minScore);
public long totalCount();
public int size(); public int size();
public boolean isEmpty(); public boolean isEmpty();

6
source/net/yacy/document/WordCache.java
Unescape View File

@ -33,9 +33,9 @@ import java.util.Set;
import java.util.SortedMap; import java.util.SortedMap;
import java.util.SortedSet; import java.util.SortedSet;
import java.util.TreeSet; import java.util.TreeSet;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.zip.GZIPInputStream; import java.util.zip.GZIPInputStream;
import net.yacy.cora.storage.IntScore;
import net.yacy.cora.storage.OrderedScoreMap; import net.yacy.cora.storage.OrderedScoreMap;
import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.util.MemoryControl; import net.yacy.kelondro.util.MemoryControl;
@ -138,10 +138,10 @@ public class WordCache {
for (final String r: t) { for (final String r: t) {
if (r.startsWith(string) && r.length() > string.length()) ret.add(r); else break; if (r.startsWith(string) && r.length() > string.length()) ret.add(r); else break;
} }
SortedMap<String, IntScore> u = this.commonWords.tailMap(string); SortedMap<String, AtomicInteger> u = this.commonWords.tailMap(string);
String vv; String vv;
try { try {
for (final Map.Entry<String, IntScore> v: u.entrySet()) { for (final Map.Entry<String, AtomicInteger> v: u.entrySet()) {
vv = v.getKey(); vv = v.getKey();
if (vv.startsWith(string) && vv.length() > string.length()) ret.add(vv); else break; if (vv.startsWith(string) && vv.length() > string.length()) ret.add(vv); else break;
} }

Write Preview
Loading…
Cancel
Save