enhanced identificator: using AtomicInteger for counter

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7785 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 14 years ago
parent 16327d1cbe
commit f667b9c289

@ -29,6 +29,7 @@ import java.util.Iterator;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Vector; import java.util.Vector;
import java.util.concurrent.atomic.AtomicInteger;
/** /**
* This class can try to identify the language a text is written in. * This class can try to identify the language a text is written in.
@ -36,15 +37,15 @@ import java.util.Vector;
public final class Identificator { public final class Identificator {
private static final LanguageStatisticsHolder languages = LanguageStatisticsHolder.getInstance(); private static final LanguageStatisticsHolder languages = LanguageStatisticsHolder.getInstance();
private final Map<Character, Integer> letter; private final Map<Character, AtomicInteger> letter;
private int letters; private int letters;
private String language; private String language;
public Identificator() { public Identificator() {
letter = new HashMap<Character, Integer>(); this.letter = new HashMap<Character, AtomicInteger>();
letters = 0; this.letters = 0;
language = null; this.language = null;
} }
/** /**
@ -58,7 +59,7 @@ public final class Identificator {
// only test the first 100000 characters of a text // only test the first 100000 characters of a text
return getLanguage(text, 100000); return getLanguage(text, 100000);
} }
/** /**
* This method tries to return the language a text is written in. The method will * This method tries to return the language a text is written in. The method will
* use the number characters defined in the parameter limit. * use the number characters defined in the parameter limit.
@ -67,82 +68,82 @@ public final class Identificator {
* @return the language or null if the method was not able to find out the language * @return the language or null if the method was not able to find out the language
*/ */
public static String getLanguage(final String text, final int limit) { public static String getLanguage(final String text, final int limit) {
int upperLimit = text.length(); int upperLimit = text.length();
if (upperLimit > limit) { if (upperLimit > limit) {
upperLimit = limit; upperLimit = limit;
} }
Identificator id = new Identificator(); final Identificator id = new Identificator();
// count number of characters in text // count number of characters in text
for (int i = 0; i < upperLimit; i++) id.inc(text.charAt(i)); for (int i = 0; i < upperLimit; i++) id.inc(text.charAt(i));
return id.getLanguage(); return id.getLanguage();
} }
public void inc(final char c) { public void inc(final char c) {
if (!Character.isLetter(c)) return; if (!Character.isLetter(c)) return;
Character cc = Character.toLowerCase(c); final Character cc = Character.toLowerCase(c);
Integer i = letter.get(cc); final AtomicInteger i = this.letter.get(cc);
if (i == null) { if (i == null) {
letter.put(cc, 1); this.letter.put(cc, new AtomicInteger(1));
} else { } else {
letter.put(cc, i.intValue() + 1); i.incrementAndGet();
} }
letters++; this.letters++;
} }
public void add(String word) { public void add(final String word) {
if (word == null) return; if (word == null) return;
for (int i = 0; i < word.length(); i++) inc(word.charAt(i)); for (int i = 0; i < word.length(); i++) inc(word.charAt(i));
} }
public String getLanguage() { public String getLanguage() {
if (language != null) return language; // don't compute that twice if (this.language != null) return this.language; // don't compute that twice
if (letters == 0) return null; // not enough information available if (this.letters == 0) return null; // not enough information available
final LanguageStatistics testStat = new LanguageStatistics("text"); final LanguageStatistics testStat = new LanguageStatistics("text");
// calculate percentage // calculate percentage
Character character; Character character;
Character maxChar = null; Character maxChar = null;
int count = 0; int count = 0;
int max = 0; int max = 0;
for (Map.Entry<Character, Integer> e: letter.entrySet()) { for (final Map.Entry<Character, AtomicInteger> e: this.letter.entrySet()) {
character = e.getKey(); character = e.getKey();
count = e.getValue().intValue(); count = e.getValue().intValue();
if (count > max) { if (count > max) {
maxChar = character; maxChar = character;
max = count; max = count;
} }
testStat.put(character, ((float) 100) * ((float) count) / ((float) letters)); testStat.put(character, ((float) 100) * ((float) count) / (this.letters));
} }
// create list with relevant languages // create list with relevant languages
final List<Integer> relevantLanguages = new Vector <Integer>(); final List<Integer> relevantLanguages = new Vector <Integer>();
for (int i = 0; i < languages.size(); i++) { for (int i = 0; i < languages.size(); i++) {
// only languages that contain the most common character in the text will be tested // only languages that contain the most common character in the text will be tested
if (languages.get(i).contains(maxChar)) { if (languages.get(i).contains(maxChar)) {
relevantLanguages.add(i); relevantLanguages.add(i);
} }
} }
if (relevantLanguages.isEmpty()) return null; if (relevantLanguages.isEmpty()) return null;
// compare characters in text with characters in statistics // compare characters in text with characters in statistics
final float[] offsetList = new float[relevantLanguages.size()]; final float[] offsetList = new float[relevantLanguages.size()];
final int[] votesList = new int[relevantLanguages.size()]; final int[] votesList = new int[relevantLanguages.size()];
Iterator<Character> iter = testStat.keySet().iterator(); final Iterator<Character> iter = testStat.keySet().iterator();
float minimum; float minimum;
float offset = 0; float offset = 0;
float valueCharacter; float valueCharacter;
int bestLanguage = -1; int bestLanguage = -1;
float value; float value;
while (iter.hasNext()) { while (iter.hasNext()) {
minimum = 100.1f; minimum = 100.1f;
character = iter.next(); character = iter.next();
@ -158,7 +159,7 @@ public final class Identificator {
} }
votesList[bestLanguage] = ++votesList[bestLanguage]; votesList[bestLanguage] = ++votesList[bestLanguage];
} }
// Now we can count how many votes each language got and how far it was away from the stats. // Now we can count how many votes each language got and how far it was away from the stats.
// If 2 languages have the same amount of votes, the one with the smaller offset wins. // If 2 languages have the same amount of votes, the one with the smaller offset wins.
int maxVotes = 0; int maxVotes = 0;
@ -170,17 +171,17 @@ public final class Identificator {
bestLanguage = i; bestLanguage = i;
} }
} }
// Only return name of language of offset is smaller than 20%. This // Only return name of language of offset is smaller than 20%. This
// prevents a language beeing reported that has won the voting, but // prevents a language beeing reported that has won the voting, but
// is still not the right language. // is still not the right language.
if (offset < 20) { if (offset < 20) {
language = languages.get(relevantLanguages.get(bestLanguage)).getName(); this.language = languages.get(relevantLanguages.get(bestLanguage)).getName();
return language; return this.language;
} }
return null; return null;
} }
} }

Loading…
Cancel
Save