|
|
@ -29,6 +29,7 @@ import java.util.Iterator;
|
|
|
|
import java.util.List;
|
|
|
|
import java.util.List;
|
|
|
|
import java.util.Map;
|
|
|
|
import java.util.Map;
|
|
|
|
import java.util.Vector;
|
|
|
|
import java.util.Vector;
|
|
|
|
|
|
|
|
import java.util.concurrent.atomic.AtomicInteger;
|
|
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
/**
|
|
|
|
* This class can try to identify the language a text is written in.
|
|
|
|
* This class can try to identify the language a text is written in.
|
|
|
@ -36,15 +37,15 @@ import java.util.Vector;
|
|
|
|
public final class Identificator {
|
|
|
|
public final class Identificator {
|
|
|
|
|
|
|
|
|
|
|
|
private static final LanguageStatisticsHolder languages = LanguageStatisticsHolder.getInstance();
|
|
|
|
private static final LanguageStatisticsHolder languages = LanguageStatisticsHolder.getInstance();
|
|
|
|
|
|
|
|
|
|
|
|
private final Map<Character, Integer> letter;
|
|
|
|
private final Map<Character, AtomicInteger> letter;
|
|
|
|
private int letters;
|
|
|
|
private int letters;
|
|
|
|
private String language;
|
|
|
|
private String language;
|
|
|
|
|
|
|
|
|
|
|
|
public Identificator() {
|
|
|
|
public Identificator() {
|
|
|
|
letter = new HashMap<Character, Integer>();
|
|
|
|
this.letter = new HashMap<Character, AtomicInteger>();
|
|
|
|
letters = 0;
|
|
|
|
this.letters = 0;
|
|
|
|
language = null;
|
|
|
|
this.language = null;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
/**
|
|
|
@ -58,7 +59,7 @@ public final class Identificator {
|
|
|
|
// only test the first 100000 characters of a text
|
|
|
|
// only test the first 100000 characters of a text
|
|
|
|
return getLanguage(text, 100000);
|
|
|
|
return getLanguage(text, 100000);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
/**
|
|
|
|
* This method tries to return the language a text is written in. The method will
|
|
|
|
* This method tries to return the language a text is written in. The method will
|
|
|
|
* use the number characters defined in the parameter limit.
|
|
|
|
* use the number characters defined in the parameter limit.
|
|
|
@ -67,82 +68,82 @@ public final class Identificator {
|
|
|
|
* @return the language or null if the method was not able to find out the language
|
|
|
|
* @return the language or null if the method was not able to find out the language
|
|
|
|
*/
|
|
|
|
*/
|
|
|
|
public static String getLanguage(final String text, final int limit) {
|
|
|
|
public static String getLanguage(final String text, final int limit) {
|
|
|
|
|
|
|
|
|
|
|
|
int upperLimit = text.length();
|
|
|
|
int upperLimit = text.length();
|
|
|
|
if (upperLimit > limit) {
|
|
|
|
if (upperLimit > limit) {
|
|
|
|
upperLimit = limit;
|
|
|
|
upperLimit = limit;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
Identificator id = new Identificator();
|
|
|
|
final Identificator id = new Identificator();
|
|
|
|
|
|
|
|
|
|
|
|
// count number of characters in text
|
|
|
|
// count number of characters in text
|
|
|
|
for (int i = 0; i < upperLimit; i++) id.inc(text.charAt(i));
|
|
|
|
for (int i = 0; i < upperLimit; i++) id.inc(text.charAt(i));
|
|
|
|
|
|
|
|
|
|
|
|
return id.getLanguage();
|
|
|
|
return id.getLanguage();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
public void inc(final char c) {
|
|
|
|
public void inc(final char c) {
|
|
|
|
if (!Character.isLetter(c)) return;
|
|
|
|
if (!Character.isLetter(c)) return;
|
|
|
|
Character cc = Character.toLowerCase(c);
|
|
|
|
final Character cc = Character.toLowerCase(c);
|
|
|
|
Integer i = letter.get(cc);
|
|
|
|
final AtomicInteger i = this.letter.get(cc);
|
|
|
|
if (i == null) {
|
|
|
|
if (i == null) {
|
|
|
|
letter.put(cc, 1);
|
|
|
|
this.letter.put(cc, new AtomicInteger(1));
|
|
|
|
} else {
|
|
|
|
} else {
|
|
|
|
letter.put(cc, i.intValue() + 1);
|
|
|
|
i.incrementAndGet();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
letters++;
|
|
|
|
this.letters++;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
public void add(String word) {
|
|
|
|
public void add(final String word) {
|
|
|
|
if (word == null) return;
|
|
|
|
if (word == null) return;
|
|
|
|
for (int i = 0; i < word.length(); i++) inc(word.charAt(i));
|
|
|
|
for (int i = 0; i < word.length(); i++) inc(word.charAt(i));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
public String getLanguage() {
|
|
|
|
public String getLanguage() {
|
|
|
|
|
|
|
|
|
|
|
|
if (language != null) return language; // don't compute that twice
|
|
|
|
if (this.language != null) return this.language; // don't compute that twice
|
|
|
|
if (letters == 0) return null; // not enough information available
|
|
|
|
if (this.letters == 0) return null; // not enough information available
|
|
|
|
|
|
|
|
|
|
|
|
final LanguageStatistics testStat = new LanguageStatistics("text");
|
|
|
|
final LanguageStatistics testStat = new LanguageStatistics("text");
|
|
|
|
|
|
|
|
|
|
|
|
// calculate percentage
|
|
|
|
// calculate percentage
|
|
|
|
Character character;
|
|
|
|
Character character;
|
|
|
|
Character maxChar = null;
|
|
|
|
Character maxChar = null;
|
|
|
|
int count = 0;
|
|
|
|
int count = 0;
|
|
|
|
int max = 0;
|
|
|
|
int max = 0;
|
|
|
|
for (Map.Entry<Character, Integer> e: letter.entrySet()) {
|
|
|
|
for (final Map.Entry<Character, AtomicInteger> e: this.letter.entrySet()) {
|
|
|
|
character = e.getKey();
|
|
|
|
character = e.getKey();
|
|
|
|
count = e.getValue().intValue();
|
|
|
|
count = e.getValue().intValue();
|
|
|
|
if (count > max) {
|
|
|
|
if (count > max) {
|
|
|
|
maxChar = character;
|
|
|
|
maxChar = character;
|
|
|
|
max = count;
|
|
|
|
max = count;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
testStat.put(character, ((float) 100) * ((float) count) / ((float) letters));
|
|
|
|
testStat.put(character, ((float) 100) * ((float) count) / (this.letters));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
// create list with relevant languages
|
|
|
|
// create list with relevant languages
|
|
|
|
final List<Integer> relevantLanguages = new Vector <Integer>();
|
|
|
|
final List<Integer> relevantLanguages = new Vector <Integer>();
|
|
|
|
for (int i = 0; i < languages.size(); i++) {
|
|
|
|
for (int i = 0; i < languages.size(); i++) {
|
|
|
|
|
|
|
|
|
|
|
|
// only languages that contain the most common character in the text will be tested
|
|
|
|
// only languages that contain the most common character in the text will be tested
|
|
|
|
if (languages.get(i).contains(maxChar)) {
|
|
|
|
if (languages.get(i).contains(maxChar)) {
|
|
|
|
relevantLanguages.add(i);
|
|
|
|
relevantLanguages.add(i);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
if (relevantLanguages.isEmpty()) return null;
|
|
|
|
if (relevantLanguages.isEmpty()) return null;
|
|
|
|
|
|
|
|
|
|
|
|
// compare characters in text with characters in statistics
|
|
|
|
// compare characters in text with characters in statistics
|
|
|
|
final float[] offsetList = new float[relevantLanguages.size()];
|
|
|
|
final float[] offsetList = new float[relevantLanguages.size()];
|
|
|
|
final int[] votesList = new int[relevantLanguages.size()];
|
|
|
|
final int[] votesList = new int[relevantLanguages.size()];
|
|
|
|
|
|
|
|
|
|
|
|
Iterator<Character> iter = testStat.keySet().iterator();
|
|
|
|
final Iterator<Character> iter = testStat.keySet().iterator();
|
|
|
|
float minimum;
|
|
|
|
float minimum;
|
|
|
|
float offset = 0;
|
|
|
|
float offset = 0;
|
|
|
|
float valueCharacter;
|
|
|
|
float valueCharacter;
|
|
|
|
int bestLanguage = -1;
|
|
|
|
int bestLanguage = -1;
|
|
|
|
float value;
|
|
|
|
float value;
|
|
|
|
|
|
|
|
|
|
|
|
while (iter.hasNext()) {
|
|
|
|
while (iter.hasNext()) {
|
|
|
|
minimum = 100.1f;
|
|
|
|
minimum = 100.1f;
|
|
|
|
character = iter.next();
|
|
|
|
character = iter.next();
|
|
|
@ -158,7 +159,7 @@ public final class Identificator {
|
|
|
|
}
|
|
|
|
}
|
|
|
|
votesList[bestLanguage] = ++votesList[bestLanguage];
|
|
|
|
votesList[bestLanguage] = ++votesList[bestLanguage];
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
// Now we can count how many votes each language got and how far it was away from the stats.
|
|
|
|
// Now we can count how many votes each language got and how far it was away from the stats.
|
|
|
|
// If 2 languages have the same amount of votes, the one with the smaller offset wins.
|
|
|
|
// If 2 languages have the same amount of votes, the one with the smaller offset wins.
|
|
|
|
int maxVotes = 0;
|
|
|
|
int maxVotes = 0;
|
|
|
@ -170,17 +171,17 @@ public final class Identificator {
|
|
|
|
bestLanguage = i;
|
|
|
|
bestLanguage = i;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
// Only return name of language of offset is smaller than 20%. This
|
|
|
|
// Only return name of language of offset is smaller than 20%. This
|
|
|
|
// prevents a language beeing reported that has won the voting, but
|
|
|
|
// prevents a language beeing reported that has won the voting, but
|
|
|
|
// is still not the right language.
|
|
|
|
// is still not the right language.
|
|
|
|
if (offset < 20) {
|
|
|
|
if (offset < 20) {
|
|
|
|
language = languages.get(relevantLanguages.get(bestLanguage)).getName();
|
|
|
|
this.language = languages.get(relevantLanguages.get(bestLanguage)).getName();
|
|
|
|
return language;
|
|
|
|
return this.language;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
return null;
|
|
|
|
return null;
|
|
|
|
|
|
|
|
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
}
|
|
|
|
}
|
|
|
|